Simple Whereabouts Golang E2E Test

Signed-off-by: nicklesimba <simha.nikhil@gmail.com>

.github/workflows/test.yml

@@ -65,6 +65,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NUMBER_OF_COMPUTE_NODES: 5
+      NUMBER_OF_THRASH_ITER: 20
+      FILL_PERCENT_CAPACITY: 20
     steps:
       - name: Set up Go version
         uses: actions/setup-go@v1
@@ -80,5 +82,8 @@ jobs:
       - name: Get tools, setup KinD cluster test environment
         run: source hack/e2e-get-test-tools.sh && ./hack/e2e-setup-kind-cluster.sh --number-of-compute $NUMBER_OF_COMPUTE_NODES
 
+      - name: Clear test-cache
+        run: go clean -testcache
+
       - name: Execute E2E tests
-        run: NUMBER_OF_THRASH_ITER=20 FILL_PERCENT_CAPACITY=20 ./hack/e2e-test.sh --number-of-compute $NUMBER_OF_COMPUTE_NODES
+        run: go test ../e2e/.

e2e/e2e_test.go

@@ -0,0 +1,258 @@
+package whereabouts_e2e
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"testing"
+
+	"time"
+
+	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	core "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// Global Constants
+const (
+	testNetworkName   = "wa-nad"
+	wbNamespace       = "kube-system"
+	testNamespace     = "default"
+	timeout           = 5000
+	testImage         = "quay.io/dougbtv/alpine:latest"
+	ipv4TestRange     = "10.10.0.0/16"
+	ipv4RangePoolName = "10.10.0.0-16"
+	singlePodName     = "whereabouts-basic-test"
+	rsName            = "whereabouts-scale-test"
+	wbLabelEqual      = "tier=whereabouts-scale-test"
+	wbLabelColon      = "tier: whereabouts-scale-test"
+)
+
+// ClientInfo contains information given from k8s client
+type ClientInfo struct {
+	Client    kubernetes.Interface
+	NetClient netclient.K8sCniCncfIoV1Interface
+}
+
+func TestWhereaboutsE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "whereabouts-e2e")
+}
+
+var _ = Describe("Whereabouts functionality", func() {
+	Context("Test setup", func() {
+		// Declare variables
+		var (
+			err                 error
+			numComputeNodes     int
+			kubeconfig          *string
+			fillPercentCapacity int
+			numThrashIter       int
+			label               map[string]string
+			annotations         map[string]string
+			clientInfo          ClientInfo
+			clientSet           *kubernetes.Clientset
+			netClient           *netclient.K8sCniCncfIoV1Client
+			netAttachDef        *nettypes.NetworkAttachmentDefinition
+			config              *rest.Config
+			netStatus           []nettypes.NetworkStatus
+			pod                 *core.Pod
+		)
+
+		BeforeEach(func() {
+
+			config, err = clientcmd.BuildConfigFromFlags("", *kubeconfig)
+			Expect(err).To(BeNil())
+
+			// Create k8sclient and ClientInfo object
+			clientSet, err = kubernetes.NewForConfig(config)
+			Expect(err).To(BeNil())
+			netClient, err = netclient.NewForConfig(config)
+			Expect(err).To(BeNil())
+
+			clientInfo = ClientInfo{
+				Client:    clientSet,
+				NetClient: netClient,
+			}
+			netAttachDef = macvlanNetworkWithWhereaboutsIPAMNetwork()
+
+			label = make(map[string]string)
+			annotations = make(map[string]string)
+			annotations["k8s.v1.cni.cncf.io/networks"] = testNetworkName
+
+			// Create a net-attach-def
+			By("creating a NetworkAttachmentDefinition for whereabouts")
+			_, err = clientInfo.addNetAttachDef(netAttachDef)
+			Expect(err).To(BeNil())
+		})
+
+		AfterEach(func() {
+			Expect(clientInfo.delNetAttachDef(netAttachDef)).To(Succeed())
+		})
+
+		Context("Single pod tests", func() {
+			BeforeEach(func() {
+				By("creating a pod with whereabouts net-attach-def")
+				label["tier"] = singlePodName
+				pod = provisionPod(label, annotations, clientSet)
+			})
+
+			AfterEach(func() {
+				By("deleting pod with whereabouts net-attach-def")
+				deletePod(pod, clientSet)
+			})
+
+			It("allocates a single pod with the correct IP range", func() {
+				// Get net1 IP address from pod
+				By("checking pod IP is within whereabouts IPAM range")
+				secondaryIfaceIP := secondaryIfaceIPValue(pod, netStatus)
+				Expect(inRange(ipv4TestRange, secondaryIfaceIP)).To(BeTrue())
+			})
+		})
+	})
+})
+
+// AddNetAttachDef adds a net-attach-def into kubernetes
+// Returns a NAD object and an error variable
+func (c *ClientInfo) addNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) (*nettypes.NetworkAttachmentDefinition, error) {
+	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Create(context.TODO(), netattach, metav1.CreateOptions{})
+}
+
+// DelNetAttachDef removes a net-attach-def from kubernetes
+// Returns an error variable
+func (c *ClientInfo) delNetAttachDef(netattach *nettypes.NetworkAttachmentDefinition) error {
+	return c.NetClient.NetworkAttachmentDefinitions(netattach.ObjectMeta.Namespace).Delete(context.TODO(), netattach.Name, metav1.DeleteOptions{})
+}
+
+// Returns a network attachment definition object configured by provided parameters
+func generateNetAttachDefSpec(name, namespace, config string) *nettypes.NetworkAttachmentDefinition {
+	return &nettypes.NetworkAttachmentDefinition{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "NetworkAttachmentDefinition",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: nettypes.NetworkAttachmentDefinitionSpec{
+			Config: config,
+		},
+	}
+}
+
+// Returns a network attachment definition object configured for whereabouts
+func macvlanNetworkWithWhereaboutsIPAMNetwork() *nettypes.NetworkAttachmentDefinition {
+	macvlanConfig := `{
+        "cniVersion": "0.3.0",
+      	"disableCheck": true,
+        "plugins": [
+            {
+                "type": "macvlan",
+              	"master": "eth0",
+              	"mode": "bridge",
+              	"ipam": {
+                    "type": "whereabouts",
+                    "leader_lease_duration": 1500,
+                    "leader_renew_deadline": 1000,
+                    "leader_retry_period": 500,
+                    "range": "10.10.0.0/16",
+                    "log_level": "debug",
+                    "log_file": "/tmp/wb"
+              	}
+            }
+        ]
+    }`
+	return generateNetAttachDefSpec(testNetworkName, testNamespace, macvlanConfig)
+}
+
+// returns a pod object and creates the pod in kubernetes
+func provisionPod(label, annotations map[string]string, clientSet *kubernetes.Clientset) *core.Pod {
+	// Create pod
+	pod := podObject(label, annotations)
+	pod, err := clientSet.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+	Expect(err).To(BeNil())
+
+	// Wait for pod to become ready
+	Expect(WaitForPodReady(clientSet, pod.Namespace, pod.Name, 10*time.Second)).To(Succeed())
+
+	// Update pod object
+	pod, err = clientSet.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+	Expect(err).To(BeNil())
+
+	return pod
+}
+
+// takes in a pod object and deletes it in kubernetes - the function also waits for the pod to explicitly not exist (aka finish terminating)
+func deletePod(pod *core.Pod, clientSet *kubernetes.Clientset) {
+	Expect(clientSet.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})).To(Succeed())
+	Eventually(func() error {
+		_, err := clientSet.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+		return err
+	}, 20*time.Second, time.Second).ShouldNot(BeNil()) // eventually, to make this cleaner, instead of this, check if error is NotFound/IsNotFound
+}
+
+// Takes in a label and whereabouts annotations
+// Returns a pod object with a whereabouts annotation
+func podObject(label, annotations map[string]string) *core.Pod {
+	return &core.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "wa-e2e-pod",
+			Namespace:   testNamespace,
+			Labels:      label,
+			Annotations: annotations,
+		},
+		Spec: core.PodSpec{
+			Containers: []core.Container{
+				{
+					Name:    "samplepod",
+					Command: containerCmd(),
+					Image:   testImage,
+				},
+			},
+		},
+	}
+}
+
+func containerCmd() []string {
+	return []string{"/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"}
+}
+
+func filterNetworkStatus(networkStatuses []nettypes.NetworkStatus, predicate func(nettypes.NetworkStatus) bool) *nettypes.NetworkStatus {
+	for i, networkStatus := range networkStatuses {
+		if predicate(networkStatus) {
+			return &networkStatuses[i]
+		}
+	}
+	return nil
+}
+
+func secondaryIfaceIPValue(pod *core.Pod, netStatus []nettypes.NetworkStatus) string {
+	podNetStatus, found := pod.Annotations[nettypes.NetworkStatusAnnot]
+	Expect(found).To(BeTrue(), "expected the pod to have a `networks-status` annotation")
+	Expect(json.Unmarshal([]byte(podNetStatus), &netStatus)).To(Succeed())
+	Expect(netStatus).NotTo(BeEmpty())
+	// Check if interface is net1 and if IP is in range
+	secondaryInterfaceNetworkStatus := filterNetworkStatus(netStatus, func(status nettypes.NetworkStatus) bool {
+		return status.Interface == "net1"
+	})
+	Expect(secondaryInterfaceNetworkStatus.IPs).NotTo(BeEmpty())
+	secondaryIfaceIP := secondaryInterfaceNetworkStatus.IPs[0]
+
+	return secondaryIfaceIP
+}
+
+func inRange(cidr string, ip string) bool {
+	_, cidrRange, err := net.ParseCIDR(cidr)
+	Expect(err).To(BeNil())
+	ipInRangeCandidate := net.ParseIP(ip)
+
+	return cidrRange.Contains(ipInRangeCandidate)
+}

e2e/pod_status.go

@@ -0,0 +1,78 @@
+// This code is based on code from the following repository
+// https://github.com/bcreane/k8sutils
+
+package whereabouts_e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+)
+
+// return a condition function that indicates whether the given pod is
+// currently running
+func isPodRunning(cs *kubernetes.Clientset, podName, namespace string) wait.ConditionFunc {
+	return func() (bool, error) {
+		fmt.Printf(".") // progress bar!
+
+		pod, err := cs.CoreV1().Pods(namespace).Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		switch pod.Status.Phase {
+		case v1.PodRunning:
+			return true, nil
+		case v1.PodFailed:
+			return false, errors.New("pod failed")
+		case v1.PodSucceeded:
+			return false, errors.New("pod succeeded")
+		}
+
+		return false, nil
+	}
+}
+
+// Poll up to timeout seconds for pod to enter steady state (running or succeeded state).
+// Returns an error if the pod never enters a steady state.
+func WaitForPodReady(cs *kubernetes.Clientset, namespace, podName string, timeout time.Duration) error {
+	return wait.PollImmediate(time.Second, timeout, isPodRunning(cs, podName, namespace))
+}
+
+// Returns the list of currently scheduled or running pods in `namespace` with the given selector
+func ListPods(cs *kubernetes.Clientset, namespace, selector string) (*v1.PodList, error) {
+	listOptions := metav1.ListOptions{LabelSelector: selector}
+	podList, err := cs.CoreV1().Pods(namespace).List(context.Background(), listOptions)
+
+	if err != nil {
+		return nil, err
+	}
+	return podList, nil
+}
+
+// Wait up to timeout seconds for all pods in 'namespace' with given 'selector' to enter provided state
+// If no pods are found, return nil.
+func WaitForPodBySelector(cs *kubernetes.Clientset, namespace, selector string, timeout int) error {
+	podList, err := ListPods(cs, namespace, selector)
+	if err != nil {
+		return err
+	}
+
+	// if there are no pods that match
+	if len(podList.Items) == 0 {
+		return nil
+	}
+
+	for _, pod := range podList.Items {
+		if err := WaitForPodReady(cs, namespace, pod.Name, time.Duration(timeout)*time.Second); err != nil {
+			return err
+		}
+	}
+	return nil
+}