From 8ca5200c4377ce37160a3dfda97fb72c0ac5e8ea Mon Sep 17 00:00:00 2001 From: Hyeongju Johannes Lee Date: Wed, 9 Mar 2022 14:48:20 +0200 Subject: [PATCH] envtest: add more tests for DaemonSets Add tests if args, initImage, nodeSelector are set as intended Signed-off-by: Hyeongju Johannes Lee --- .../dlbdeviceplugin_controller_test.go | 47 ++++++++--- .../dsadeviceplugin_controller_test.go | 79 ++++++++++++++++-- .../fpgadeviceplugin_controller_test.go | 61 ++++++++++++-- .../gpudeviceplugin_controller_test.go | 66 +++++++++++++-- .../iaadeviceplugin_controller_test.go | 82 +++++++++++++++++-- .../qatdeviceplugin_controller_test.go | 63 ++++++++++++-- .../sgxdeviceplugin_controller_test.go | 67 +++++++++++++-- 7 files changed, 418 insertions(+), 47 deletions(-) diff --git a/test/envtest/dlbdeviceplugin_controller_test.go b/test/envtest/dlbdeviceplugin_controller_test.go index a98b2386..3adaaf06 100644 --- a/test/envtest/dlbdeviceplugin_controller_test.go +++ b/test/envtest/dlbdeviceplugin_controller_test.go @@ -16,10 +16,12 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,9 +36,8 @@ var _ = Describe("DlbDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle DlbDevicePlugin objects correctly", func() { spec := devicepluginv1.DlbDevicePluginSpec{ - Image: "testimage", + Image: "dlb-testimage", NodeSelector: map[string]string{"feature.node.kubernetes.io/dlb": "true"}, - LogLevel: 4, } key := types.NamespacedName{ @@ -60,14 +61,15 @@ var _ = Describe("DlbDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - Eventually(func() devicepluginv1.DlbDevicePluginSpec { - _ = k8sClient.Get(context.Background(), key, fetched) - return fetched.Spec - }, timeout, interval).Should(Equal(spec)) + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dlb-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) - By("updating image name successfully") + By("updating DlbDevicePlugin successfully") updatedImage := "updated-dlb-testimage" - updatedNodeSelector := map[string]string{"updated-dlb-nodeSelector": "true"} + updatedNodeSelector := map[string]string{"updated-dlb-nodeselector": "true"} updatedLogLevel := 3 fetched.Spec.Image = updatedImage fetched.Spec.NodeSelector = updatedNodeSelector @@ -78,11 +80,30 @@ var _ = Describe("DlbDevicePlugin Controller", func() { Eventually(func() devicepluginv1.DlbDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) return fetchedUpdated.Spec - }, timeout, interval).Should(Equal( - devicepluginv1.DlbDevicePluginSpec{ - Image: updatedImage, - NodeSelector: updatedNodeSelector, - LogLevel: updatedLogLevel})) + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dlb-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + } + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating DlbDevicePlugin with different values successfully") + updatedNodeSelector = map[string]string{} + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dlb-plugin"}, ds) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting DlbDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/dsadeviceplugin_controller_test.go b/test/envtest/dsadeviceplugin_controller_test.go index d2075df4..144ff8f5 100644 --- a/test/envtest/dsadeviceplugin_controller_test.go +++ b/test/envtest/dsadeviceplugin_controller_test.go @@ -16,10 +16,13 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,7 +37,9 @@ var _ = Describe("DsaDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle DsaDevicePlugin objects correctly", func() { spec := devicepluginv1.DsaDevicePluginSpec{ - Image: "testimage", + Image: "testimage", + InitImage: "testinitimage", + NodeSelector: map[string]string{"dsa-nodeselector": "true"}, } key := types.NamespacedName{ @@ -58,16 +63,80 @@ var _ = Describe("DsaDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - By("updating image name successfully") + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dsa-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + + By("updating DsaDevicePlugin successfully") updatedImage := "updated-dsa-testimage" + updatedInitImage := "updated-dsa-testinitimage" + updatedProvisioningConfig := "updated-dsa-provisioningconfig" + updatedLogLevel := 2 + updatedSharedDevNum := 42 + updatedNodeSelector := map[string]string{"updated-dsa-nodeselector": "true"} + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.ProvisioningConfig = updatedProvisioningConfig + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.SharedDevNum = updatedSharedDevNum + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.DsaDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.DsaDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dsa-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-shared-dev-num", + strconv.Itoa(updatedSharedDevNum), + } + mode := int32(420) + expectedVolume := v1.Volume{ + Name: "intel-dsa-config-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: updatedProvisioningConfig}, + DefaultMode: &mode, + }, + }, + } + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(expectedVolume)) + + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating DsaDevicePlugin with different values successfully") + updatedInitImage = "" + updatedProvisioningConfig = "" + updatedNodeSelector = map[string]string{} + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.ProvisioningConfig = updatedProvisioningConfig + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dsa-plugin"}, ds) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(expectedVolume)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting DsaDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/fpgadeviceplugin_controller_test.go b/test/envtest/fpgadeviceplugin_controller_test.go index 4f481269..3f8c2b4c 100644 --- a/test/envtest/fpgadeviceplugin_controller_test.go +++ b/test/envtest/fpgadeviceplugin_controller_test.go @@ -16,10 +16,12 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,8 +36,9 @@ var _ = Describe("FpgaDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle FpgaDevicePlugin objects correctly", func() { spec := devicepluginv1.FpgaDevicePluginSpec{ - Image: "testimage", - InitImage: "testinitimage", + Image: "fpga-testimage", + InitImage: "fpga-testinitimage", + NodeSelector: map[string]string{"fpga-nodeselector": "true"}, } key := types.NamespacedName{ @@ -59,16 +62,62 @@ var _ = Describe("FpgaDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - By("updating image name successfully") + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-fpga-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + + By("updating FpgaDevicePlugin successfully") updatedImage := "updated-fpga-testimage" + updatedInitImage := "updated-fpga-testinitimage" + updatedLogLevel := 2 + updatedMode := "region" + updatedNodeSelector := map[string]string{"updated-fpga-nodeselector": "true"} + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.Mode = updatedMode + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.FpgaDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.FpgaDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-fpga-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-mode", + updatedMode, + } + + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating FpgaDevicePlugin with different values successfully") + updatedNodeSelector = map[string]string{} + fetched.Spec.NodeSelector = updatedNodeSelector + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-fpga-plugin"}, ds) + time.Sleep(interval) + + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting FpgaDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/gpudeviceplugin_controller_test.go b/test/envtest/gpudeviceplugin_controller_test.go index bc9b6c32..2b638258 100644 --- a/test/envtest/gpudeviceplugin_controller_test.go +++ b/test/envtest/gpudeviceplugin_controller_test.go @@ -16,10 +16,12 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,7 +36,9 @@ var _ = Describe("GpuDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle GpuDevicePlugin objects correctly", func() { spec := devicepluginv1.GpuDevicePluginSpec{ - Image: "testimage", + Image: "gpu-testimage", + InitImage: "gpu-testinitimage", + NodeSelector: map[string]string{"gpu-nodeselector": "true"}, } key := types.NamespacedName{ @@ -58,16 +62,66 @@ var _ = Describe("GpuDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - By("updating image name successfully") - updatedImage := "updated-testimage" + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-gpu-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + + By("updating GpuDevicePlugin successfully") + updatedImage := "updated-gpu-testimage" + updatedInitImage := "updated-gpu-testinitimage" + updatedLogLevel := 2 + updatedSharedDevNum := 42 + updatedNodeSelector := map[string]string{"updated-gpu-nodeselector": "true"} + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.SharedDevNum = updatedSharedDevNum + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.GpuDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.GpuDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-gpu-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-shared-dev-num", + strconv.Itoa(updatedSharedDevNum), + "-allocation-policy", + "none", + } + + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating GpuDevicePlugin with different values successfully") + updatedInitImage = "" + updatedNodeSelector = map[string]string{} + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-gpu-plugin"}, ds) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting GpuDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/iaadeviceplugin_controller_test.go b/test/envtest/iaadeviceplugin_controller_test.go index b4d9ab32..384adcf0 100644 --- a/test/envtest/iaadeviceplugin_controller_test.go +++ b/test/envtest/iaadeviceplugin_controller_test.go @@ -16,10 +16,13 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,7 +37,10 @@ var _ = Describe("IaaDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle IaaDevicePlugin objects correctly", func() { spec := devicepluginv1.IaaDevicePluginSpec{ - Image: "testimage", + Image: "testimage", + InitImage: "testinitimage", + NodeSelector: map[string]string{"iaa-nodeselector": "true", + "kubernetes.io/arch": "amd64"}, } key := types.NamespacedName{ @@ -60,16 +66,82 @@ var _ = Describe("IaaDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - By("updating image name successfully") + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-iaa-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + + By("updating IaaDevicePlugin successfully") updatedImage := "updated-iaa-testimage" + updatedInitImage := "updated-iaa-testinitimage" + updatedProvisioningConfig := "updated-iaa-provisioningconfig" fetched.Spec.Image = updatedImage + updatedLogLevel := 2 + updatedSharedDevNum := 42 + updatedNodeSelector := map[string]string{"updated-iaa-nodeselector": "true", + "kubernetes.io/arch": "amd64"} + + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.ProvisioningConfig = updatedProvisioningConfig + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.SharedDevNum = updatedSharedDevNum + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.IaaDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.IaaDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-iaa-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-shared-dev-num", + strconv.Itoa(updatedSharedDevNum), + } + mode := int32(420) + expectedVolume := v1.Volume{ + Name: "intel-iaa-config-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: updatedProvisioningConfig}, + DefaultMode: &mode, + }, + }, + } + + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(expectedVolume)) + + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating IaaDevicePlugin with different values successfully") + updatedInitImage = "" + updatedNodeSelector = map[string]string{} + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-iaa-plugin"}, ds) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(expectedVolume)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting IaaDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/qatdeviceplugin_controller_test.go b/test/envtest/qatdeviceplugin_controller_test.go index c5bc3123..bce70ea6 100644 --- a/test/envtest/qatdeviceplugin_controller_test.go +++ b/test/envtest/qatdeviceplugin_controller_test.go @@ -16,10 +16,12 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,7 +36,8 @@ var _ = Describe("QatDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle QatDevicePlugin objects correctly", func() { spec := devicepluginv1.QatDevicePluginSpec{ - Image: "testimage", + Image: "qat-testimage", + NodeSelector: map[string]string{"qat-nodeselector": "true"}, } key := types.NamespacedName{ @@ -63,6 +66,12 @@ var _ = Describe("QatDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + By("copy annotations successfully") Expect(&(fetched.Annotations) == &annotations).ShouldNot(BeTrue()) Eventually(fetched.Annotations).Should(Equal(annotations)) @@ -77,16 +86,58 @@ var _ = Describe("QatDevicePlugin Controller", func() { return updated.Annotations }, timeout, interval).Should(Equal(updatedAnnotations)) - By("updating image name successfully") - updatedImage := "updated-testimage" + By("updating QatDevicePlugin successfully") + updatedImage := "updated-qat-testimage" + updatedLogLevel := 2 + updatedDpdkDriver := "igb_uio" + updatedKernelVfDrivers := "c3xxxvf" + updatedMaxNumDevices := 16 + updatedNodeSelector := map[string]string{"updated-qat-nodeselector": "true"} + fetched.Spec.Image = updatedImage + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.DpdkDriver = updatedDpdkDriver + fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)} + fetched.Spec.MaxNumDevices = updatedMaxNumDevices + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.QatDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.QatDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-dpdk-driver", + updatedDpdkDriver, + "-kernel-vf-drivers", + updatedKernelVfDrivers, + "-max-num-devices", + strconv.Itoa(updatedMaxNumDevices), + } + + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating QatDevicePlugin with different values successfully") + updatedNodeSelector = map[string]string{} + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting QatDevicePlugin successfully") Eventually(func() error { diff --git a/test/envtest/sgxdeviceplugin_controller_test.go b/test/envtest/sgxdeviceplugin_controller_test.go index 1e0bd238..f0eabd6e 100644 --- a/test/envtest/sgxdeviceplugin_controller_test.go +++ b/test/envtest/sgxdeviceplugin_controller_test.go @@ -16,10 +16,12 @@ package envtest import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,8 +36,9 @@ var _ = Describe("SgxDevicePlugin Controller", func() { Context("Basic CRUD operations", func() { It("should handle SgxDevicePlugin objects correctly", func() { spec := devicepluginv1.SgxDevicePluginSpec{ - Image: "sgx-testimage", - InitImage: "sgx-testinitimage", + Image: "sgx-testimage", + InitImage: "sgx-testinitimage", + NodeSelector: map[string]string{"sgx-nodeselector": "true"}, } key := types.NamespacedName{ @@ -59,16 +62,68 @@ var _ = Describe("SgxDevicePlugin Controller", func() { return len(fetched.Status.ControlledDaemonSet.UID) > 0 }, timeout, interval).Should(BeTrue()) - By("updating image name successfully") + By("checking DaemonSet is created successfully") + ds := &apps.DaemonSet{} + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-sgx-plugin"}, ds) + Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) + + By("updating SgxDevicePlugin successfully") updatedImage := "updated-sgx-testimage" + updatedInitImage := "updated-sgx-testinitimage" + updatedLogLevel := 2 + updatedEnclaveLimit := 2 + updatedProvisionLimit := 2 + updatedNodeSelector := map[string]string{"updated-sgx-nodeselector": "true"} + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.LogLevel = updatedLogLevel + fetched.Spec.EnclaveLimit = updatedEnclaveLimit + fetched.Spec.ProvisionLimit = updatedProvisionLimit + fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.SgxDevicePlugin{} - Eventually(func() string { + Eventually(func() devicepluginv1.SgxDevicePluginSpec { _ = k8sClient.Get(context.Background(), key, fetchedUpdated) - return fetchedUpdated.Spec.Image - }, timeout, interval).Should(Equal(updatedImage)) + return fetchedUpdated.Spec + }, timeout, interval).Should(Equal(fetched.Spec)) + time.Sleep(interval) + + By("checking DaemonSet is updated successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-sgx-plugin"}, ds) + + expectArgs := []string{ + "-v", + strconv.Itoa(updatedLogLevel), + "-enclave-limit", + strconv.Itoa(updatedEnclaveLimit), + "-provision-limit", + strconv.Itoa(updatedProvisionLimit), + } + + Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) + + By("updating SgxDevicePlugin with different values successfully") + updatedInitImage = "" + updatedNodeSelector = map[string]string{} + fetched.Spec.InitImage = updatedInitImage + fetched.Spec.NodeSelector = updatedNodeSelector + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + time.Sleep(interval) + + By("checking DaemonSet is updated with different values successfully") + _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-sgx-plugin"}, ds) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting SgxDevicePlugin successfully") Eventually(func() error {