envtest: Add tests for upgrade

Signed-off-by: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>

test/envtest/dlbdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle DlbDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.DlbDevicePluginSpec{
@@ -102,4 +97,14 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.DlbDevicePlugin{}
+
+		var image string
+
+		testUpgrade("dlb", dp, &image, nil)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+	})
 })

test/envtest/dsadeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("DsaDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle DsaDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.DsaDevicePluginSpec{
@@ -87,4 +82,15 @@ var _ = Describe("DsaDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.DsaDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("dsa", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/fpgadeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("FpgaDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle FpgaDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.FpgaDevicePluginSpec{
@@ -88,4 +83,15 @@ var _ = Describe("FpgaDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.FpgaDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("fpga", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/gpudeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("GpuDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle GpuDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.GpuDevicePluginSpec{
@@ -87,4 +82,15 @@ var _ = Describe("GpuDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.GpuDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("gpu", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/qatdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle QatDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.QatDevicePluginSpec{
@@ -106,4 +101,14 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.QatDevicePlugin{}
+
+		var image string
+
+		testUpgrade("qat", dp, &image, nil)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+	})
 })

test/envtest/sgxdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle SgxDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.SgxDevicePluginSpec{
@@ -88,4 +83,15 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.SgxDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("sgx", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,11 +17,17 @@ package envtest
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	apps "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2/klogr"
@@ -30,7 +36,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/intel/intel-device-plugins-for-kubernetes/deployments"
 	devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1"
+	ctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
 	dlbctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dlb"
 	dsactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dsa"
 	fpgactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga"
@@ -43,12 +51,16 @@ import (
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
 var (
-	cfg        *rest.Config
-	k8sClient  client.Client
-	k8sManager ctrl.Manager
-	testEnv    *envtest.Environment
-	ctx        context.Context
-	cancel     context.CancelFunc
+	cfg         *rest.Config
+	k8sClient   client.Client
+	k8sManager  ctrl.Manager
+	testEnv     *envtest.Environment
+	ctx         context.Context
+	cancel      context.CancelFunc
+	err         error
+	ns          = metav1.NamespaceSystem
+	version     = ctr.ImageMinVersion.String()
+	prevVersion = ctr.ImageMinVersion.WithMinor(ctr.ImageMinVersion.Minor() - 1).String()
 )
 
 func TestAPIs(t *testing.T) {
@@ -59,57 +71,203 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	logf.SetLogger(klogr.New())
-	ctx, cancel = context.WithCancel(context.TODO())
-
 	By("bootstrapping test environment")
+
+	logf.SetLogger(klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog)))
+
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{filepath.Join("..", "..", "deployments", "operator", "crd", "bases")},
 	}
-
-	var err error
 	cfg, err = testEnv.Start()
-	Expect(err).ToNot(HaveOccurred())
-	Expect(cfg).ToNot(BeNil())
 
-	err = devicepluginv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
+	Expect(err == nil && cfg != nil).To(BeTrue())
 
-	// +kubebuilder:scaffold:scheme
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 
-	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
-		Scheme: scheme.Scheme,
-	})
-	Expect(err).ToNot(HaveOccurred())
+	Expect(err == nil && k8sClient != nil).To(BeTrue())
 
-	withWebhook := true
-
-	err = gpuctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = sgxctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = qatctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = fpgactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = dsactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = dlbctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-
-	go func() {
-		err = k8sManager.Start(ctx)
-		Expect(err).ToNot(HaveOccurred())
-	}()
-
-	k8sClient = k8sManager.GetClient()
-	Expect(k8sClient).ToNot(BeNil())
+	Expect(devicepluginv1.AddToScheme(scheme.Scheme)).To(BeNil())
 
 }, 60)
 
 var _ = AfterSuite(func() {
-	cancel()
 	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).ToNot(HaveOccurred())
+
+	Expect(testEnv.Stop()).To(BeNil())
 })
+
+var _ = BeforeEach(func() {
+	up()
+})
+
+var _ = AfterEach(func() {
+	down()
+})
+
+func up() {
+	k8sManager, _ = ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme.Scheme})
+
+	withWebhook := true
+
+	Expect(dlbctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(dsactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(fpgactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(gpuctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(qatctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(sgxctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	ctx, cancel = context.WithCancel(context.TODO())
+
+	go func() {
+		Expect(k8sManager.Start(ctx)).To(BeNil())
+	}()
+
+	time.Sleep(time.Second)
+}
+
+func down() {
+	time.Sleep(time.Second)
+
+	ctx = context.TODO()
+
+	cancel()
+}
+
+func testUpgrade(name string, dp interface{}, pimage, pinitimage *string) {
+	down()
+
+	prefix := "intel/intel-" + name
+	image0 := prefix + "-plugin:" + prevVersion
+	initimage0 := prefix + "-initcontainer:" + prevVersion
+	image := prefix + "-plugin:" + version
+	initimage := prefix + "-initcontainer:" + version
+
+	*pimage = image
+
+	if pinitimage != nil {
+		*pinitimage = initimage
+	}
+
+	ds0 := makeDaemonSet(name, image0, initimage0)
+
+	Expect(k8sClient.Create(ctx, ds0)).To(BeNil())
+
+	dp0 := makeDevicePlugin(name, image0, initimage0)
+
+	Expect(k8sClient.Create(ctx, dp0)).To(BeNil())
+
+	up()
+
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, dp.(client.Object))).To(BeNil())
+
+	ds := &apps.DaemonSet{}
+
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "intel-" + name + "-plugin"}, ds)).To(BeNil())
+
+	Expect(ds.Spec.Template.Spec.Containers[0].Image == image).To(BeTrue())
+
+	if pinitimage != nil {
+		Expect(ds.Spec.Template.Spec.InitContainers[0].Image == initimage).To(BeTrue())
+	}
+
+	Expect(k8sClient.Delete(ctx, dp.(client.Object))).To(BeNil())
+}
+
+func makeDevicePlugin(name, image, initimage string) client.Object {
+	var obj client.Object
+
+	switch name {
+	case "dlb":
+		obj = &devicepluginv1.DlbDevicePlugin{
+			Spec: devicepluginv1.DlbDevicePluginSpec{
+				Image: image,
+			},
+		}
+	case "dsa":
+		obj = &devicepluginv1.DsaDevicePlugin{
+			Spec: devicepluginv1.DsaDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "fpga":
+		obj = &devicepluginv1.FpgaDevicePlugin{
+			Spec: devicepluginv1.FpgaDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "gpu":
+		obj = &devicepluginv1.GpuDevicePlugin{
+			Spec: devicepluginv1.GpuDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "qat":
+		obj = &devicepluginv1.QatDevicePlugin{
+			Spec: devicepluginv1.QatDevicePluginSpec{
+				Image: image,
+			},
+		}
+	case "sgx":
+		obj = &devicepluginv1.SgxDevicePlugin{
+			Spec: devicepluginv1.SgxDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	}
+
+	obj.SetName(name)
+
+	return obj
+}
+
+func makeDaemonSet(name, image, initimage string) *apps.DaemonSet {
+	var ds *apps.DaemonSet
+
+	initcontainerName := "intel-" + name + "-initcontainer"
+
+	switch name {
+	case "dlb":
+		ds = deployments.DLBPluginDaemonSet()
+	case "dsa":
+		ds = deployments.DSAPluginDaemonSet()
+		initcontainerName = "intel-idxd-config-initcontainer"
+	case "gpu":
+		ds = deployments.GPUPluginDaemonSet()
+	case "fpga":
+		ds = deployments.FPGAPluginDaemonSet()
+	case "qat":
+		ds = deployments.QATPluginDaemonSet()
+	case "sgx":
+		ds = deployments.SGXPluginDaemonSet()
+	}
+
+	ds.ObjectMeta.Namespace = ns
+	ds.Spec.Template.Spec.Containers[0].Image = image
+
+	if len(initimage) > 0 {
+		ds.Spec.Template.Spec.InitContainers = []corev1.Container{{
+			Name:  initcontainerName,
+			Image: initimage,
+		}}
+	}
+
+	yes := true
+	ds.OwnerReferences = []metav1.OwnerReference{{
+		APIVersion: "deviceplugin.intel.com/v1",
+		Kind:       strings.Title(name) + "DevicePlugin",
+		Name:       name,
+		Controller: &yes,
+		UID:        uuid.NewUUID(),
+	}}
+
+	return ds
+}