Merge pull request #857 from ozhuraki/operator-upgrade

operator: Support upgrade of plugins

cmd/operator/README.md

@@ -4,6 +4,7 @@ Table of Contents
 
 * [Introduction](#introduction)
 * [Installation](#installation)
+* [Upgrade](#upgrade)
 * [Known issues](#known-issues)
 
 ## Introduction
@@ -145,6 +146,19 @@ In this case, create a new kustomization with the necessary resources
 that passes the desired device types to the operator using `--device`
 command line argument multiple times.
 
+## Upgrade
+
+The upgrade of the deployed plugins can be done by simply installing a new release of the operator.
+
+The operator auto-upgrades operator-managed plugins (CR images and thus corresponding deployed daemonsets) to the current release of the operator.
+
+The [registry-url]/[namespace]/[image] are kept intact on the upgrade.
+
+No upgrade is done for:
+
+- Non-operator managed deployments
+- Operator deployments without numeric tags
+
 ## Known issues
 
 When the operator is run with leader election enabled, that is with the option

cmd/operator/main.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -101,7 +101,7 @@ func main() {
 		pm                    *patcher.Manager
 	)
 
-	ctrl.SetLogger(klogr.New())
+	ctrl.SetLogger(klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog)))
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&devicePluginNamespace, "deviceplugin-namespace", metav1.NamespaceSystem, "The namespace where deviceplugin daemonsets are created")

deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml

@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fpgadeviceplugin
+  name: intel-fpga-plugin
   namespace: system
   labels:
     app: intel-fpga-plugin

deployments/fpga_plugin/overlays/region/mode-region.yaml

@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fpgadeviceplugin
+  name: intel-fpga-plugin
   namespace: system
 spec:
   template:

pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// dlbdevicepluginlog is for logging in this package.
 	dlbdevicepluginlog = logf.Log.WithName("dlbdeviceplugin-resource")
 
-	dlbMinVersion = version.MustParseSemantic(imageMinVersion)
+	dlbMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for DlbDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// dsadevicepluginlog is for logging in this package.
 	dsadevicepluginlog = logf.Log.WithName("dsadeviceplugin-resource")
 
-	dsaMinVersion = version.MustParseSemantic(imageMinVersion)
+	dsaMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for DsaDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// fpgadevicepluginlog is for logging in this package.
 	fpgadevicepluginlog = logf.Log.WithName("fpgadeviceplugin-resource")
 
-	fpgaMinVersion = version.MustParseSemantic(imageMinVersion)
+	fpgaMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for FpgaDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// gpudevicepluginlog is for logging in this package.
 	gpudevicepluginlog = logf.Log.WithName("gpudeviceplugin-resource")
 
-	gpuMinVersion = version.MustParseSemantic(imageMinVersion)
+	gpuMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for GpuDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// qatdevicepluginlog is for logging in this package.
 	qatdevicepluginlog = logf.Log.WithName("qatdeviceplugin-resource")
 
-	qatMinVersion = version.MustParseSemantic(imageMinVersion)
+	qatMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for QatDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go

@@ -17,7 +17,6 @@ package v1
 import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -33,7 +32,7 @@ var (
 	// sgxdevicepluginlog is for logging in this package.
 	sgxdevicepluginlog = logf.Log.WithName("sgxdeviceplugin-resource")
 
-	sgxMinVersion = version.MustParseSemantic(imageMinVersion)
+	sgxMinVersion = controllers.ImageMinVersion
 )
 
 // SetupWebhookWithManager sets up a webhook for SgxDevicePlugin custom resources.

pkg/apis/deviceplugin/v1/webhook_common.go

@@ -22,9 +22,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 )
 
-// common constants for webhooks.
-const imageMinVersion string = "0.23.0"
-
 // common functions for webhooks
 
 func validatePluginImage(image, expectedImageName string, expectedMinVersion *version.Version) error {

pkg/controllers/dlb/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.DlbDevicePlugin{}
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.DlbDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, nil)
+}
+
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
 	var list devicepluginv1.DlbDevicePluginList
 	if err := clnt.List(ctx, &list); err != nil {

pkg/controllers/dsa/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -69,6 +69,11 @@ func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.DsaDevicePlugin{}
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.DsaDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
+}
+
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
 	var list devicepluginv1.DsaDevicePluginList
 	if err := clnt.List(ctx, &list); err != nil {

pkg/controllers/fpga/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.FpgaDevicePlugin{}
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.FpgaDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
+}
+
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
 	var list devicepluginv1.FpgaDevicePluginList
 	if err := clnt.List(ctx, &list); err != nil {

pkg/controllers/fpga/controller_test.go

@@ -44,7 +44,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: c.ns,
-			Name:      "fpgadeviceplugin",
+			Name:      "intel-fpga-plugin",
 			Labels: map[string]string{
 				"app": appLabel,
 			},

pkg/controllers/gpu/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -70,6 +70,11 @@ func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.GpuDevicePlugin{}
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.GpuDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
+}
+
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
 	var list devicepluginv1.GpuDevicePluginList
 	if err := clnt.List(ctx, &list); err != nil {

pkg/controllers/qat/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.QatDevicePlugin{}
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.QatDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, nil)
+}
+
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
 	var list devicepluginv1.QatDevicePluginList
 	if err := clnt.List(ctx, &list); err != nil {

pkg/controllers/reconciler.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,22 +17,27 @@ package controllers
 
 import (
 	"context"
+	"strings"
 	"sync"
 
 	"github.com/go-logr/logr"
+	"github.com/google/go-cmp/cmp"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/diff"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 var (
-	bKeeper = &bookKeeper{}
+	bKeeper         = &bookKeeper{}
+	ImageMinVersion = versionutil.MustParseSemantic("0.23.0")
 )
 
 func init() {
@@ -97,6 +102,7 @@ type DevicePluginController interface {
 	NewDaemonSet(devicePlugin client.Object) *apps.DaemonSet
 	UpdateDaemonSet(client.Object, *apps.DaemonSet) (updated bool)
 	UpdateStatus(client.Object, *apps.DaemonSet, []string) (updated bool, err error)
+	Upgrade(ctx context.Context, obj client.Object) (upgrade bool)
 }
 
 type reconciler struct {
@@ -159,6 +165,33 @@ func (r *reconciler) createObjects(ctx context.Context,
 	return result, nil
 }
 
+func UpgradeImages(image *string, initimage *string) (upgrade bool) {
+	for _, s := range []*string{image, initimage} {
+		if s == nil {
+			continue
+		}
+
+		if parts := strings.SplitN(*s, ":", 2); len(parts) == 2 && len(parts[0]) > 0 {
+			name, version := parts[0], parts[1]
+			if ver, err := versionutil.ParseSemantic(version); err == nil && ver.LessThan(ImageMinVersion) {
+				*s = name + ":" + ImageMinVersion.String()
+				upgrade = true
+			}
+		}
+	}
+
+	return upgrade
+}
+
+func upgrade(ctx context.Context, r *reconciler, devicePlugin client.Object) {
+	if r.controller.Upgrade(ctx, devicePlugin) {
+		if err := r.Update(ctx, devicePlugin); err != nil {
+			log := log.FromContext(ctx)
+			log.Error(err, "unable to update devicePlugin")
+		}
+	}
+}
+
 // Reconcile reconciles a device plugin object.
 func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
@@ -182,6 +215,8 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return result, err
 	}
 
+	upgrade(ctx, r, devicePlugin)
+
 	// Create a daemon set for the plugin if it doesn't exist.
 	if len(childDaemonSets.Items) == 0 {
 		return r.createDaemonSet(ctx, devicePlugin, log)
@@ -189,8 +224,12 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	ds := &childDaemonSets.Items[0]
 
+	ds0 := ds.DeepCopy()
+
 	// Synchronize the DaemonSet with its owner.
 	if r.controller.UpdateDaemonSet(devicePlugin, ds) {
+		log.Info("", cmp.Diff(ds0.Spec.Template.Spec, ds.Spec.Template.Spec, diff.IgnoreUnset()))
+
 		if err := r.Update(ctx, ds); err != nil {
 			log.Error(err, "unable to update DaemonSet", "DaemonSet", ds)
 			return ctrl.Result{}, err

pkg/controllers/reconciler_test.go

@@ -0,0 +1,68 @@
+// Copyright 2022 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+	"testing"
+)
+
+func TestUpgrade(test *testing.T) {
+	tests := []struct {
+		image             string
+		initimage         string
+		expectedImage     string
+		expectedInitimage string
+		upgrade           bool
+	}{
+		{
+			image:             "intel/intel-dsa-plugin:0.22.0",
+			expectedImage:     "intel/intel-dsa-plugin:0.23.0",
+			initimage:         "intel/intel-idxd-config-initcontainer:0.22.0",
+			expectedInitimage: "intel/intel-idxd-config-initcontainer:0.23.0",
+			upgrade:           true,
+		},
+		{
+			image:             "intel/intel-dsa-plugin:0.23.0",
+			expectedImage:     "intel/intel-dsa-plugin:0.23.0",
+			initimage:         "intel/intel-idxd-config-initcontainer:0.23.0",
+			expectedInitimage: "intel/intel-idxd-config-initcontainer:0.23.0",
+			upgrade:           false,
+		},
+		{
+			image:             "intel/intel-dsa-plugin:latest",
+			expectedImage:     "intel/intel-dsa-plugin:latest",
+			initimage:         "intel/intel-idxd-config-initcontainer:latest",
+			expectedInitimage: "intel/intel-idxd-config-initcontainer:latest",
+			upgrade:           false,
+		},
+		{
+			image:             "intel/intel-dsa-plugin",
+			expectedImage:     "intel/intel-dsa-plugin",
+			initimage:         "intel/intel-idxd-config-initcontainer",
+			expectedInitimage: "intel/intel-idxd-config-initcontainer",
+			upgrade:           false,
+		},
+	}
+
+	for _, t := range tests {
+		upgrade := UpgradeImages(&t.image, &t.initimage)
+
+		if !(upgrade == t.upgrade && t.image == t.expectedImage && t.initimage == t.expectedInitimage) {
+			test.Errorf("expectedUpgrade: %v, received: %v", t.upgrade, upgrade)
+			test.Errorf("expectedImage: %s, received: %s", t.expectedImage, t.image)
+			test.Errorf("expectedInitimage: %s, received: %s", t.expectedInitimage, t.initimage)
+		}
+	}
+}

pkg/controllers/sgx/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -62,6 +62,11 @@ type controller struct {
 	ns     string
 }
 
+func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
+	dp := obj.(*devicepluginv1.SgxDevicePlugin)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
+}
+
 func (c *controller) CreateEmptyObject() client.Object {
 	return &devicepluginv1.SgxDevicePlugin{}
 }
@@ -134,6 +139,18 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	return daemonSet
 }
 
+func removeVolume(volumes []v1.Volume, name string) []v1.Volume {
+	newVolumes := []v1.Volume{}
+
+	for _, volume := range volumes {
+		if volume.Name != name {
+			newVolumes = append(newVolumes, volume)
+		}
+	}
+
+	return newVolumes
+}
+
 func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) {
 	dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
@@ -142,6 +159,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if dp.Spec.InitImage == "" {
+		if ds.Spec.Template.Spec.InitContainers != nil {
+			ds.Spec.Template.Spec.InitContainers = nil
+			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "nfd-source-hooks")
+			updated = true
+		}
+	} else {
+		setInitContainer(&ds.Spec.Template.Spec, dp.Spec.InitImage)
+		updated = true
+	}
+
 	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector

test/envtest/dlbdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle DlbDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.DlbDevicePluginSpec{
@@ -102,4 +97,14 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.DlbDevicePlugin{}
+
+		var image string
+
+		testUpgrade("dlb", dp, &image, nil)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+	})
 })

test/envtest/dsadeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("DsaDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle DsaDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.DsaDevicePluginSpec{
@@ -87,4 +82,15 @@ var _ = Describe("DsaDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.DsaDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("dsa", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/fpgadeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("FpgaDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle FpgaDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.FpgaDevicePluginSpec{
@@ -88,4 +83,15 @@ var _ = Describe("FpgaDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.FpgaDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("fpga", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/gpudeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("GpuDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle GpuDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.GpuDevicePluginSpec{
@@ -87,4 +82,15 @@ var _ = Describe("GpuDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.GpuDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("gpu", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/qatdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle QatDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.QatDevicePluginSpec{
@@ -106,4 +101,14 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.QatDevicePlugin{}
+
+		var image string
+
+		testUpgrade("qat", dp, &image, nil)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+	})
 })

test/envtest/sgxdeviceplugin_controller_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,11 +31,6 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	AfterEach(func() {
-		time.Sleep(time.Second * 2)
-
-	})
-
 	Context("Basic CRUD operations", func() {
 		It("should handle SgxDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.SgxDevicePluginSpec{
@@ -88,4 +83,15 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})
+
+	It("upgrades", func() {
+		dp := &devicepluginv1.SgxDevicePlugin{}
+
+		var image, initimage string
+
+		testUpgrade("sgx", dp, &image, &initimage)
+
+		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
+	})
 })

test/envtest/suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,11 +17,17 @@ package envtest
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	apps "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2/klogr"
@@ -30,7 +36,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/intel/intel-device-plugins-for-kubernetes/deployments"
 	devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1"
+	ctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
 	dlbctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dlb"
 	dsactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dsa"
 	fpgactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga"
@@ -43,12 +51,16 @@ import (
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
 var (
-	cfg        *rest.Config
-	k8sClient  client.Client
-	k8sManager ctrl.Manager
-	testEnv    *envtest.Environment
-	ctx        context.Context
-	cancel     context.CancelFunc
+	cfg         *rest.Config
+	k8sClient   client.Client
+	k8sManager  ctrl.Manager
+	testEnv     *envtest.Environment
+	ctx         context.Context
+	cancel      context.CancelFunc
+	err         error
+	ns          = metav1.NamespaceSystem
+	version     = ctr.ImageMinVersion.String()
+	prevVersion = ctr.ImageMinVersion.WithMinor(ctr.ImageMinVersion.Minor() - 1).String()
 )
 
 func TestAPIs(t *testing.T) {
@@ -59,57 +71,203 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	logf.SetLogger(klogr.New())
-	ctx, cancel = context.WithCancel(context.TODO())
-
 	By("bootstrapping test environment")
+
+	logf.SetLogger(klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog)))
+
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{filepath.Join("..", "..", "deployments", "operator", "crd", "bases")},
 	}
-
-	var err error
 	cfg, err = testEnv.Start()
-	Expect(err).ToNot(HaveOccurred())
-	Expect(cfg).ToNot(BeNil())
 
-	err = devicepluginv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
+	Expect(err == nil && cfg != nil).To(BeTrue())
 
-	// +kubebuilder:scaffold:scheme
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 
-	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
-		Scheme: scheme.Scheme,
-	})
-	Expect(err).ToNot(HaveOccurred())
+	Expect(err == nil && k8sClient != nil).To(BeTrue())
 
-	withWebhook := true
-
-	err = gpuctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = sgxctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = qatctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = fpgactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = dsactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-	err = dlbctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook)
-	Expect(err).ToNot(HaveOccurred())
-
-	go func() {
-		err = k8sManager.Start(ctx)
-		Expect(err).ToNot(HaveOccurred())
-	}()
-
-	k8sClient = k8sManager.GetClient()
-	Expect(k8sClient).ToNot(BeNil())
+	Expect(devicepluginv1.AddToScheme(scheme.Scheme)).To(BeNil())
 
 }, 60)
 
 var _ = AfterSuite(func() {
-	cancel()
 	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).ToNot(HaveOccurred())
+
+	Expect(testEnv.Stop()).To(BeNil())
 })
+
+var _ = BeforeEach(func() {
+	up()
+})
+
+var _ = AfterEach(func() {
+	down()
+})
+
+func up() {
+	k8sManager, _ = ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme.Scheme})
+
+	withWebhook := true
+
+	Expect(dlbctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(dsactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(fpgactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(gpuctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(qatctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	Expect(sgxctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil())
+
+	ctx, cancel = context.WithCancel(context.TODO())
+
+	go func() {
+		Expect(k8sManager.Start(ctx)).To(BeNil())
+	}()
+
+	time.Sleep(time.Second)
+}
+
+func down() {
+	time.Sleep(time.Second)
+
+	ctx = context.TODO()
+
+	cancel()
+}
+
+func testUpgrade(name string, dp interface{}, pimage, pinitimage *string) {
+	down()
+
+	prefix := "intel/intel-" + name
+	image0 := prefix + "-plugin:" + prevVersion
+	initimage0 := prefix + "-initcontainer:" + prevVersion
+	image := prefix + "-plugin:" + version
+	initimage := prefix + "-initcontainer:" + version
+
+	*pimage = image
+
+	if pinitimage != nil {
+		*pinitimage = initimage
+	}
+
+	ds0 := makeDaemonSet(name, image0, initimage0)
+
+	Expect(k8sClient.Create(ctx, ds0)).To(BeNil())
+
+	dp0 := makeDevicePlugin(name, image0, initimage0)
+
+	Expect(k8sClient.Create(ctx, dp0)).To(BeNil())
+
+	up()
+
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, dp.(client.Object))).To(BeNil())
+
+	ds := &apps.DaemonSet{}
+
+	Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "intel-" + name + "-plugin"}, ds)).To(BeNil())
+
+	Expect(ds.Spec.Template.Spec.Containers[0].Image == image).To(BeTrue())
+
+	if pinitimage != nil {
+		Expect(ds.Spec.Template.Spec.InitContainers[0].Image == initimage).To(BeTrue())
+	}
+
+	Expect(k8sClient.Delete(ctx, dp.(client.Object))).To(BeNil())
+}
+
+func makeDevicePlugin(name, image, initimage string) client.Object {
+	var obj client.Object
+
+	switch name {
+	case "dlb":
+		obj = &devicepluginv1.DlbDevicePlugin{
+			Spec: devicepluginv1.DlbDevicePluginSpec{
+				Image: image,
+			},
+		}
+	case "dsa":
+		obj = &devicepluginv1.DsaDevicePlugin{
+			Spec: devicepluginv1.DsaDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "fpga":
+		obj = &devicepluginv1.FpgaDevicePlugin{
+			Spec: devicepluginv1.FpgaDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "gpu":
+		obj = &devicepluginv1.GpuDevicePlugin{
+			Spec: devicepluginv1.GpuDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	case "qat":
+		obj = &devicepluginv1.QatDevicePlugin{
+			Spec: devicepluginv1.QatDevicePluginSpec{
+				Image: image,
+			},
+		}
+	case "sgx":
+		obj = &devicepluginv1.SgxDevicePlugin{
+			Spec: devicepluginv1.SgxDevicePluginSpec{
+				Image:     image,
+				InitImage: initimage,
+			},
+		}
+	}
+
+	obj.SetName(name)
+
+	return obj
+}
+
+func makeDaemonSet(name, image, initimage string) *apps.DaemonSet {
+	var ds *apps.DaemonSet
+
+	initcontainerName := "intel-" + name + "-initcontainer"
+
+	switch name {
+	case "dlb":
+		ds = deployments.DLBPluginDaemonSet()
+	case "dsa":
+		ds = deployments.DSAPluginDaemonSet()
+		initcontainerName = "intel-idxd-config-initcontainer"
+	case "gpu":
+		ds = deployments.GPUPluginDaemonSet()
+	case "fpga":
+		ds = deployments.FPGAPluginDaemonSet()
+	case "qat":
+		ds = deployments.QATPluginDaemonSet()
+	case "sgx":
+		ds = deployments.SGXPluginDaemonSet()
+	}
+
+	ds.ObjectMeta.Namespace = ns
+	ds.Spec.Template.Spec.Containers[0].Image = image
+
+	if len(initimage) > 0 {
+		ds.Spec.Template.Spec.InitContainers = []corev1.Container{{
+			Name:  initcontainerName,
+			Image: initimage,
+		}}
+	}
+
+	yes := true
+	ds.OwnerReferences = []metav1.OwnerReference{{
+		APIVersion: "deviceplugin.intel.com/v1",
+		Kind:       strings.Title(name) + "DevicePlugin",
+		Name:       name,
+		Controller: &yes,
+		UID:        uuid.NewUUID(),
+	}}
+
+	return ds
+}