diff --git a/cmd/operator/README.md b/cmd/operator/README.md index 7b865644..d0803038 100644 --- a/cmd/operator/README.md +++ b/cmd/operator/README.md @@ -4,6 +4,7 @@ Table of Contents * [Introduction](#introduction) * [Installation](#installation) +* [Upgrade](#upgrade) * [Known issues](#known-issues) ## Introduction @@ -145,6 +146,19 @@ In this case, create a new kustomization with the necessary resources that passes the desired device types to the operator using `--device` command line argument multiple times. +## Upgrade + +The upgrade of the deployed plugins can be done by simply installing a new release of the operator. + +The operator auto-upgrades operator-managed plugins (CR images and thus corresponding deployed daemonsets) to the current release of the operator. + +The [registry-url]/[namespace]/[image] are kept intact on the upgrade. + +No upgrade is done for: + +- Non-operator managed deployments +- Operator deployments without numeric tags + ## Known issues When the operator is run with leader election enabled, that is with the option diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 1e87625a..1dba8343 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -101,7 +101,7 @@ func main() { pm *patcher.Manager ) - ctrl.SetLogger(klogr.New()) + ctrl.SetLogger(klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog))) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&devicePluginNamespace, "deviceplugin-namespace", metav1.NamespaceSystem, "The namespace where deviceplugin daemonsets are created") diff --git a/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml b/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml index 041a012e..6c64bdb5 100644 --- a/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml +++ b/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: fpgadeviceplugin + name: intel-fpga-plugin namespace: system labels: app: intel-fpga-plugin diff --git a/deployments/fpga_plugin/overlays/region/mode-region.yaml b/deployments/fpga_plugin/overlays/region/mode-region.yaml index bfbd15a8..8e700d29 100644 --- a/deployments/fpga_plugin/overlays/region/mode-region.yaml +++ b/deployments/fpga_plugin/overlays/region/mode-region.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: fpgadeviceplugin + name: intel-fpga-plugin namespace: system spec: template: diff --git a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go index f9416d5a..4ac45166 100644 --- a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // dlbdevicepluginlog is for logging in this package. dlbdevicepluginlog = logf.Log.WithName("dlbdeviceplugin-resource") - dlbMinVersion = version.MustParseSemantic(imageMinVersion) + dlbMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for DlbDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go index 30cbeccd..eb0982fe 100644 --- a/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // dsadevicepluginlog is for logging in this package. dsadevicepluginlog = logf.Log.WithName("dsadeviceplugin-resource") - dsaMinVersion = version.MustParseSemantic(imageMinVersion) + dsaMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for DsaDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go index 4ca17d64..5d3413b6 100644 --- a/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // fpgadevicepluginlog is for logging in this package. fpgadevicepluginlog = logf.Log.WithName("fpgadeviceplugin-resource") - fpgaMinVersion = version.MustParseSemantic(imageMinVersion) + fpgaMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for FpgaDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go index 5f8aad41..286a245b 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // gpudevicepluginlog is for logging in this package. gpudevicepluginlog = logf.Log.WithName("gpudeviceplugin-resource") - gpuMinVersion = version.MustParseSemantic(imageMinVersion) + gpuMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for GpuDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go index d4d8f659..309f393f 100644 --- a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // qatdevicepluginlog is for logging in this package. qatdevicepluginlog = logf.Log.WithName("qatdeviceplugin-resource") - qatMinVersion = version.MustParseSemantic(imageMinVersion) + qatMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for QatDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go index 8e899992..5947b670 100644 --- a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go @@ -17,7 +17,6 @@ package v1 import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,7 +32,7 @@ var ( // sgxdevicepluginlog is for logging in this package. sgxdevicepluginlog = logf.Log.WithName("sgxdeviceplugin-resource") - sgxMinVersion = version.MustParseSemantic(imageMinVersion) + sgxMinVersion = controllers.ImageMinVersion ) // SetupWebhookWithManager sets up a webhook for SgxDevicePlugin custom resources. diff --git a/pkg/apis/deviceplugin/v1/webhook_common.go b/pkg/apis/deviceplugin/v1/webhook_common.go index 25347853..e0af950d 100644 --- a/pkg/apis/deviceplugin/v1/webhook_common.go +++ b/pkg/apis/deviceplugin/v1/webhook_common.go @@ -22,9 +22,6 @@ import ( "k8s.io/apimachinery/pkg/util/version" ) -// common constants for webhooks. -const imageMinVersion string = "0.23.0" - // common functions for webhooks func validatePluginImage(image, expectedImageName string, expectedMinVersion *version.Version) error { diff --git a/pkg/controllers/dlb/controller.go b/pkg/controllers/dlb/controller.go index 9b3b9001..e027a710 100644 --- a/pkg/controllers/dlb/controller.go +++ b/pkg/controllers/dlb/controller.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.DlbDevicePlugin{} } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.DlbDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, nil) +} + func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { var list devicepluginv1.DlbDevicePluginList if err := clnt.List(ctx, &list); err != nil { diff --git a/pkg/controllers/dsa/controller.go b/pkg/controllers/dsa/controller.go index a635f509..be9b3b75 100644 --- a/pkg/controllers/dsa/controller.go +++ b/pkg/controllers/dsa/controller.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -69,6 +69,11 @@ func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.DsaDevicePlugin{} } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.DsaDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) +} + func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { var list devicepluginv1.DsaDevicePluginList if err := clnt.List(ctx, &list); err != nil { diff --git a/pkg/controllers/fpga/controller.go b/pkg/controllers/fpga/controller.go index c006dbdd..d6e21d28 100644 --- a/pkg/controllers/fpga/controller.go +++ b/pkg/controllers/fpga/controller.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.FpgaDevicePlugin{} } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.FpgaDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) +} + func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { var list devicepluginv1.FpgaDevicePluginList if err := clnt.List(ctx, &list); err != nil { diff --git a/pkg/controllers/fpga/controller_test.go b/pkg/controllers/fpga/controller_test.go index 810e4e04..8040dd99 100644 --- a/pkg/controllers/fpga/controller_test.go +++ b/pkg/controllers/fpga/controller_test.go @@ -44,7 +44,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet }, ObjectMeta: metav1.ObjectMeta{ Namespace: c.ns, - Name: "fpgadeviceplugin", + Name: "intel-fpga-plugin", Labels: map[string]string{ "app": appLabel, }, diff --git a/pkg/controllers/gpu/controller.go b/pkg/controllers/gpu/controller.go index 5573fad8..b602116e 100644 --- a/pkg/controllers/gpu/controller.go +++ b/pkg/controllers/gpu/controller.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -70,6 +70,11 @@ func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.GpuDevicePlugin{} } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.GpuDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) +} + func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { var list devicepluginv1.GpuDevicePluginList if err := clnt.List(ctx, &list); err != nil { diff --git a/pkg/controllers/qat/controller.go b/pkg/controllers/qat/controller.go index 758d28da..5547d5df 100644 --- a/pkg/controllers/qat/controller.go +++ b/pkg/controllers/qat/controller.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,6 +65,11 @@ func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.QatDevicePlugin{} } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.QatDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, nil) +} + func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { var list devicepluginv1.QatDevicePluginList if err := clnt.List(ctx, &list); err != nil { diff --git a/pkg/controllers/reconciler.go b/pkg/controllers/reconciler.go index 9ab4719c..4de6fbce 100644 --- a/pkg/controllers/reconciler.go +++ b/pkg/controllers/reconciler.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,22 +17,27 @@ package controllers import ( "context" + "strings" "sync" "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/diff" + versionutil "k8s.io/apimachinery/pkg/util/version" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) var ( - bKeeper = &bookKeeper{} + bKeeper = &bookKeeper{} + ImageMinVersion = versionutil.MustParseSemantic("0.23.0") ) func init() { @@ -97,6 +102,7 @@ type DevicePluginController interface { NewDaemonSet(devicePlugin client.Object) *apps.DaemonSet UpdateDaemonSet(client.Object, *apps.DaemonSet) (updated bool) UpdateStatus(client.Object, *apps.DaemonSet, []string) (updated bool, err error) + Upgrade(ctx context.Context, obj client.Object) (upgrade bool) } type reconciler struct { @@ -159,6 +165,33 @@ func (r *reconciler) createObjects(ctx context.Context, return result, nil } +func UpgradeImages(image *string, initimage *string) (upgrade bool) { + for _, s := range []*string{image, initimage} { + if s == nil { + continue + } + + if parts := strings.SplitN(*s, ":", 2); len(parts) == 2 && len(parts[0]) > 0 { + name, version := parts[0], parts[1] + if ver, err := versionutil.ParseSemantic(version); err == nil && ver.LessThan(ImageMinVersion) { + *s = name + ":" + ImageMinVersion.String() + upgrade = true + } + } + } + + return upgrade +} + +func upgrade(ctx context.Context, r *reconciler, devicePlugin client.Object) { + if r.controller.Upgrade(ctx, devicePlugin) { + if err := r.Update(ctx, devicePlugin); err != nil { + log := log.FromContext(ctx) + log.Error(err, "unable to update devicePlugin") + } + } +} + // Reconcile reconciles a device plugin object. func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) @@ -182,6 +215,8 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } + upgrade(ctx, r, devicePlugin) + // Create a daemon set for the plugin if it doesn't exist. if len(childDaemonSets.Items) == 0 { return r.createDaemonSet(ctx, devicePlugin, log) @@ -189,8 +224,12 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu ds := &childDaemonSets.Items[0] + ds0 := ds.DeepCopy() + // Synchronize the DaemonSet with its owner. if r.controller.UpdateDaemonSet(devicePlugin, ds) { + log.Info("", cmp.Diff(ds0.Spec.Template.Spec, ds.Spec.Template.Spec, diff.IgnoreUnset())) + if err := r.Update(ctx, ds); err != nil { log.Error(err, "unable to update DaemonSet", "DaemonSet", ds) return ctrl.Result{}, err diff --git a/pkg/controllers/reconciler_test.go b/pkg/controllers/reconciler_test.go new file mode 100644 index 00000000..63dfd930 --- /dev/null +++ b/pkg/controllers/reconciler_test.go @@ -0,0 +1,68 @@ +// Copyright 2022 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "testing" +) + +func TestUpgrade(test *testing.T) { + tests := []struct { + image string + initimage string + expectedImage string + expectedInitimage string + upgrade bool + }{ + { + image: "intel/intel-dsa-plugin:0.22.0", + expectedImage: "intel/intel-dsa-plugin:0.23.0", + initimage: "intel/intel-idxd-config-initcontainer:0.22.0", + expectedInitimage: "intel/intel-idxd-config-initcontainer:0.23.0", + upgrade: true, + }, + { + image: "intel/intel-dsa-plugin:0.23.0", + expectedImage: "intel/intel-dsa-plugin:0.23.0", + initimage: "intel/intel-idxd-config-initcontainer:0.23.0", + expectedInitimage: "intel/intel-idxd-config-initcontainer:0.23.0", + upgrade: false, + }, + { + image: "intel/intel-dsa-plugin:latest", + expectedImage: "intel/intel-dsa-plugin:latest", + initimage: "intel/intel-idxd-config-initcontainer:latest", + expectedInitimage: "intel/intel-idxd-config-initcontainer:latest", + upgrade: false, + }, + { + image: "intel/intel-dsa-plugin", + expectedImage: "intel/intel-dsa-plugin", + initimage: "intel/intel-idxd-config-initcontainer", + expectedInitimage: "intel/intel-idxd-config-initcontainer", + upgrade: false, + }, + } + + for _, t := range tests { + upgrade := UpgradeImages(&t.image, &t.initimage) + + if !(upgrade == t.upgrade && t.image == t.expectedImage && t.initimage == t.expectedInitimage) { + test.Errorf("expectedUpgrade: %v, received: %v", t.upgrade, upgrade) + test.Errorf("expectedImage: %s, received: %s", t.expectedImage, t.image) + test.Errorf("expectedInitimage: %s, received: %s", t.expectedInitimage, t.initimage) + } + } +} diff --git a/pkg/controllers/sgx/controller.go b/pkg/controllers/sgx/controller.go index ebe5a14d..a783ae75 100644 --- a/pkg/controllers/sgx/controller.go +++ b/pkg/controllers/sgx/controller.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -62,6 +62,11 @@ type controller struct { ns string } +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.SgxDevicePlugin) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) +} + func (c *controller) CreateEmptyObject() client.Object { return &devicepluginv1.SgxDevicePlugin{} } @@ -134,6 +139,18 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { return daemonSet } +func removeVolume(volumes []v1.Volume, name string) []v1.Volume { + newVolumes := []v1.Volume{} + + for _, volume := range volumes { + if volume.Name != name { + newVolumes = append(newVolumes, volume) + } + } + + return newVolumes +} + func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { dp := rawObj.(*devicepluginv1.SgxDevicePlugin) @@ -142,6 +159,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( updated = true } + if dp.Spec.InitImage == "" { + if ds.Spec.Template.Spec.InitContainers != nil { + ds.Spec.Template.Spec.InitContainers = nil + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "nfd-source-hooks") + updated = true + } + } else { + setInitContainer(&ds.Spec.Template.Spec, dp.Spec.InitImage) + updated = true + } + if len(dp.Spec.NodeSelector) > 0 { if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector diff --git a/test/envtest/dlbdeviceplugin_controller_test.go b/test/envtest/dlbdeviceplugin_controller_test.go index e82cb570..a98b2386 100644 --- a/test/envtest/dlbdeviceplugin_controller_test.go +++ b/test/envtest/dlbdeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("DlbDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle DlbDevicePlugin objects correctly", func() { spec := devicepluginv1.DlbDevicePluginSpec{ @@ -102,4 +97,14 @@ var _ = Describe("DlbDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.DlbDevicePlugin{} + + var image string + + testUpgrade("dlb", dp, &image, nil) + + Expect(dp.Spec.Image == image).To(BeTrue()) + }) }) diff --git a/test/envtest/dsadeviceplugin_controller_test.go b/test/envtest/dsadeviceplugin_controller_test.go index a33809f8..d2075df4 100644 --- a/test/envtest/dsadeviceplugin_controller_test.go +++ b/test/envtest/dsadeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("DsaDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle DsaDevicePlugin objects correctly", func() { spec := devicepluginv1.DsaDevicePluginSpec{ @@ -87,4 +82,15 @@ var _ = Describe("DsaDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.DsaDevicePlugin{} + + var image, initimage string + + testUpgrade("dsa", dp, &image, &initimage) + + Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) + }) }) diff --git a/test/envtest/fpgadeviceplugin_controller_test.go b/test/envtest/fpgadeviceplugin_controller_test.go index 0ef1f53f..4f481269 100644 --- a/test/envtest/fpgadeviceplugin_controller_test.go +++ b/test/envtest/fpgadeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("FpgaDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle FpgaDevicePlugin objects correctly", func() { spec := devicepluginv1.FpgaDevicePluginSpec{ @@ -88,4 +83,15 @@ var _ = Describe("FpgaDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.FpgaDevicePlugin{} + + var image, initimage string + + testUpgrade("fpga", dp, &image, &initimage) + + Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) + }) }) diff --git a/test/envtest/gpudeviceplugin_controller_test.go b/test/envtest/gpudeviceplugin_controller_test.go index 5d2fdad3..bc9b6c32 100644 --- a/test/envtest/gpudeviceplugin_controller_test.go +++ b/test/envtest/gpudeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("GpuDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle GpuDevicePlugin objects correctly", func() { spec := devicepluginv1.GpuDevicePluginSpec{ @@ -87,4 +82,15 @@ var _ = Describe("GpuDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.GpuDevicePlugin{} + + var image, initimage string + + testUpgrade("gpu", dp, &image, &initimage) + + Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) + }) }) diff --git a/test/envtest/qatdeviceplugin_controller_test.go b/test/envtest/qatdeviceplugin_controller_test.go index 47052da7..c5bc3123 100644 --- a/test/envtest/qatdeviceplugin_controller_test.go +++ b/test/envtest/qatdeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("QatDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle QatDevicePlugin objects correctly", func() { spec := devicepluginv1.QatDevicePluginSpec{ @@ -106,4 +101,14 @@ var _ = Describe("QatDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.QatDevicePlugin{} + + var image string + + testUpgrade("qat", dp, &image, nil) + + Expect(dp.Spec.Image == image).To(BeTrue()) + }) }) diff --git a/test/envtest/sgxdeviceplugin_controller_test.go b/test/envtest/sgxdeviceplugin_controller_test.go index 96b53c27..1e0bd238 100644 --- a/test/envtest/sgxdeviceplugin_controller_test.go +++ b/test/envtest/sgxdeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,11 +31,6 @@ var _ = Describe("SgxDevicePlugin Controller", func() { const timeout = time.Second * 30 const interval = time.Second * 1 - AfterEach(func() { - time.Sleep(time.Second * 2) - - }) - Context("Basic CRUD operations", func() { It("should handle SgxDevicePlugin objects correctly", func() { spec := devicepluginv1.SgxDevicePluginSpec{ @@ -88,4 +83,15 @@ var _ = Describe("SgxDevicePlugin Controller", func() { }, timeout, interval).ShouldNot(Succeed()) }) }) + + It("upgrades", func() { + dp := &devicepluginv1.SgxDevicePlugin{} + + var image, initimage string + + testUpgrade("sgx", dp, &image, &initimage) + + Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) + }) }) diff --git a/test/envtest/suite_test.go b/test/envtest/suite_test.go index 0d16f93d..d4fd5c18 100644 --- a/test/envtest/suite_test.go +++ b/test/envtest/suite_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,11 +17,17 @@ package envtest import ( "context" "path/filepath" + "strings" "testing" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/klog/v2/klogr" @@ -30,7 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" + ctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" dlbctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dlb" dsactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dsa" fpgactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga" @@ -43,12 +51,16 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - cfg *rest.Config - k8sClient client.Client - k8sManager ctrl.Manager - testEnv *envtest.Environment - ctx context.Context - cancel context.CancelFunc + cfg *rest.Config + k8sClient client.Client + k8sManager ctrl.Manager + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + err error + ns = metav1.NamespaceSystem + version = ctr.ImageMinVersion.String() + prevVersion = ctr.ImageMinVersion.WithMinor(ctr.ImageMinVersion.Minor() - 1).String() ) func TestAPIs(t *testing.T) { @@ -59,57 +71,203 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - logf.SetLogger(klogr.New()) - ctx, cancel = context.WithCancel(context.TODO()) - By("bootstrapping test environment") + + logf.SetLogger(klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog))) + testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "deployments", "operator", "crd", "bases")}, } - - var err error cfg, err = testEnv.Start() - Expect(err).ToNot(HaveOccurred()) - Expect(cfg).ToNot(BeNil()) - err = devicepluginv1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + Expect(err == nil && cfg != nil).To(BeTrue()) - // +kubebuilder:scaffold:scheme + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - }) - Expect(err).ToNot(HaveOccurred()) + Expect(err == nil && k8sClient != nil).To(BeTrue()) - withWebhook := true - - err = gpuctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - err = sgxctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - err = qatctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - err = fpgactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - err = dsactr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - err = dlbctr.SetupReconciler(k8sManager, metav1.NamespaceSystem, !withWebhook) - Expect(err).ToNot(HaveOccurred()) - - go func() { - err = k8sManager.Start(ctx) - Expect(err).ToNot(HaveOccurred()) - }() - - k8sClient = k8sManager.GetClient() - Expect(k8sClient).ToNot(BeNil()) + Expect(devicepluginv1.AddToScheme(scheme.Scheme)).To(BeNil()) }, 60) var _ = AfterSuite(func() { - cancel() By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) + + Expect(testEnv.Stop()).To(BeNil()) }) + +var _ = BeforeEach(func() { + up() +}) + +var _ = AfterEach(func() { + down() +}) + +func up() { + k8sManager, _ = ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme.Scheme}) + + withWebhook := true + + Expect(dlbctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + Expect(dsactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + Expect(fpgactr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + Expect(gpuctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + Expect(qatctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + Expect(sgxctr.SetupReconciler(k8sManager, ns, !withWebhook)).To(BeNil()) + + ctx, cancel = context.WithCancel(context.TODO()) + + go func() { + Expect(k8sManager.Start(ctx)).To(BeNil()) + }() + + time.Sleep(time.Second) +} + +func down() { + time.Sleep(time.Second) + + ctx = context.TODO() + + cancel() +} + +func testUpgrade(name string, dp interface{}, pimage, pinitimage *string) { + down() + + prefix := "intel/intel-" + name + image0 := prefix + "-plugin:" + prevVersion + initimage0 := prefix + "-initcontainer:" + prevVersion + image := prefix + "-plugin:" + version + initimage := prefix + "-initcontainer:" + version + + *pimage = image + + if pinitimage != nil { + *pinitimage = initimage + } + + ds0 := makeDaemonSet(name, image0, initimage0) + + Expect(k8sClient.Create(ctx, ds0)).To(BeNil()) + + dp0 := makeDevicePlugin(name, image0, initimage0) + + Expect(k8sClient.Create(ctx, dp0)).To(BeNil()) + + up() + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, dp.(client.Object))).To(BeNil()) + + ds := &apps.DaemonSet{} + + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "intel-" + name + "-plugin"}, ds)).To(BeNil()) + + Expect(ds.Spec.Template.Spec.Containers[0].Image == image).To(BeTrue()) + + if pinitimage != nil { + Expect(ds.Spec.Template.Spec.InitContainers[0].Image == initimage).To(BeTrue()) + } + + Expect(k8sClient.Delete(ctx, dp.(client.Object))).To(BeNil()) +} + +func makeDevicePlugin(name, image, initimage string) client.Object { + var obj client.Object + + switch name { + case "dlb": + obj = &devicepluginv1.DlbDevicePlugin{ + Spec: devicepluginv1.DlbDevicePluginSpec{ + Image: image, + }, + } + case "dsa": + obj = &devicepluginv1.DsaDevicePlugin{ + Spec: devicepluginv1.DsaDevicePluginSpec{ + Image: image, + InitImage: initimage, + }, + } + case "fpga": + obj = &devicepluginv1.FpgaDevicePlugin{ + Spec: devicepluginv1.FpgaDevicePluginSpec{ + Image: image, + InitImage: initimage, + }, + } + case "gpu": + obj = &devicepluginv1.GpuDevicePlugin{ + Spec: devicepluginv1.GpuDevicePluginSpec{ + Image: image, + InitImage: initimage, + }, + } + case "qat": + obj = &devicepluginv1.QatDevicePlugin{ + Spec: devicepluginv1.QatDevicePluginSpec{ + Image: image, + }, + } + case "sgx": + obj = &devicepluginv1.SgxDevicePlugin{ + Spec: devicepluginv1.SgxDevicePluginSpec{ + Image: image, + InitImage: initimage, + }, + } + } + + obj.SetName(name) + + return obj +} + +func makeDaemonSet(name, image, initimage string) *apps.DaemonSet { + var ds *apps.DaemonSet + + initcontainerName := "intel-" + name + "-initcontainer" + + switch name { + case "dlb": + ds = deployments.DLBPluginDaemonSet() + case "dsa": + ds = deployments.DSAPluginDaemonSet() + initcontainerName = "intel-idxd-config-initcontainer" + case "gpu": + ds = deployments.GPUPluginDaemonSet() + case "fpga": + ds = deployments.FPGAPluginDaemonSet() + case "qat": + ds = deployments.QATPluginDaemonSet() + case "sgx": + ds = deployments.SGXPluginDaemonSet() + } + + ds.ObjectMeta.Namespace = ns + ds.Spec.Template.Spec.Containers[0].Image = image + + if len(initimage) > 0 { + ds.Spec.Template.Spec.InitContainers = []corev1.Container{{ + Name: initcontainerName, + Image: initimage, + }} + } + + yes := true + ds.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "deviceplugin.intel.com/v1", + Kind: strings.Title(name) + "DevicePlugin", + Name: name, + Controller: &yes, + UID: uuid.NewUUID(), + }} + + return ds +}