operator: add node selection constraint (amd64 arch)

In order to make controllers consistent, I add a nodeselector constraint of daemonset to dlb, fpga, qat too.
Since the same code is commonly used in many files, I add a function that replaces duplicated code.

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml

@@ -66,3 +66,5 @@ spec:
         hostPath:
           path: /etc/containers/oci/hooks.d
           type: DirectoryOrCreate
+      nodeSelector:
+        kubernetes.io/arch: amd64

deployments/qat_plugin/base/intel-qat-kernel-plugin.yaml

@@ -39,3 +39,5 @@ spec:
       - name: devfs
         hostPath:
           path: /dev
+      nodeSelector:
+        kubernetes.io/arch: amd64

deployments/qat_plugin/base/intel-qat-plugin.yaml

@@ -37,3 +37,5 @@ spec:
       - name: kubeletsockets
         hostPath:
           path: /var/lib/kubelet/device-plugins
+      nodeSelector:
+        kubernetes.io/arch: amd64

deployments/sgx_plugin/base/intel-sgx-plugin.yaml

@@ -47,3 +47,5 @@ spec:
         hostPath:
           path: /dev/sgx_provision
           type: CharDevice
+      nodeSelector:
+        kubernetes.io/arch: amd64

pkg/controllers/dlb/controller.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2021 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -35,6 +35,8 @@ import (
 
 const ownerKey = ".metadata.controller.dlb"
 
+var defaultNodeSelector map[string]string = deployments.DLBPluginDaemonSet().Spec.Template.Spec.NodeSelector
+
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dlbdeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dlbdeviceplugins/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dlbdeviceplugins/finalizers,verbs=update
@@ -76,6 +78,9 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.DlbDevicePlugin)
 
 	daemonSet := deployments.DLBPluginDaemonSet()
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
@@ -92,10 +97,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/dsa/controller.go

@@ -36,10 +36,11 @@ import (
 
 const (
 	ownerKey         = ".metadata.controller.dsa"
-	amd64            = "amd64"
 	inicontainerName = "intel-idxd-config-initcontainer"
 )
 
+var defaultNodeSelector = deployments.DSAPluginDaemonSet().Spec.Template.Spec.NodeSelector
+
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins/finalizers,verbs=update
@@ -159,20 +160,10 @@ func addInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) {
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.DsaDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = amd64
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	}
-
 	daemonSet := deployments.DSAPluginDaemonSet()
-	daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -226,15 +217,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
-	if dp.Spec.NodeSelector == nil {
-		dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	} else {
-		dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64"
-	}
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/dsa/controller_test.go

@@ -34,18 +34,6 @@ const appLabel = "intel-dsa-plugin"
 func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.DsaDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = "amd64"
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
-	}
-
 	yes := true
 	daemonSet := apps.DaemonSet{
 		TypeMeta: metav1.TypeMeta{
@@ -114,7 +102,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 							},
 						},
 					},
-					NodeSelector: nodeSelector,
+					NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"},
 					Volumes: []v1.Volume{
 						{
 							Name: "devfs",

pkg/controllers/fpga/controller.go

@@ -33,9 +33,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	ownerKey = ".metadata.controller.fpga"
-)
+const ownerKey = ".metadata.controller.fpga"
+
+var defaultNodeSelector = deployments.FPGAPluginDaemonSet().Spec.Template.Spec.NodeSelector
 
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=fpgadeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=fpgadeviceplugins/status,verbs=get;update;patch
@@ -78,6 +78,9 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.FpgaDevicePlugin)
 
 	daemonSet := deployments.FPGAPluginDaemonSet()
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -98,10 +101,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/fpga/controller_test.go

@@ -119,7 +119,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 							},
 						},
 					},
-					NodeSelector: devicePlugin.Spec.NodeSelector,
+					NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"},
 					Volumes: []v1.Volume{
 						{
 							Name: "devfs",

pkg/controllers/gpu/controller.go

@@ -39,9 +39,10 @@ import (
 const (
 	ownerKey           = ".metadata.controller.gpu"
 	serviceAccountName = "gpu-manager-sa"
-	amd64              = "amd64"
 )
 
+var defaultNodeSelector = deployments.GPUPluginDaemonSet().Spec.Template.Spec.NodeSelector
+
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=gpudeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=gpudeviceplugins/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=gpudeviceplugins/finalizers,verbs=update
@@ -121,20 +122,10 @@ func (c *controller) NewClusterRoleBinding(rawObj client.Object) *rbacv1.Cluster
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.GpuDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = amd64
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	}
-
 	daemonSet := deployments.GPUPluginDaemonSet()
-	daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -245,15 +236,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
-	if dp.Spec.NodeSelector == nil {
-		dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	} else {
-		dp.Spec.NodeSelector["kubernetes.io/arch"] = amd64
-	}
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/gpu/controller_test.go

@@ -35,18 +35,6 @@ const appLabel = "intel-gpu-plugin"
 func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.GpuDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = "amd64"
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
-	}
-
 	yes := true
 	daemonSet := apps.DaemonSet{
 		TypeMeta: metav1.TypeMeta{
@@ -110,7 +98,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 							},
 						},
 					},
-					NodeSelector: nodeSelector,
+					NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"},
 					Volumes: []v1.Volume{
 						{
 							Name: "devfs",

pkg/controllers/qat/controller.go

@@ -33,9 +33,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	ownerKey = ".metadata.controller.qat"
-)
+const ownerKey = ".metadata.controller.qat"
+
+var defaultNodeSelector = deployments.QATPluginDaemonSet().Spec.Template.Spec.NodeSelector
 
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=qatdeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=qatdeviceplugins/status,verbs=get;update;patch
@@ -82,6 +82,9 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	daemonSet := deployments.QATPluginDaemonSet()
 	daemonSet.Annotations = annotations
 	daemonSet.Spec.Template.Annotations = annotations
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -104,10 +107,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/qat/controller_test.go

@@ -89,7 +89,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 							},
 						},
 					},
-					NodeSelector: devicePlugin.Spec.NodeSelector,
+					NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"},
 					Volumes: []v1.Volume{
 						{
 							Name: "devdir",

pkg/controllers/sgx/controller.go

@@ -34,10 +34,9 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
-const (
-	ownerKey = ".metadata.controller.sgx"
-	amd64    = "amd64"
-)
+const ownerKey = ".metadata.controller.sgx"
+
+var defaultNodeSelector = deployments.SGXPluginDaemonSet().Spec.Template.Spec.NodeSelector
 
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins/status,verbs=get;update;patch
@@ -117,20 +116,10 @@ func setInitContainer(spec *v1.PodSpec, imageName string) {
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = amd64
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	}
-
 	daemonSet := deployments.SGXPluginDaemonSet()
-	daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector
+	if len(devicePlugin.Spec.NodeSelector) > 0 {
+		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+	}
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -151,15 +140,15 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
-	if dp.Spec.NodeSelector == nil {
-		dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64}
-	} else {
-		dp.Spec.NodeSelector["kubernetes.io/arch"] = amd64
-	}
+	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
 			updated = true
 		}
+	} else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector
+		updated = true
+	}
 
 	newargs := getPodArgs(dp)
 	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {

pkg/controllers/sgx/controller_test.go

@@ -35,18 +35,6 @@ const appLabel = "intel-sgx-plugin"
 func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
-	var nodeSelector map[string]string
-	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
-	if dpNodeSelectorSize > 0 {
-		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
-		for k, v := range devicePlugin.Spec.NodeSelector {
-			nodeSelector[k] = v
-		}
-		nodeSelector["kubernetes.io/arch"] = "amd64"
-	} else {
-		nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
-	}
-
 	yes := true
 	charDevice := v1.HostPathCharDev
 	directoryOrCreate := v1.HostPathDirectoryOrCreate
@@ -107,7 +95,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 							},
 						},
 					},
-					NodeSelector: nodeSelector,
+					NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"},
 					Volumes: []v1.Volume{
 						{
 							Name: "kubeletsockets",