From b6caadfc63b318e59772394d493906e5ce795474 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 17 Nov 2021 19:01:49 +0200 Subject: [PATCH] operator: use go:embed to generate daemonset objects Signed-off-by: Ed Bartosh --- deployments/daemonsets.go | 75 +++++++ .../base/intel-fpga-plugin-daemonset.yaml | 5 - go.mod | 2 +- pkg/controllers/dlb/controller.go | 107 +--------- pkg/controllers/dsa/controller.go | 122 ++---------- pkg/controllers/fpga/controller.go | 141 +------------ pkg/controllers/gpu/controller.go | 117 ++--------- pkg/controllers/qat/controller.go | 97 ++------- pkg/controllers/sgx/controller.go | 185 ++++++------------ 9 files changed, 191 insertions(+), 660 deletions(-) create mode 100644 deployments/daemonsets.go diff --git a/deployments/daemonsets.go b/deployments/daemonsets.go new file mode 100644 index 00000000..d3508da4 --- /dev/null +++ b/deployments/daemonsets.go @@ -0,0 +1,75 @@ +// Copyright 2021 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package deployments generates plugin DaemonSet object out of plugin yaml. +package deployments + +import ( + _ "embed" + + apps "k8s.io/api/apps/v1" + "sigs.k8s.io/yaml" +) + +//go:embed dlb_plugin/base/*plugin*.yaml +var contentDLB []byte + +func DLBPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentDLB).DeepCopy() +} + +//go:embed dsa_plugin/base/*plugin*.yaml +var contentDSA []byte + +func DSAPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentDSA).DeepCopy() +} + +//go:embed fpga_plugin/base/*plugin*.yaml +var contentFPGA []byte + +func FPGAPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentFPGA).DeepCopy() +} + +//go:embed gpu_plugin/base/*plugin.yaml +var contentGPU []byte + +func GPUPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentGPU).DeepCopy() +} + +//go:embed qat_plugin/base/*qat-plugin.yaml +var contentQAT []byte + +func QATPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentQAT).DeepCopy() +} + +//go:embed sgx_plugin/base/*plugin*.yaml +var contentSGX []byte + +func SGXPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentSGX).DeepCopy() +} + +// getDaemonset unmarshalls yaml content into a DaemonSet object. +func getDaemonset(content []byte) *apps.DaemonSet { + var result apps.DaemonSet + err := yaml.Unmarshal(content, &result) + if err != nil { + panic(err) + } + return &result +} diff --git a/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml b/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml index 81871d0a..c2ba8530 100644 --- a/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml +++ b/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: system ---- apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/go.mod b/go.mod index 6898fe58..8d40c420 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( k8s.io/kubernetes v1.22.2 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a sigs.k8s.io/controller-runtime v0.10.0 + sigs.k8s.io/yaml v1.2.0 ) require ( @@ -90,7 +91,6 @@ require ( k8s.io/kubectl v0.0.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) replace ( diff --git a/pkg/controllers/dlb/controller.go b/pkg/controllers/dlb/controller.go index 4bcf8aed..7098ac94 100644 --- a/pkg/controllers/dlb/controller.go +++ b/pkg/controllers/dlb/controller.go @@ -22,22 +22,18 @@ import ( "strings" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" ) -const ( - ownerKey = ".metadata.controller.dlb" - appLabel = "intel-dlb-plugin" -) +const ownerKey = ".metadata.controller.dlb" // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dlbdeviceplugins,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dlbdeviceplugins/status,verbs=get;update;patch @@ -77,101 +73,14 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { devicePlugin := rawObj.(*devicepluginv1.DlbDevicePlugin) - yes := true - var nodeSelector map[string]string + daemonSet := deployments.DLBPluginDaemonSet() + daemonSet.ObjectMeta.Namespace = c.ns - daemonSet := apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: appLabel, - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - TerminationMessagePath: "/tmp/termination-log", - Args: getPodArgs(devicePlugin), - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "devfs", - MountPath: "/dev", - ReadOnly: true, - }, - { - Name: "sysfs", - MountPath: "/sys/class/dlb2", - ReadOnly: true, - }, - { - Name: "kubeletsockets", - MountPath: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - NodeSelector: nodeSelector, - Volumes: []v1.Volume{ - { - Name: "devfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "sysfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys/class/dlb2", - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - }, - }, - }, - } - return &daemonSet + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + return daemonSet } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { diff --git a/pkg/controllers/dsa/controller.go b/pkg/controllers/dsa/controller.go index 92168e8a..d2bf17ef 100644 --- a/pkg/controllers/dsa/controller.go +++ b/pkg/controllers/dsa/controller.go @@ -23,12 +23,12 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" @@ -36,7 +36,7 @@ import ( const ( ownerKey = ".metadata.controller.dsa" - appLabel = "intel-dsa-plugin" + amd64 = "amd64" ) // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins,verbs=get;list;watch;create;update;patch;delete @@ -114,116 +114,18 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { for k, v := range devicePlugin.Spec.NodeSelector { nodeSelector[k] = v } - nodeSelector["kubernetes.io/arch"] = "amd64" + nodeSelector["kubernetes.io/arch"] = amd64 } else { - nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + nodeSelector = map[string]string{"kubernetes.io/arch": amd64} } - yes := true - daemonSet := apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: appLabel, - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - Args: getPodArgs(devicePlugin), - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "devfs", - MountPath: "/dev/dsa", - ReadOnly: true, - }, - { - Name: "chardevs", - MountPath: "/dev/char", - ReadOnly: true, - }, - { - Name: "sysfs", - MountPath: "/sys/bus", - ReadOnly: true, - }, - { - Name: "kubeletsockets", - MountPath: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - NodeSelector: nodeSelector, - Volumes: []v1.Volume{ - { - Name: "devfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/dsa", - }, - }, - }, - { - Name: "chardevs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/char", - }, - }, - }, - { - Name: "sysfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys/bus", - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - }, - }, - }, - } + daemonSet := deployments.DSAPluginDaemonSet() + daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image - // add the optional InitImage + // add the optional init container if devicePlugin.Spec.InitImage != "" { setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage) @@ -256,7 +158,7 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { } } - return &daemonSet + return daemonSet } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { @@ -268,7 +170,7 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( } if dp.Spec.NodeSelector == nil { - dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64} } else { dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64" } diff --git a/pkg/controllers/fpga/controller.go b/pkg/controllers/fpga/controller.go index a70ce90f..51acaebc 100644 --- a/pkg/controllers/fpga/controller.go +++ b/pkg/controllers/fpga/controller.go @@ -22,13 +22,12 @@ import ( "strings" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" @@ -36,7 +35,6 @@ import ( const ( ownerKey = ".metadata.controller.fpga" - appLabel = "intel-fpga-plugin" ) // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=fpgadeviceplugins,verbs=get;list;watch;create;update;patch;delete @@ -77,136 +75,13 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { devicePlugin := rawObj.(*devicepluginv1.FpgaDevicePlugin) - yes := true - directoryOrCreate := v1.HostPathDirectoryOrCreate - return &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Args: getPodArgs(devicePlugin), - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - Name: appLabel, - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - TerminationMessagePath: "/tmp/termination-log", - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "/dev", - Name: "devfs", - ReadOnly: true, - }, - { - MountPath: "/sys/class", - Name: "sysfs", - ReadOnly: true, - }, - { - MountPath: "/var/lib/kubelet/device-plugins", - Name: "kubeletsockets", - }, - }, - }, - }, - InitContainers: []v1.Container{ - { - Image: devicePlugin.Spec.InitImage, - ImagePullPolicy: "IfNotPresent", - Name: "intel-fpga-initcontainer", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "/opt/intel/fpga-sw", - Name: "intel-fpga-sw", - }, - { - MountPath: "/etc/containers/oci/hooks.d", - Name: "oci-hooks-config", - }, - }, - }, - }, - NodeSelector: devicePlugin.Spec.NodeSelector, - Volumes: []v1.Volume{ - { - Name: "devfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "sysfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys/class", - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - { - Name: "intel-fpga-sw", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/opt/intel/fpga-sw", - Type: &directoryOrCreate, - }, - }, - }, - { - Name: "oci-hooks-config", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/containers/oci/hooks.d", - Type: &directoryOrCreate, - }, - }, - }, - }, - }, - }, - }, - } + + daemonSet := deployments.FPGAPluginDaemonSet() + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + return daemonSet } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { diff --git a/pkg/controllers/gpu/controller.go b/pkg/controllers/gpu/controller.go index 64c59e16..9c229b22 100644 --- a/pkg/controllers/gpu/controller.go +++ b/pkg/controllers/gpu/controller.go @@ -30,6 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" @@ -37,8 +38,8 @@ import ( const ( ownerKey = ".metadata.controller.gpu" - appLabel = "intel-gpu-plugin" serviceAccountName = "gpu-manager-sa" + amd64 = "amd64" ) // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=gpudeviceplugins,verbs=get;list;watch;create;update;patch;delete @@ -126,112 +127,32 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { for k, v := range devicePlugin.Spec.NodeSelector { nodeSelector[k] = v } - nodeSelector["kubernetes.io/arch"] = "amd64" + nodeSelector["kubernetes.io/arch"] = amd64 } else { - nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + nodeSelector = map[string]string{"kubernetes.io/arch": amd64} } - yes := true - daemonSet := apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: appLabel, - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - Args: getPodArgs(devicePlugin), - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "devfs", - MountPath: "/dev/dri", - ReadOnly: true, - }, - { - Name: "sysfs", - MountPath: "/sys/class/drm", - ReadOnly: true, - }, - { - Name: "kubeletsockets", - MountPath: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - NodeSelector: nodeSelector, - Volumes: []v1.Volume{ - { - Name: "devfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/dri", - }, - }, - }, - { - Name: "sysfs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys/class/drm", - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - }, - }, - }, - } - // add the optional InitImage - if devicePlugin.Spec.InitImage != "" { + daemonSet := deployments.GPUPluginDaemonSet() + daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + if devicePlugin.Spec.InitImage == "" { + daemonSet.Spec.Template.Spec.InitContainers = nil + daemonSet.Spec.Template.Spec.Volumes = removeVolume(daemonSet.Spec.Template.Spec.Volumes, "nfd-source-hooks") + } else { setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage) } + // add service account if resource manager is enabled if devicePlugin.Spec.ResourceManager { daemonSet.Spec.Template.Spec.ServiceAccountName = serviceAccountName addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources") } - return &daemonSet + + return daemonSet } func addVolumeMountIfMissing(spec *v1.PodSpec, name, mountPath string) { @@ -324,9 +245,9 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( } if dp.Spec.NodeSelector == nil { - dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64} } else { - dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64" + dp.Spec.NodeSelector["kubernetes.io/arch"] = amd64 } if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector diff --git a/pkg/controllers/qat/controller.go b/pkg/controllers/qat/controller.go index b98dc1a6..bbc71f35 100644 --- a/pkg/controllers/qat/controller.go +++ b/pkg/controllers/qat/controller.go @@ -22,13 +22,12 @@ import ( "strings" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" @@ -36,7 +35,6 @@ import ( const ( ownerKey = ".metadata.controller.qat" - appLabel = "intel-qat-plugin" ) // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=qatdeviceplugins,verbs=get;list;watch;create;update;patch;delete @@ -77,88 +75,17 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { devicePlugin := rawObj.(*devicepluginv1.QatDevicePlugin) - yes := true - pluginAnnotations := devicePlugin.ObjectMeta.DeepCopy().Annotations - return &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - Annotations: pluginAnnotations, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - Annotations: pluginAnnotations, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: appLabel, - Args: getPodArgs(devicePlugin), - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "devdir", - MountPath: "/dev/vfio", - ReadOnly: true, - }, - { - Name: "pcidir", - MountPath: "/sys/bus/pci", - }, - { - Name: "kubeletsockets", - MountPath: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - NodeSelector: devicePlugin.Spec.NodeSelector, - Volumes: []v1.Volume{ - { - Name: "devdir", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/vfio", - }, - }, - }, - { - Name: "pcidir", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys/bus/pci", - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - }, - }, - }, - } + + annotations := devicePlugin.ObjectMeta.DeepCopy().Annotations + + daemonSet := deployments.QATPluginDaemonSet() + daemonSet.Annotations = annotations + daemonSet.Spec.Template.Annotations = annotations + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + return daemonSet } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { diff --git a/pkg/controllers/sgx/controller.go b/pkg/controllers/sgx/controller.go index ced2b3ce..0fcf37a1 100644 --- a/pkg/controllers/sgx/controller.go +++ b/pkg/controllers/sgx/controller.go @@ -22,21 +22,21 @@ import ( "strings" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" ) const ( ownerKey = ".metadata.controller.sgx" - appLabel = "intel-sgx-plugin" + amd64 = "amd64" ) // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=get;list;watch;create;update;patch;delete @@ -75,6 +75,44 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client return len(list.Items), nil } +func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) { + for _, vol := range spec.Volumes { + if vol.Name == name { + return + } + } + + spec.Volumes = append(spec.Volumes, v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: path, + Type: &hpType, + }, + }, + }) +} + +func setInitContainer(spec *v1.PodSpec, imageName string) { + yes := true + spec.InitContainers = []v1.Container{ + { + Image: imageName, + ImagePullPolicy: "IfNotPresent", + Name: "intel-sgx-initcontainer", + SecurityContext: &v1.SecurityContext{ + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/etc/kubernetes/node-feature-discovery/source.d/", + Name: "nfd-source-hooks", + }, + }, + }} + addVolumeIfMissing(spec, "nfd-source-hooks", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate) +} + func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin) @@ -85,134 +123,23 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { for k, v := range devicePlugin.Spec.NodeSelector { nodeSelector[k] = v } - nodeSelector["kubernetes.io/arch"] = "amd64" + nodeSelector["kubernetes.io/arch"] = amd64 } else { - nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + nodeSelector = map[string]string{"kubernetes.io/arch": amd64} } - yes := true - charDevice := v1.HostPathCharDev - directoryOrCreate := v1.HostPathDirectoryOrCreate - return &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: c.ns, - GenerateName: devicePlugin.Name + "-", - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": appLabel, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": appLabel, - }, - }, - Spec: v1.PodSpec{ - InitContainers: []v1.Container{ - { - Image: devicePlugin.Spec.InitImage, - ImagePullPolicy: "IfNotPresent", - Name: "intel-sgx-initcontainer", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "/etc/kubernetes/node-feature-discovery/source.d/", - Name: "nfd-source-hooks", - }, - }, - }, - }, - Containers: []v1.Container{ - { - Name: appLabel, - Args: getPodArgs(devicePlugin), - Image: devicePlugin.Spec.Image, - ImagePullPolicy: "IfNotPresent", - SecurityContext: &v1.SecurityContext{ - ReadOnlyRootFilesystem: &yes, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "sgxdevices", - MountPath: "/dev/sgx", - ReadOnly: true, - }, - { - Name: "sgx-enclave", - MountPath: "/dev/sgx_enclave", - ReadOnly: true, - }, - { - Name: "sgx-provision", - MountPath: "/dev/sgx_provision", - ReadOnly: true, - }, - { - Name: "kubeletsockets", - MountPath: "/var/lib/kubelet/device-plugins", - }, - }, - }, - }, - NodeSelector: nodeSelector, - Volumes: []v1.Volume{ - { - Name: "sgxdevices", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/sgx", - Type: &directoryOrCreate, - }, - }, - }, - { - Name: "sgx-enclave", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/sgx_enclave", - Type: &charDevice, - }, - }, - }, - { - Name: "sgx-provision", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/dev/sgx_provision", - Type: &charDevice, - }, - }, - }, - { - Name: "kubeletsockets", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/device-plugins", - }, - }, - }, - { - Name: "nfd-source-hooks", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/node-feature-discovery/source.d/", - Type: &directoryOrCreate, - }, - }, - }, - }, - }, - }, - }, + daemonSet := deployments.SGXPluginDaemonSet() + daemonSet.Spec.Template.Spec.NodeSelector = nodeSelector + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + // add the optional init container + if devicePlugin.Spec.InitImage != "" { + setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage) } + + return daemonSet } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { @@ -224,9 +151,9 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( } if dp.Spec.NodeSelector == nil { - dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": amd64} } else { - dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64" + dp.Spec.NodeSelector["kubernetes.io/arch"] = amd64 } if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector