From c935570bab3dddce249900e01ad649914dd4c38f Mon Sep 17 00:00:00 2001 From: Ukri Niemimuukko Date: Tue, 13 Oct 2020 14:30:44 +0300 Subject: [PATCH] operator: GPU-plugin initImage This adds the initImage field to the custom resource definition and takes it into use. The fpga webhook image validation function is split off into a separate file. Signed-off-by: Ukri Niemimuukko --- ...viceplugin.intel.com_gpudeviceplugins.yaml | 4 ++ .../v1/fpgadeviceplugin_webhook.go | 28 -------- .../deviceplugin/v1/gpudeviceplugin_types.go | 3 + .../v1/gpudeviceplugin_webhook.go | 28 ++------ pkg/apis/deviceplugin/v1/webhook_common.go | 50 ++++++++++++++ pkg/controllers/gpu/controller.go | 66 ++++++++++++++++++- 6 files changed, 127 insertions(+), 52 deletions(-) create mode 100644 pkg/apis/deviceplugin/v1/webhook_common.go diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml index 958db17f..efb4b665 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml @@ -52,6 +52,10 @@ spec: image: description: Image is a container image with GPU device plugin executable. type: string + initImage: + description: InitImage is a container image with tools (e.g., GPU NFD + source hook) installed on each node. + type: string logLevel: description: LogLevel sets the plugin's log level. minimum: 0 diff --git a/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go index 0c938ac3..e59edc80 100644 --- a/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/fpgadeviceplugin_webhook.go @@ -15,8 +15,6 @@ package v1 import ( - "strings" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" @@ -98,29 +96,3 @@ func (r *FpgaDevicePlugin) validatePlugin() error { return validatePluginImage(r.Spec.InitImage, "intel-fpga-initcontainer", fpgaMinVersion) } - -func validatePluginImage(image, expectedName string, expectedMinVersion *version.Version) error { - parts := strings.SplitN(image, ":", 2) - if len(parts) != 2 { - return errors.Errorf("incorrect image field %q", image) - } - namespacedName := parts[0] - versionStr := parts[1] - - parts = strings.Split(namespacedName, "/") - name := parts[len(parts)-1] - if name != expectedName { - return errors.Errorf("incorrect image name %q. Make sure you use '/%s:'", name, expectedName) - } - - ver, err := version.ParseSemantic(versionStr) - if err != nil { - return errors.Wrapf(err, "unable to parse version %q", versionStr) - } - - if !ver.AtLeast(expectedMinVersion) { - return errors.Errorf("version %q is too low. Should be at least %q", ver, expectedMinVersion) - } - - return nil -} diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go index 2530d56d..fff16d25 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go @@ -28,6 +28,9 @@ type GpuDevicePluginSpec struct { // Image is a container image with GPU device plugin executable. Image string `json:"image,omitempty"` + // InitImage is a container image with tools (e.g., GPU NFD source hook) installed on each node. + InitImage string `json:"initImage,omitempty"` + // SharedDevNum is a number of containers that can share the same GPU device. // +kubebuilder:validation:Minimum=1 SharedDevNum int `json:"sharedDevNum,omitempty"` diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go index 4573cc89..d65c5707 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go @@ -15,8 +15,6 @@ package v1 import ( - "strings" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" @@ -88,27 +86,11 @@ func (r *GpuDevicePlugin) ValidateDelete() error { } func (r *GpuDevicePlugin) validatePlugin() error { - parts := strings.SplitN(r.Spec.Image, ":", 2) - if len(parts) != 2 { - return errors.Errorf("incorrect image field %q", r.Spec.Image) - } - namespacedName := parts[0] - versionStr := parts[1] - - parts = strings.Split(namespacedName, "/") - name := parts[len(parts)-1] - if name != "intel-gpu-plugin" { - return errors.Errorf("incorrect image name %q. Make sure you use '/intel-gpu-plugin:'", name) + if r.Spec.InitImage != "" { + if err := validatePluginImage(r.Spec.InitImage, "intel-gpu-initcontainer", gpuMinVersion); err != nil { + return err + } } - ver, err := version.ParseSemantic(versionStr) - if err != nil { - return errors.Wrapf(err, "unable to parse version %q", versionStr) - } - - if !ver.AtLeast(gpuMinVersion) { - return errors.Errorf("version %q is too low. Should be at least %q", ver, gpuMinVersion) - } - - return nil + return validatePluginImage(r.Spec.Image, "intel-gpu-plugin", gpuMinVersion) } diff --git a/pkg/apis/deviceplugin/v1/webhook_common.go b/pkg/apis/deviceplugin/v1/webhook_common.go new file mode 100644 index 00000000..e0b15580 --- /dev/null +++ b/pkg/apis/deviceplugin/v1/webhook_common.go @@ -0,0 +1,50 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "strings" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/version" +) + +// common functions for webhooks + +func validatePluginImage(image, expectedName string, expectedMinVersion *version.Version) error { + parts := strings.SplitN(image, ":", 2) + if len(parts) != 2 { + return errors.Errorf("incorrect image field %q", image) + } + namespacedName := parts[0] + versionStr := parts[1] + + parts = strings.Split(namespacedName, "/") + name := parts[len(parts)-1] + if name != expectedName { + return errors.Errorf("incorrect image name %q. Make sure you use '/%s:'", name, expectedName) + } + + ver, err := version.ParseSemantic(versionStr) + if err != nil { + return errors.Wrapf(err, "unable to parse version %q", versionStr) + } + + if !ver.AtLeast(expectedMinVersion) { + return errors.Errorf("version %q is too low. Should be at least %q", ver, expectedMinVersion) + } + + return nil +} diff --git a/pkg/controllers/gpu/controller.go b/pkg/controllers/gpu/controller.go index dd5a847a..908f582d 100644 --- a/pkg/controllers/gpu/controller.go +++ b/pkg/controllers/gpu/controller.go @@ -81,7 +81,7 @@ func (c *controller) NewDaemonSet(rawObj runtime.Object) *apps.DaemonSet { } yes := true - return &apps.DaemonSet{ + daemonSet := apps.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: devicePlugin.Namespace, GenerateName: devicePlugin.Name + "-", @@ -170,6 +170,59 @@ func (c *controller) NewDaemonSet(rawObj runtime.Object) *apps.DaemonSet { }, }, } + // add the optional InitImage + if devicePlugin.Spec.InitImage != "" { + setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage) + } + return &daemonSet +} + +func setInitContainer(spec *v1.PodSpec, imageName string) { + yes := true + spec.InitContainers = []v1.Container{ + { + Image: imageName, + ImagePullPolicy: "IfNotPresent", + Name: "intel-gpu-initcontainer", + SecurityContext: &v1.SecurityContext{ + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/etc/kubernetes/node-feature-discovery/source.d/", + Name: "nfd-source-hooks", + }, + }, + }} + directoryOrCreate := v1.HostPathDirectoryOrCreate + missing := true + for _, vol := range spec.Volumes { + if vol.Name == "nfd-source-hooks" { + missing = false + break + } + } + if missing { + spec.Volumes = append(spec.Volumes, v1.Volume{ + Name: "nfd-source-hooks", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/node-feature-discovery/source.d/", + Type: &directoryOrCreate, + }, + }, + }) + } +} + +func removeVolume(volumes []v1.Volume, name string) []v1.Volume { + newVolumes := []v1.Volume{} + for _, volume := range volumes { + if volume.Name != name { + newVolumes = append(newVolumes, volume) + } + } + return newVolumes } func (c *controller) UpdateDaemonSet(rawObj runtime.Object, ds *apps.DaemonSet) (updated bool) { @@ -180,6 +233,17 @@ func (c *controller) UpdateDaemonSet(rawObj runtime.Object, ds *apps.DaemonSet) updated = true } + if dp.Spec.InitImage == "" { + if ds.Spec.Template.Spec.InitContainers != nil { + ds.Spec.Template.Spec.InitContainers = nil + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "nfd-source-hooks") + updated = true + } + } else { + setInitContainer(&ds.Spec.Template.Spec, dp.Spec.InitImage) + updated = true + } + if dp.Spec.NodeSelector == nil { dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} } else {