From df419b3a82a49226c4dde7a05c422cc913f3ec80 Mon Sep 17 00:00:00 2001 From: Hyeongju Johannes Lee Date: Fri, 25 Mar 2022 15:39:04 +0200 Subject: [PATCH] qat: add initimage to plugin Signed-off-by: Hyeongju Johannes Lee --- .github/workflows/ci.yaml | 1 + .github/workflows/e2e-qat.yml | 2 +- .../docker/intel-qat-initcontainer.Dockerfile | 58 ++++++++++++ build/docker/toybox-config | 24 ++--- demo/qat-init.sh | 11 +++ ...viceplugin.intel.com_qatdeviceplugins.yaml | 4 + .../deviceplugin_v1_qatdeviceplugin.yaml | 1 + .../sriov_numvfs/sriov_numvfs_init.yaml | 12 +-- .../deviceplugin/v1/qatdeviceplugin_types.go | 3 + .../v1/qatdeviceplugin_webhook.go | 6 ++ pkg/controllers/qat/controller.go | 91 ++++++++++++++++++- .../qatdeviceplugin_controller_test.go | 15 ++- test/envtest/suite_test.go | 3 +- 13 files changed, 203 insertions(+), 28 deletions(-) create mode 100644 build/docker/intel-qat-initcontainer.Dockerfile create mode 100755 demo/qat-init.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a95a91eb..523f441f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,6 +100,7 @@ jobs: - intel-gpu-initcontainer - intel-gpu-plugin - intel-fpga-plugin + - intel-qat-initcontainer - intel-qat-plugin - intel-qat-plugin-kerneldrv - intel-vpu-plugin diff --git a/.github/workflows/e2e-qat.yml b/.github/workflows/e2e-qat.yml index ca45020b..7ceab3b1 100644 --- a/.github/workflows/e2e-qat.yml +++ b/.github/workflows/e2e-qat.yml @@ -9,7 +9,7 @@ on: - 'release-*' env: - IMAGES: 'intel-qat-plugin crypto-perf' + IMAGES: 'intel-qat-plugin intel-qat-initcontainer crypto-perf' jobs: e2e-qat: diff --git a/build/docker/intel-qat-initcontainer.Dockerfile b/build/docker/intel-qat-initcontainer.Dockerfile new file mode 100644 index 00000000..ad970956 --- /dev/null +++ b/build/docker/intel-qat-initcontainer.Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2022 Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG GOLANG_BASE=golang:1.17-bullseye + +# FINAL_BASE can be used to configure the base image of the final image. +# +# This is used in two ways: +# 1) make BUILDER= +# 2) docker build ... -f .Dockerfile +# +# The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static +# (see build-image.sh). +ARG FINAL_BASE=registry.access.redhat.com/ubi8-micro + +FROM ${GOLANG_BASE} as builder + +ARG DIR=/intel-device-plugins-for-kubernetes +WORKDIR $DIR +COPY . . + +ARG ROOT=/install_root +ARG TOYBOX_VERSION="0.8.6" +ARG TOYBOX_SHA256="e2c4f72a158581a12f4303d0d1aeec196b01f293e495e535bcdaf75eb9ae0987" +RUN apt update && apt -y install musl musl-tools musl-dev +RUN curl -SL https://github.com/landley/toybox/archive/refs/tags/$TOYBOX_VERSION.tar.gz -o toybox.tar.gz \ + && echo "$TOYBOX_SHA256 toybox.tar.gz" | sha256sum -c - \ + && tar -xzf toybox.tar.gz \ + && rm toybox.tar.gz \ + && cd toybox-$TOYBOX_VERSION \ + && KCONFIG_CONFIG=${DIR}/build/docker/toybox-config LDFLAGS="--static" CC=musl-gcc PREFIX=$ROOT V=2 make toybox install \ + && install -D LICENSE $ROOT/licenses/toybox \ + && cp -r /usr/share/doc/musl $ROOT/licenses/ + +FROM ${FINAL_BASE} + +LABEL name='intel-qat-initcontainer' +LABEL vendor='Intel®' +LABEL version='devel' +LABEL release='1' +LABEL summary='Intel® QAT initcontainer for Kubernetes' +LABEL description='Intel QAT initcontainer initializes devices' + +COPY --from=builder /install_root / + +ADD demo/qat-init.sh /qat-init/ + +ENTRYPOINT /qat-init/qat-init.sh \ No newline at end of file diff --git a/build/docker/toybox-config b/build/docker/toybox-config index df9e6d32..93bb0e1b 100644 --- a/build/docker/toybox-config +++ b/build/docker/toybox-config @@ -1,10 +1,10 @@ # # Automatically generated make config: don't edit # ToyBox version: KCONFIG_VERSION -# Wed Dec 22 10:34:08 2021 +# Fri Mar 25 12:05:23 2022 # -CONFIG_TOYBOX_CONTAINER=y -CONFIG_TOYBOX_FIFREEZE=y +# CONFIG_TOYBOX_CONTAINER is not set +# CONFIG_TOYBOX_FIFREEZE is not set CONFIG_TOYBOX_ICONV=y CONFIG_TOYBOX_UTMPX=y CONFIG_TOYBOX_SHADOW=y @@ -14,14 +14,14 @@ CONFIG_TOYBOX_FORK=y CONFIG_TOYBOX_PRLIMIT=y CONFIG_TOYBOX_GETRANDOM=y # CONFIG_TOYBOX_COPYFILERANGE is not set -# CONFIG_TOYBOX_HASTIMERS is not set +CONFIG_TOYBOX_HASTIMERS=y # # Posix commands # # CONFIG_BASENAME is not set # CONFIG_CAL is not set -# CONFIG_CAT is not set +CONFIG_CAT=y # CONFIG_CAT_V is not set # CONFIG_CATV is not set # CONFIG_CHGRP is not set @@ -47,9 +47,9 @@ CONFIG_CP=y # CONFIG_FILE is not set # CONFIG_FIND is not set # CONFIG_GETCONF is not set -# CONFIG_GREP is not set -# CONFIG_EGREP is not set -# CONFIG_FGREP is not set +CONFIG_GREP=y +CONFIG_EGREP=y +CONFIG_FGREP=y # CONFIG_HEAD is not set # CONFIG_ICONV is not set # CONFIG_ID is not set @@ -91,9 +91,9 @@ CONFIG_LS=y # CONFIG_STRINGS is not set # CONFIG_TAIL is not set # CONFIG_TAR is not set -# CONFIG_TEE is not set -# CONFIG_TEST is not set -# CONFIG_TEST_GLUE is not set +CONFIG_TEE=y +CONFIG_TEST=y +CONFIG_TEST_GLUE=y # CONFIG_TIME is not set # CONFIG_TOUCH is not set # CONFIG_TRUE is not set @@ -242,7 +242,7 @@ CONFIG_SH=y # CONFIG_LSATTR is not set # CONFIG_CHATTR is not set # CONFIG_LSMOD is not set -# CONFIG_LSPCI is not set +CONFIG_LSPCI=y # CONFIG_LSPCI_TEXT is not set # CONFIG_LSUSB is not set # CONFIG_MAKEDEVS is not set diff --git a/demo/qat-init.sh b/demo/qat-init.sh new file mode 100755 index 00000000..4d306dbf --- /dev/null +++ b/demo/qat-init.sh @@ -0,0 +1,11 @@ +#!/bin/bash +ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940} +DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done) + +for dev in $DEVS; do + DEVPATH="/sys/bus/pci/devices/0000:$dev" + NUMVFS="$DEVPATH/sriov_numvfs" + if test -w "$NUMVFS" -a "$(cat "$NUMVFS")" -eq 0 ; then + tee "$NUMVFS" < "$DEVPATH/sriov_totalvfs" + fi +done diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml index 4e7798e6..ad7d480c 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml @@ -60,6 +60,10 @@ spec: image: description: Image is a container image with QAT device plugin executable. type: string + initImage: + description: InitImage is a container image with a script that initialize + devices. + type: string kernelVfDrivers: description: KernelVfDrivers is a list of VF device drivers for the QuickAssist devices in the system. diff --git a/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml index 7a11e280..5f203298 100644 --- a/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml +++ b/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml @@ -10,6 +10,7 @@ metadata: # container.apparmor.security.beta.kubernetes.io/intel-qat-plugin: unconfined spec: image: intel/intel-qat-plugin:0.23.0 + initImage: intel/intel-qat-initcontainer:0.23.0 dpdkDriver: vfio-pci kernelVfDrivers: - c6xxvf diff --git a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml index 4f7ccdc6..9472646b 100644 --- a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml +++ b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml @@ -7,17 +7,7 @@ spec: spec: initContainers: - name: sriov-numvfs - image: busybox - command: ["/bin/sh", "-c"] - args: - - > - for dev in `for pf in 0434 0435 19a3 37c8 6f54 4940 18a0; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do - DEVPATH="/sys/bus/pci/devices/$dev" - NUMVFS="$DEVPATH/sriov_numvfs" - if [ -w "$NUMVFS" -a $(cat "$NUMVFS") -eq 0 ]; then - cat "$DEVPATH/sriov_totalvfs" | tee "$NUMVFS" - fi - done + image: intel/intel-qat-initcontainer:devel securityContext: readOnlyRootFilesystem: true privileged: true diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go index bf64bd13..72c01691 100644 --- a/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go @@ -33,6 +33,9 @@ type QatDevicePluginSpec struct { // Image is a container image with QAT device plugin executable. Image string `json:"image,omitempty"` + // InitImage is a container image with a script that initialize devices. + InitImage string `json:"initImage,omitempty"` + // DpdkDriver is a DPDK device driver for configuring the QAT device. // +kubebuilder:validation:Enum=igb_uio;vfio-pci DpdkDriver string `json:"dpdkDriver,omitempty"` diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go index 309f393f..8e0dd446 100644 --- a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go @@ -85,5 +85,11 @@ func (r *QatDevicePlugin) ValidateDelete() error { } func (r *QatDevicePlugin) validatePlugin() error { + if r.Spec.InitImage != "" { + if err := validatePluginImage(r.Spec.InitImage, "intel-qat-initcontainer", qatMinVersion); err != nil { + return err + } + } + return validatePluginImage(r.Spec.Image, "intel-qat-plugin", qatMinVersion) } diff --git a/pkg/controllers/qat/controller.go b/pkg/controllers/qat/controller.go index 49b91230..31b5a280 100644 --- a/pkg/controllers/qat/controller.go +++ b/pkg/controllers/qat/controller.go @@ -22,6 +22,7 @@ import ( "strings" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" @@ -67,7 +68,7 @@ func (c *controller) CreateEmptyObject() client.Object { func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { dp := obj.(*devicepluginv1.QatDevicePlugin) - return controllers.UpgradeImages(&dp.Spec.Image, nil) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) } func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { @@ -92,6 +93,13 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector } + if devicePlugin.Spec.InitImage == "" { + daemonSet.Spec.Template.Spec.InitContainers = nil + daemonSet.Spec.Template.Spec.Volumes = removeVolume(daemonSet.Spec.Template.Spec.Volumes, "sysfs") + } else { + setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec) + } + daemonSet.ObjectMeta.Namespace = c.ns daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image @@ -114,6 +122,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( updated = true } + if dp.Spec.InitImage == "" { + if ds.Spec.Template.Spec.InitContainers != nil { + ds.Spec.Template.Spec.InitContainers = nil + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs") + updated = true + } + } else { + setInitContainer(&ds.Spec.Template.Spec, dp.Spec) + updated = true + } + if len(dp.Spec.NodeSelector) > 0 { if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector @@ -164,6 +183,76 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node return updated, nil } +func removeVolume(volumes []v1.Volume, name string) []v1.Volume { + newVolumes := []v1.Volume{} + + for _, volume := range volumes { + if volume.Name != name { + newVolumes = append(newVolumes, volume) + } + } + + return newVolumes +} + +func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginSpec) { + yes := true + + qatDeviceDriver := map[string]string{ + "dh895xccvf": "0434 0435", + "c3xxxvf": "19e2", + "c6xxvf": "37c8", + "d15xxvf": "6f54", + "4xxxvf": "4940", + "c4xxxvf": "18a0", + } + + enablingPfPciIDs := make([]string, 0, len(qatDeviceDriver)) + for _, v := range dpSpec.KernelVfDrivers { + enablingPfPciIDs = append(enablingPfPciIDs, qatDeviceDriver[string(v)]) + } + + dsSpec.InitContainers = []v1.Container{ + { + Image: dpSpec.InitImage, + ImagePullPolicy: "IfNotPresent", + Name: "init-sriov-numvfs", + Env: []v1.EnvVar{{ + Name: "ENABLED_QAT_PF_PCIIDS", + Value: strings.Join(enablingPfPciIDs, " "), + }}, + SecurityContext: &v1.SecurityContext{ + Privileged: &yes, + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "sysfs", + MountPath: "/sys", + }, + }, + }} + addVolumeIfMissing(dsSpec, "sysfs", "/sys", v1.HostPathDirectoryOrCreate) +} + +func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) { + for _, vol := range spec.Volumes { + if vol.Name == name { + return + } + } + + spec.Volumes = append(spec.Volumes, v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: path, + Type: &hpType, + }, + }, + }) +} + func getPodArgs(qdp *devicepluginv1.QatDevicePlugin) []string { args := make([]string, 0, 8) args = append(args, "-v", strconv.Itoa(qdp.Spec.LogLevel)) diff --git a/test/envtest/qatdeviceplugin_controller_test.go b/test/envtest/qatdeviceplugin_controller_test.go index bce70ea6..6504438a 100644 --- a/test/envtest/qatdeviceplugin_controller_test.go +++ b/test/envtest/qatdeviceplugin_controller_test.go @@ -37,6 +37,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { It("should handle QatDevicePlugin objects correctly", func() { spec := devicepluginv1.QatDevicePluginSpec{ Image: "qat-testimage", + InitImage: "qat-testinitimage", NodeSelector: map[string]string{"qat-nodeselector": "true"}, } @@ -70,6 +71,8 @@ var _ = Describe("QatDevicePlugin Controller", func() { ds := &apps.DaemonSet{} _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds) Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) By("copy annotations successfully") @@ -88,6 +91,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { By("updating QatDevicePlugin successfully") updatedImage := "updated-qat-testimage" + updatedInitImage := "updated-qat-testinitimage" updatedLogLevel := 2 updatedDpdkDriver := "igb_uio" updatedKernelVfDrivers := "c3xxxvf" @@ -95,6 +99,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { updatedNodeSelector := map[string]string{"updated-qat-nodeselector": "true"} fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage fetched.Spec.LogLevel = updatedLogLevel fetched.Spec.DpdkDriver = updatedDpdkDriver fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)} @@ -125,10 +130,15 @@ var _ = Describe("QatDevicePlugin Controller", func() { Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) By("updating QatDevicePlugin with different values successfully") + updatedInitImage = "" updatedNodeSelector = map[string]string{} + + fetched.Spec.InitImage = updatedInitImage fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) @@ -156,10 +166,11 @@ var _ = Describe("QatDevicePlugin Controller", func() { It("upgrades", func() { dp := &devicepluginv1.QatDevicePlugin{} - var image string + var image, initimage string - testUpgrade("qat", dp, &image, nil) + testUpgrade("qat", dp, &image, &initimage) Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) }) }) diff --git a/test/envtest/suite_test.go b/test/envtest/suite_test.go index eb6d5309..dccec45e 100644 --- a/test/envtest/suite_test.go +++ b/test/envtest/suite_test.go @@ -222,7 +222,8 @@ func makeDevicePlugin(name, image, initimage string) client.Object { case "qat": obj = &devicepluginv1.QatDevicePlugin{ Spec: devicepluginv1.QatDevicePluginSpec{ - Image: image, + Image: image, + InitImage: initimage, }, } case "sgx":