Merge pull request #915 from hj-johannes-lee/qat-initimage

qat: add initimage to plugin

.github/workflows/ci.yaml

@@ -99,6 +99,7 @@ jobs:
           - intel-gpu-initcontainer
           - intel-gpu-plugin
           - intel-fpga-plugin
+          - intel-qat-initcontainer
           - intel-qat-plugin
           - intel-qat-plugin-kerneldrv
           - intel-vpu-plugin

.github/workflows/e2e-qat.yml

@@ -9,7 +9,7 @@ on:
       - 'release-*'
 
 env:
-  IMAGES: 'intel-qat-plugin crypto-perf'
+  IMAGES: 'intel-qat-plugin intel-qat-initcontainer crypto-perf'
 
 jobs:
   e2e-qat:

build/docker/intel-qat-initcontainer.Dockerfile

@@ -0,0 +1,58 @@
+# Copyright 2022 Intel Corporation. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG GOLANG_BASE=golang:1.17-bullseye
+
+# FINAL_BASE can be used to configure the base image of the final image.
+#
+# This is used in two ways:
+# 1) make <image-name> BUILDER=<docker|buildah>
+# 2) docker build ... -f <image-name>.Dockerfile
+#
+# The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static
+# (see build-image.sh).
+ARG FINAL_BASE=registry.access.redhat.com/ubi8-micro
+
+FROM ${GOLANG_BASE} as builder
+
+ARG DIR=/intel-device-plugins-for-kubernetes
+WORKDIR $DIR
+COPY . .
+
+ARG ROOT=/install_root
+ARG TOYBOX_VERSION="0.8.6"
+ARG TOYBOX_SHA256="e2c4f72a158581a12f4303d0d1aeec196b01f293e495e535bcdaf75eb9ae0987"
+RUN apt update && apt -y install musl musl-tools musl-dev
+RUN curl -SL https://github.com/landley/toybox/archive/refs/tags/$TOYBOX_VERSION.tar.gz -o toybox.tar.gz \
+    && echo "$TOYBOX_SHA256 toybox.tar.gz" | sha256sum -c - \
+    && tar -xzf toybox.tar.gz \
+    && rm toybox.tar.gz \
+    && cd toybox-$TOYBOX_VERSION \
+    && KCONFIG_CONFIG=${DIR}/build/docker/toybox-config LDFLAGS="--static" CC=musl-gcc PREFIX=$ROOT V=2 make toybox install \
+    && install -D LICENSE $ROOT/licenses/toybox \
+    && cp -r /usr/share/doc/musl $ROOT/licenses/
+
+FROM ${FINAL_BASE}
+
+LABEL name='intel-qat-initcontainer'
+LABEL vendor='Intel®'
+LABEL version='devel'
+LABEL release='1'
+LABEL summary='Intel® QAT initcontainer for Kubernetes'
+LABEL description='Intel QAT initcontainer initializes devices'
+
+COPY --from=builder /install_root /
+
+ADD demo/qat-init.sh /qat-init/
+
+ENTRYPOINT /qat-init/qat-init.sh

build/docker/toybox-config

@@ -1,10 +1,10 @@
 #
 # Automatically generated make config: don't edit
 # ToyBox version: KCONFIG_VERSION
-# Wed Dec 22 10:34:08 2021
+# Fri Mar 25 12:05:23 2022
 #
-CONFIG_TOYBOX_CONTAINER=y
+# CONFIG_TOYBOX_CONTAINER is not set
-CONFIG_TOYBOX_FIFREEZE=y
+# CONFIG_TOYBOX_FIFREEZE is not set
 CONFIG_TOYBOX_ICONV=y
 CONFIG_TOYBOX_UTMPX=y
 CONFIG_TOYBOX_SHADOW=y
@@ -14,14 +14,14 @@ CONFIG_TOYBOX_FORK=y
 CONFIG_TOYBOX_PRLIMIT=y
 CONFIG_TOYBOX_GETRANDOM=y
 # CONFIG_TOYBOX_COPYFILERANGE is not set
-# CONFIG_TOYBOX_HASTIMERS is not set
+CONFIG_TOYBOX_HASTIMERS=y
 
 #
 # Posix commands
 #
 # CONFIG_BASENAME is not set
 # CONFIG_CAL is not set
-# CONFIG_CAT is not set
+CONFIG_CAT=y
 # CONFIG_CAT_V is not set
 # CONFIG_CATV is not set
 # CONFIG_CHGRP is not set
@@ -47,9 +47,9 @@ CONFIG_CP=y
 # CONFIG_FILE is not set
 # CONFIG_FIND is not set
 # CONFIG_GETCONF is not set
-# CONFIG_GREP is not set
+CONFIG_GREP=y
-# CONFIG_EGREP is not set
+CONFIG_EGREP=y
-# CONFIG_FGREP is not set
+CONFIG_FGREP=y
 # CONFIG_HEAD is not set
 # CONFIG_ICONV is not set
 # CONFIG_ID is not set
@@ -91,9 +91,9 @@ CONFIG_LS=y
 # CONFIG_STRINGS is not set
 # CONFIG_TAIL is not set
 # CONFIG_TAR is not set
-# CONFIG_TEE is not set
+CONFIG_TEE=y
-# CONFIG_TEST is not set
+CONFIG_TEST=y
-# CONFIG_TEST_GLUE is not set
+CONFIG_TEST_GLUE=y
 # CONFIG_TIME is not set
 # CONFIG_TOUCH is not set
 # CONFIG_TRUE is not set
@@ -242,7 +242,7 @@ CONFIG_SH=y
 # CONFIG_LSATTR is not set
 # CONFIG_CHATTR is not set
 # CONFIG_LSMOD is not set
-# CONFIG_LSPCI is not set
+CONFIG_LSPCI=y
 # CONFIG_LSPCI_TEXT is not set
 # CONFIG_LSUSB is not set
 # CONFIG_MAKEDEVS is not set

demo/qat-init.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940}
+DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done)
+
+for dev in $DEVS; do
+  DEVPATH="/sys/bus/pci/devices/0000:$dev"
+  NUMVFS="$DEVPATH/sriov_numvfs"
+  if test -w "$NUMVFS" -a "$(cat "$NUMVFS")" -eq 0 ; then
+    tee "$NUMVFS" < "$DEVPATH/sriov_totalvfs"
+  fi
+done

deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml

@@ -60,6 +60,10 @@ spec:
               image:
                 description: Image is a container image with QAT device plugin executable.
                 type: string
+              initImage:
+                description: InitImage is a container image with a script that initialize
+                  devices.
+                type: string
               kernelVfDrivers:
                 description: KernelVfDrivers is a list of VF device drivers for the
                   QuickAssist devices in the system.

deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml

@@ -10,6 +10,7 @@ metadata:
   #   container.apparmor.security.beta.kubernetes.io/intel-qat-plugin: unconfined
 spec:
   image: intel/intel-qat-plugin:0.23.0
+  initImage: intel/intel-qat-initcontainer:0.23.0
   dpdkDriver: vfio-pci
   kernelVfDrivers:
     - c6xxvf

deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml

@@ -7,17 +7,7 @@ spec:
     spec:
       initContainers:
       - name: sriov-numvfs
-        image: busybox
+        image: intel/intel-qat-initcontainer:devel
-        command: ["/bin/sh", "-c"]
-        args:
-        - >
-          for dev in `for pf in 0434 0435 19a3 37c8 6f54 4940 18a0; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do
-            DEVPATH="/sys/bus/pci/devices/$dev"
-            NUMVFS="$DEVPATH/sriov_numvfs"
-            if [ -w "$NUMVFS" -a $(cat "$NUMVFS") -eq 0 ]; then
-              cat "$DEVPATH/sriov_totalvfs" | tee "$NUMVFS"
-            fi
-          done
         securityContext:
           readOnlyRootFilesystem: true
           privileged: true

pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go

@@ -33,6 +33,9 @@ type QatDevicePluginSpec struct {
 	// Image is a container image with QAT device plugin executable.
 	Image string `json:"image,omitempty"`
 
+	// InitImage is a container image with a script that initialize devices.
+	InitImage string `json:"initImage,omitempty"`
+
 	// DpdkDriver is a DPDK device driver for configuring the QAT device.
 	// +kubebuilder:validation:Enum=igb_uio;vfio-pci
 	DpdkDriver string `json:"dpdkDriver,omitempty"`

pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go

@@ -85,5 +85,11 @@ func (r *QatDevicePlugin) ValidateDelete() error {
 }
 
 func (r *QatDevicePlugin) validatePlugin() error {
+	if r.Spec.InitImage != "" {
+		if err := validatePluginImage(r.Spec.InitImage, "intel-qat-initcontainer", qatMinVersion); err != nil {
+			return err
+		}
+	}
+
 	return validatePluginImage(r.Spec.Image, "intel-qat-plugin", qatMinVersion)
 }

pkg/controllers/qat/controller.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/reference"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -67,7 +68,7 @@ func (c *controller) CreateEmptyObject() client.Object {
 
 func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
 	dp := obj.(*devicepluginv1.QatDevicePlugin)
-	return controllers.UpgradeImages(&dp.Spec.Image, nil)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
 }
 
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
@@ -92,6 +93,13 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
 	}
 
+	if devicePlugin.Spec.InitImage == "" {
+		daemonSet.Spec.Template.Spec.InitContainers = nil
+		daemonSet.Spec.Template.Spec.Volumes = removeVolume(daemonSet.Spec.Template.Spec.Volumes, "sysfs")
+	} else {
+		setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec)
+	}
+
 	daemonSet.ObjectMeta.Namespace = c.ns
 	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
 	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
@@ -114,6 +122,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if dp.Spec.InitImage == "" {
+		if ds.Spec.Template.Spec.InitContainers != nil {
+			ds.Spec.Template.Spec.InitContainers = nil
+			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs")
+			updated = true
+		}
+	} else {
+		setInitContainer(&ds.Spec.Template.Spec, dp.Spec)
+		updated = true
+	}
+
 	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
@@ -164,6 +183,76 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node
 	return updated, nil
 }
 
+func removeVolume(volumes []v1.Volume, name string) []v1.Volume {
+	newVolumes := []v1.Volume{}
+
+	for _, volume := range volumes {
+		if volume.Name != name {
+			newVolumes = append(newVolumes, volume)
+		}
+	}
+
+	return newVolumes
+}
+
+func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginSpec) {
+	yes := true
+
+	qatDeviceDriver := map[string]string{
+		"dh895xccvf": "0434 0435",
+		"c3xxxvf":    "19e2",
+		"c6xxvf":     "37c8",
+		"d15xxvf":    "6f54",
+		"4xxxvf":     "4940",
+		"c4xxxvf":    "18a0",
+	}
+
+	enablingPfPciIDs := make([]string, 0, len(qatDeviceDriver))
+	for _, v := range dpSpec.KernelVfDrivers {
+		enablingPfPciIDs = append(enablingPfPciIDs, qatDeviceDriver[string(v)])
+	}
+
+	dsSpec.InitContainers = []v1.Container{
+		{
+			Image:           dpSpec.InitImage,
+			ImagePullPolicy: "IfNotPresent",
+			Name:            "init-sriov-numvfs",
+			Env: []v1.EnvVar{{
+				Name:  "ENABLED_QAT_PF_PCIIDS",
+				Value: strings.Join(enablingPfPciIDs, " "),
+			}},
+			SecurityContext: &v1.SecurityContext{
+				Privileged:             &yes,
+				ReadOnlyRootFilesystem: &yes,
+			},
+			VolumeMounts: []v1.VolumeMount{
+				{
+					Name:      "sysfs",
+					MountPath: "/sys",
+				},
+			},
+		}}
+	addVolumeIfMissing(dsSpec, "sysfs", "/sys", v1.HostPathDirectoryOrCreate)
+}
+
+func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) {
+	for _, vol := range spec.Volumes {
+		if vol.Name == name {
+			return
+		}
+	}
+
+	spec.Volumes = append(spec.Volumes, v1.Volume{
+		Name: name,
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: path,
+				Type: &hpType,
+			},
+		},
+	})
+}
+
 func getPodArgs(qdp *devicepluginv1.QatDevicePlugin) []string {
 	args := make([]string, 0, 8)
 	args = append(args, "-v", strconv.Itoa(qdp.Spec.LogLevel))

test/envtest/qatdeviceplugin_controller_test.go

@@ -37,6 +37,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 		It("should handle QatDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.QatDevicePluginSpec{
 				Image:        "qat-testimage",
+				InitImage:    "qat-testinitimage",
 				NodeSelector: map[string]string{"qat-nodeselector": "true"},
 			}
 
@@ -70,6 +71,8 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			ds := &apps.DaemonSet{}
 			_ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds)
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image))
+			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector))
 
 			By("copy annotations successfully")
@@ -88,6 +91,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 
 			By("updating QatDevicePlugin successfully")
 			updatedImage := "updated-qat-testimage"
+			updatedInitImage := "updated-qat-testinitimage"
 			updatedLogLevel := 2
 			updatedDpdkDriver := "igb_uio"
 			updatedKernelVfDrivers := "c3xxxvf"
@@ -95,6 +99,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			updatedNodeSelector := map[string]string{"updated-qat-nodeselector": "true"}
 
 			fetched.Spec.Image = updatedImage
+			fetched.Spec.InitImage = updatedInitImage
 			fetched.Spec.LogLevel = updatedLogLevel
 			fetched.Spec.DpdkDriver = updatedDpdkDriver
 			fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)}
@@ -125,10 +130,15 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage))
+			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))
 
 			By("updating QatDevicePlugin with different values successfully")
+			updatedInitImage = ""
 			updatedNodeSelector = map[string]string{}
+
+			fetched.Spec.InitImage = updatedInitImage
 			fetched.Spec.NodeSelector = updatedNodeSelector
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
@@ -156,10 +166,11 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 	It("upgrades", func() {
 		dp := &devicepluginv1.QatDevicePlugin{}
 
-		var image string
+		var image, initimage string
 
-		testUpgrade("qat", dp, &image, nil)
+		testUpgrade("qat", dp, &image, &initimage)
 
 		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
 	})
 })

test/envtest/suite_test.go

@@ -225,6 +225,7 @@ func makeDevicePlugin(name, image, initimage string) client.Object {
 		obj = &devicepluginv1.QatDevicePlugin{
 			Spec: devicepluginv1.QatDevicePluginSpec{
 				Image:     image,
+				InitImage: initimage,
 			},
 		}
 	case "sgx":