Merge pull request #1234 from hj-johannes-lee/qat-cfgServices

qat: add configuration of cfgServices to qat initcontainer

build/docker/intel-qat-initcontainer.Dockerfile

@@ -62,4 +62,5 @@ LABEL summary='Intel® QAT initcontainer for Kubernetes'
 LABEL description='Intel QAT initcontainer initializes devices'
 COPY --from=builder /install_root /
 COPY demo/qat-init.sh /usr/local/bin/
+WORKDIR /qat-init
 ENTRYPOINT [ "/bin/bash", "/usr/local/bin/qat-init.sh"]

build/docker/templates/intel-qat-initcontainer.Dockerfile.in

@@ -21,4 +21,6 @@ COPY --from=builder /install_root /
 
 COPY demo/qat-init.sh /usr/local/bin/
 
+WORKDIR /qat-init
+
 ENTRYPOINT [ "/bin/bash", "/usr/local/bin/qat-init.sh"]

build/docker/toybox-config

@@ -1,7 +1,7 @@
 #
 # Automatically generated make config: don't edit
 # ToyBox version: KCONFIG_VERSION
-# Tue Oct 11 13:47:43 2022
+# Thu Nov 17 14:23:21 2022
 #
 # CONFIG_TOYBOX_ON_ANDROID is not set
 CONFIG_TOYBOX_FORK=y
@@ -23,7 +23,7 @@ CONFIG_CP=y
 # CONFIG_MV is not set
 # CONFIG_INSTALL is not set
 # CONFIG_CPIO is not set
-# CONFIG_CUT is not set
+CONFIG_CUT=y
 # CONFIG_DATE is not set
 # CONFIG_DF is not set
 # CONFIG_DIRNAME is not set

cmd/qat_plugin/README.md

@@ -120,6 +120,34 @@ $ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/
 > socket creation and kubelet registration. Furthermore, the deployments `securityContext` must
 > be configured with appropriate `runAsUser/runAsGroup`.
 
+#### Automatic Provisioning
+
+There's a sample [qat initcontainer](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/build/docker/intel-qat-initcontainer.Dockerfile). Regardless of device types, the script running inside the initcontainer enables QAT SR-IOV VFs. 
+
+To deploy, run as follows:
+
+```bash
+$ kubectl apply -k deployments/qat_plugin/overlays/qat_initcontainer/
+```
+
+In addition to the default configuration, you can add device-specific configurations via ConfigMap.
+
+| Device | Possible Configuration | How To Customize | Options | Notes | 
+|:-------|:-----------------------|:-----------------|:--------|:------|
+| 4xxx, 401xx | [cfg_services](https://github.com/torvalds/linux/blob/42e66b1cc3a070671001f8a1e933a80818a192bf/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=<value>` | compress:`dc`, crypto:`sym;asym` | Linux 6.0+ kernel is required. | 
+
+To create a provisioning config after customizing, run as follows:
+
+```bash
+$ kubectl create configmap --namespace=inteldeviceplugins-system qat-config --from-file=deployments/qat_plugin/overlays/qat_initcontainer/qat.conf
+```
+> **Note**: When deploying the overlay qat_initcontainer, such a manual creation is not necessary since ConfigMap is generated automatically. Just set the values in the config file and deploy the overlay.
+
+When using the operator for deploying the plugin with provisioning config, use `provisioningConfig` field for the name of the ConfigMap, then the config is passed to initcontainer through the volume mount.
+
+There's also a possibility for a node specific congfiguration through passing a nodename via `NODE_NAME` into initcontainer's environment and passing a node specific profile (`qat-$NODE_NAME.conf`) via ConfigMap volume mount.
+
+
 #### Verify Plugin Registration
 
 Verification of the plugin deployment and detection of QAT hardware can be confirmed by

demo/qat-init.sh

@@ -1,8 +1,55 @@
-#!/bin/sh -eu
+#!/bin/sh
+# This script is based on qatlib's qat_init.sh
+NODE_NAME="${NODE_NAME:-}"
 ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942}
 DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done)
+SERVICES_LIST="sym;asym dc"
+QAT_4XXX_DEVICE_PCI_ID="0x4940"
+QAT_401XX_DEVICE_PCI_ID="0x4942"
+SERVICES_ENABLED="NONE"
+SERVICES_ENABLED_FOUND="FALSE"
 
-for dev in $DEVS; do
+check_config() {
+  [ -f "conf/qat.conf" ] && SERVICES_ENABLED=$(cut -d= -f 2 conf/qat.conf | grep '\S')
+  [ -f "conf/qat-$NODE_NAME.conf" ] && SERVICES_ENABLED=$(cut -d= -f 2 conf/qat-$NODE_NAME.conf | grep '\S')
+
+  if [ "$SERVICES_ENABLED" != "NONE" ]; then
+    SERVICES_ENABLED_FOUND="FALSE"
+    for SERVICE in $SERVICES_LIST
+    do
+      if [ "$SERVICE" = "$SERVICES_ENABLED" ]; then
+        SERVICES_ENABLED_FOUND="TRUE"
+        break
+      fi
+    done
+  fi
+}
+
+sysfs_config() {
+  if [ "$SERVICES_ENABLED_FOUND" = "TRUE" ]; then
+    for dev in $DEVS; do
+      DEVPATH="/sys/bus/pci/devices/0000:$dev"
+      PCI_DEV=$(cat "$DEVPATH"/device 2> /dev/null)
+      if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ]; then
+        continue
+      fi
+
+      CURRENT_SERVICES=$(cat "$DEVPATH"/qat/cfg_services)
+      if [ "$CURRENT_SERVICES" != "$SERVICES_ENABLED" ]; then
+        CURRENT_STATE=$(cat "$DEVPATH"/qat/state)
+        if [ "$CURRENT_STATE" = "up" ]; then
+          echo down > "$DEVPATH"/qat/state
+        fi
+        echo "$SERVICES_ENABLED" > "$DEVPATH"/qat/cfg_services
+        CURRENT_SERVICES=$(cat "$DEVPATH"/qat/cfg_services)
+      fi
+      echo "Device $dev configured with services: $CURRENT_SERVICES"
+    done
+  fi
+}
+
+enable_sriov() {
+  for dev in $DEVS; do
   DEVPATH="/sys/bus/pci/devices/0000:$dev"
   NUMVFS="$DEVPATH/sriov_numvfs"
   if ! test -w "$NUMVFS"; then
@@ -14,4 +61,9 @@ for dev in $DEVS; do
   else
     tee "$NUMVFS" < "$DEVPATH/sriov_totalvfs"
   fi
-done
+  done
+}
+
+check_config
+sysfs_config
+enable_sriov

deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml

@@ -102,6 +102,10 @@ spec:
                 - balanced
                 - packed
                 type: string
+              provisioningConfig:
+                description: ProvisioningConfig is a ConfigMap used to pass the configuration
+                  of QAT devices into qat initcontainer.
+                type: string
             type: object
           status:
             description: 'QatDevicePluginStatus defines the observed state of QatDevicePlugin.

deployments/qat_plugin/base/intel-qat-plugin.yaml

@@ -16,6 +16,11 @@ spec:
       automountServiceAccountToken: false
       containers:
       - name: intel-qat-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
         image: intel/intel-qat-plugin:devel
         securityContext:
           seLinuxOptions:

deployments/qat_plugin/overlays/e2e/kustomization.yaml

@@ -3,7 +3,7 @@ commonAnnotations:
   container.apparmor.security.beta.kubernetes.io/intel-qat-plugin: unconfined
 
 resources:
-- ../sriov_numvfs
+- ../qat_initcontainer
 
 patches:
   - path: add-args.yaml

deployments/qat_plugin/overlays/qat_initcontainer/kustomization.yaml

@@ -0,0 +1,8 @@
+bases:
+- ../../base
+patchesStrategicMerge:
+- qat_initcontainer.yaml
+configMapGenerator:
+- name: qat-config
+  files:
+  - qat.conf

deployments/qat_plugin/overlays/qat_initcontainer/qat.conf

@@ -0,0 +1 @@
+ServicesEnabled=sym;asym

deployments/qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml

@@ -6,7 +6,12 @@ spec:
   template:
     spec:
       initContainers:
-      - name: sriov-numvfs
+      - name: intel-qat-initcontainer
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
         image: intel/intel-qat-initcontainer:devel
         securityContext:
           readOnlyRootFilesystem: true
@@ -14,7 +19,13 @@ spec:
         volumeMounts:
         - name: sysfs
           mountPath: /sys
+        - name: qat-config
+          mountPath: /qat-init/conf
       volumes:
       - name: sysfs
         hostPath:
           path: /sys
+      - name: qat-config
+        configMap:
+          name: qat-config
+          defaultMode: 0440

deployments/qat_plugin/overlays/sriov_numvfs/kustomization.yaml

@@ -1,4 +0,0 @@
-bases:
-- ../../base
-patchesStrategicMerge:
-- sriov_numvfs_init.yaml

pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go

@@ -35,6 +35,10 @@ type QatDevicePluginSpec struct {
 
 	// InitImage is a container image with a script that initialize devices.
 	InitImage string `json:"initImage,omitempty"`
+
+	// ProvisioningConfig is a ConfigMap used to pass the configuration of QAT devices into qat initcontainer.
+	ProvisioningConfig string `json:"provisioningConfig,omitempty"`
+
 	// PreferredAllocationPolicy sets the mode of allocating QAT devices on a node.
 	// See documentation for detailed description of the policies.
 	// +kubebuilder:validation:Enum=balanced;packed

pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go

@@ -91,5 +91,28 @@ func (r *QatDevicePlugin) validatePlugin() error {
 		}
 	}
 
+	if len(r.Spec.ProvisioningConfig) > 0 {
+		if len(r.Spec.InitImage) == 0 {
+			return errors.Errorf("ProvisioningConfig is set with no InitImage")
+		}
+
+		// check if 4xxxvf is enabled
+		contains := false
+		devicesWithCapabilities := map[KernelVfDriver]struct{}{
+			"4xxxvf": {},
+		}
+
+		for _, kernelVfDriver := range r.Spec.KernelVfDrivers {
+			if _, ok := devicesWithCapabilities[kernelVfDriver]; ok {
+				contains = true
+				break
+			}
+		}
+
+		if !contains {
+			return errors.Errorf("ProvisioningConfig is available only for 4xxx devices")
+		}
+	}
+
 	return validatePluginImage(r.Spec.Image, "intel-qat-plugin", qatMinVersion)
 }

pkg/controllers/qat/controller.go

@@ -34,7 +34,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-const ownerKey = ".metadata.controller.qat"
+const (
+	ownerKey          = ".metadata.controller.qat"
+	initcontainerName = "intel-qat-initcontainer"
+)
 
 var defaultNodeSelector = deployments.QATPluginDaemonSet().Spec.Template.Spec.NodeSelector
 
@@ -126,6 +129,7 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		if ds.Spec.Template.Spec.InitContainers != nil {
 			ds.Spec.Template.Spec.InitContainers = nil
 			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs")
+			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "qat-config")
 			updated = true
 		}
 	} else {
@@ -217,10 +221,20 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS
 			Image:           dpSpec.InitImage,
 			ImagePullPolicy: "IfNotPresent",
 			Name:            "init-sriov-numvfs",
-			Env: []v1.EnvVar{{
-				Name:  "ENABLED_QAT_PF_PCIIDS",
-				Value: strings.Join(enablingPfPciIDs, " "),
-			}},
+			Env: []v1.EnvVar{
+				{
+					Name:  "ENABLED_QAT_PF_PCIIDS",
+					Value: strings.Join(enablingPfPciIDs, " "),
+				},
+				{
+					Name: "NODE_NAME",
+					ValueFrom: &v1.EnvVarSource{
+						FieldRef: &v1.ObjectFieldSelector{
+							FieldPath: "spec.nodeName",
+						},
+					},
+				},
+			},
 			SecurityContext: &v1.SecurityContext{
 				SELinuxOptions: &v1.SELinuxOptions{
 					Type: "container_device_plugin_init_t",
@@ -236,6 +250,29 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS
 			},
 		}}
 	addVolumeIfMissing(dsSpec, "sysfs", "/sys", v1.HostPathDirectoryOrCreate)
+
+	mode := int32(0440)
+
+	if dpSpec.ProvisioningConfig != "" {
+		dsSpec.Volumes = append(dsSpec.Volumes, v1.Volume{
+			Name: "qat-config",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{Name: dpSpec.ProvisioningConfig},
+					DefaultMode:          &mode,
+				},
+			},
+		})
+
+		for i, initcontainer := range dsSpec.InitContainers {
+			if initcontainer.Name == initcontainerName {
+				dsSpec.InitContainers[i].VolumeMounts = append(dsSpec.InitContainers[i].VolumeMounts, v1.VolumeMount{
+					Name:      "qat-config",
+					MountPath: "/qat-init/conf",
+				})
+			}
+		}
+	}
 }
 
 func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) {

pkg/controllers/qat/controller_test.go

@@ -38,7 +38,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 	no := false
 	pluginAnnotations := devicePlugin.ObjectMeta.DeepCopy().Annotations
 
-	return &apps.DaemonSet{
+	daemonSet := apps.DaemonSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "DaemonSet",
 			APIVersion: "apps/v1",
@@ -68,7 +68,17 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 					AutomountServiceAccountToken: &no,
 					Containers: []v1.Container{
 						{
-							Name:            appLabel,
+							Name: appLabel,
+							Env: []v1.EnvVar{
+								{
+									Name: "NODE_NAME",
+									ValueFrom: &v1.EnvVarSource{
+										FieldRef: &v1.ObjectFieldSelector{
+											FieldPath: "spec.nodeName",
+										},
+									},
+								},
+							},
 							Args:            getPodArgs(devicePlugin),
 							Image:           devicePlugin.Spec.Image,
 							ImagePullPolicy: "IfNotPresent",
@@ -140,6 +150,12 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 			},
 		},
 	}
+	// add the optional init container
+	if devicePlugin.Spec.InitImage != "" {
+		setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec)
+	}
+
+	return &daemonSet
 }
 
 // Test that QAT daemonset created by using go:embed is

test/envtest/qatdeviceplugin_controller_test.go

@@ -22,6 +22,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
@@ -92,6 +93,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			By("updating QatDevicePlugin successfully")
 			updatedImage := "updated-qat-testimage"
 			updatedInitImage := "updated-qat-testinitimage"
+			updatedProvisioningConfig := "updated-qat-provisioningconfig"
 			updatedLogLevel := 2
 			updatedDpdkDriver := "igb_uio"
 			updatedKernelVfDrivers := "c3xxxvf"
@@ -101,6 +103,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 
 			fetched.Spec.Image = updatedImage
 			fetched.Spec.InitImage = updatedInitImage
+			fetched.Spec.ProvisioningConfig = updatedProvisioningConfig
 			fetched.Spec.LogLevel = updatedLogLevel
 			fetched.Spec.DpdkDriver = updatedDpdkDriver
 			fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)}
@@ -131,11 +134,24 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 				"-allocation-policy",
 				updatedPreferredAllocationPolicy,
 			}
+			mode := int32(0440)
+			expectedVolume := v1.Volume{
+				Name: "qat-config",
+				VolumeSource: v1.VolumeSource{
+					ConfigMap: &v1.ConfigMapVolumeSource{
+						LocalObjectReference: v1.LocalObjectReference{Name: updatedProvisioningConfig},
+						DefaultMode:          &mode,
+					},
+				},
+			}
 
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage))
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
 			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage))
+
+			Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(expectedVolume))
+
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))
 
 			By("updating QatDevicePlugin with different values successfully")
@@ -151,6 +167,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			By("checking DaemonSet is updated with different values successfully")
 			_ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds)
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0))
+			Expect(ds.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(expectedVolume))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64")))
 
 			By("deleting QatDevicePlugin successfully")