dlb: add initcontainer to plugin

initcontainer enables vfs and configures vfs
 - only first pf is used to configure a vf
 - only one vf is configured from the pf
add dlb-initcontainer kustomize overlay
update CRD to have initImage
implment operator to run initcontainer
update e2e test to run initcontainer overlay
update envtest to test initimage

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

.github/workflows/ci.yaml

@@ -120,6 +120,7 @@ jobs:
           - intel-iaa-plugin
           - intel-idxd-config-initcontainer
           - intel-dlb-plugin
+          - intel-dlb-initcontainer
 
           # Demo images
           - crypto-perf

.github/workflows/e2e-dlb.yml

@@ -13,7 +13,7 @@ on:
       - 'release-*'
 
 env:
-  IMAGES: 'intel-dlb-plugin dlb-libdlb-demo'
+  IMAGES: 'intel-dlb-plugin intel-dlb-initcontainer dlb-libdlb-demo'
 
 permissions:
   contents: read

build/docker/intel-dlb-initcontainer.Dockerfile

@@ -0,0 +1,61 @@
+## This is a generated file, do not edit directly. Edit build/docker/templates/intel-dlb-initcontainer.Dockerfile.in instead.
+##
+## Copyright 2022 Intel Corporation. All Rights Reserved.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+###
+## FINAL_BASE can be used to configure the base image of the final image.
+##
+## This is used in two ways:
+## 1) make <image-name> BUILDER=<docker|buildah>
+## 2) docker build ... -f <image-name>.Dockerfile
+##
+## The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static
+## (see build-image.sh).
+## 2) and the default FINAL_BASE is primarily used to build Redhat Certified Openshift Operator container images that must be UBI based.
+## The RedHat build tool does not allow additional image build parameters.
+ARG FINAL_BASE=registry.access.redhat.com/ubi8-micro
+###
+##
+## GOLANG_BASE can be used to make the build reproducible by choosing an
+## image by its hash:
+## GOLANG_BASE=golang@sha256:9d64369fd3c633df71d7465d67d43f63bb31192193e671742fa1c26ebc3a6210
+##
+## This is used on release branches before tagging a stable version.
+## The main branch defaults to using the latest Golang base image.
+ARG GOLANG_BASE=golang:1.19-bullseye
+###
+FROM ${GOLANG_BASE} as builder
+ARG DIR=/intel-device-plugins-for-kubernetes
+WORKDIR $DIR
+COPY . .
+ARG TOYBOX_VERSION="0.8.7"
+ARG TOYBOX_SHA256="b6f43d5738df54623ed21c32f430d1d5c5ac7ef465a6a883890f104b59d5d9e4"
+ARG ROOT=/install_root
+RUN apt update && apt -y install musl musl-tools musl-dev
+RUN curl -SL https://github.com/landley/toybox/archive/refs/tags/$TOYBOX_VERSION.tar.gz -o toybox.tar.gz \
+    && echo "$TOYBOX_SHA256 toybox.tar.gz" | sha256sum -c - \
+    && tar -xzf toybox.tar.gz \
+    && rm toybox.tar.gz \
+    && cd toybox-$TOYBOX_VERSION \
+    && KCONFIG_CONFIG=${DIR}/build/docker/toybox-config LDFLAGS="--static" CC=musl-gcc PREFIX=$ROOT V=2 make toybox install \
+    && install -D LICENSE $ROOT/licenses/toybox \
+    && cp -r /usr/share/doc/musl $ROOT/licenses/
+###
+FROM ${FINAL_BASE}
+LABEL vendor='Intel®'
+LABEL version='devel'
+LABEL release='1'
+COPY --from=builder /install_root /
+COPY demo/dlb-init.sh /usr/bin/
+ENTRYPOINT [ "/bin/bash", "dlb-init.sh"]

build/docker/templates/intel-dlb-initcontainer.Dockerfile.in

@@ -0,0 +1,19 @@
+#include "final_base.docker"
+#include "golang_base.docker"
+
+FROM ${GOLANG_BASE} as builder
+
+ARG DIR=/intel-device-plugins-for-kubernetes
+WORKDIR $DIR
+COPY . .
+
+#include "toybox_build.docker"
+
+FROM ${FINAL_BASE}
+
+#include "default_labels.docker"
+
+COPY --from=builder /install_root /
+
+COPY demo/dlb-init.sh /usr/bin/
+ENTRYPOINT [ "/bin/bash", "dlb-init.sh"]

build/docker/toybox-config

@@ -1,7 +1,7 @@
 #
 # Automatically generated make config: don't edit
 # ToyBox version: KCONFIG_VERSION
-# Wed Jun  8 09:08:35 2022
+# Thu Sep 29 05:22:30 2022
 #
 CONFIG_TOYBOX_CONTAINER=y
 CONFIG_TOYBOX_FIFREEZE=y
@@ -19,7 +19,7 @@ CONFIG_TOYBOX_COPYFILERANGE=y
 #
 # Posix commands
 #
-# CONFIG_BASENAME is not set
+CONFIG_BASENAME=y
 # CONFIG_CAL is not set
 CONFIG_CAT=y
 # CONFIG_CHGRP is not set
@@ -38,7 +38,7 @@ CONFIG_CP=y
 # CONFIG_DF is not set
 # CONFIG_DIRNAME is not set
 # CONFIG_DU is not set
-# CONFIG_ECHO is not set
+CONFIG_ECHO=y
 # CONFIG_ENV is not set
 # CONFIG_EXPAND is not set
 # CONFIG_FALSE is not set
@@ -48,7 +48,7 @@ CONFIG_CP=y
 CONFIG_GREP=y
 CONFIG_EGREP=y
 CONFIG_FGREP=y
-# CONFIG_HEAD is not set
+CONFIG_HEAD=y
 # CONFIG_ICONV is not set
 # CONFIG_ID is not set
 # CONFIG_ID_Z is not set
@@ -263,7 +263,7 @@ CONFIG_LSPCI=y
 # CONFIG_READAHEAD is not set
 # CONFIG_READELF is not set
 # CONFIG_READLINK is not set
-# CONFIG_REALPATH is not set
+CONFIG_REALPATH=y
 # CONFIG_REBOOT is not set
 # CONFIG_RESET is not set
 # CONFIG_REV is not set

demo/dlb-init.sh

@@ -0,0 +1,49 @@
+#!/bin/sh -eu
+
+enable_and_configure_vfs() {
+  devpath=$1
+  
+  sriov_numvfs_path="$devpath/sriov_numvfs"
+  if ! test -w "$sriov_numvfs_path"; then
+    echo "error: $sriov_numvfs_path is not found or not writable. Check if dlb driver module is loaded"
+    exit 1
+  fi
+  if [ "$(cat "$sriov_numvfs_path")" -ne 0 ]; then
+    echo "$devpath already configured"
+    exit 0
+  fi
+
+  # enable sriov
+  echo -n 1 > "$sriov_numvfs_path"
+
+  # configure vf
+  # unbind vf
+  vf_pciid=$(basename "$(realpath "$devpath/virtfn0")")
+  dlb_pci_driver_path=/sys/bus/pci/drivers/dlb2
+  echo -n "$vf_pciid" > $dlb_pci_driver_path/unbind
+
+  # assign resources to vf
+  vf_resources_path="$devpath/vf0_resources"
+  echo -n 2048 > "$vf_resources_path/num_atomic_inflights"
+  echo -n 2048 > "$vf_resources_path/num_dir_credits"
+  echo -n 8 > "$vf_resources_path/num_dir_ports"
+  echo -n 2048 > "$vf_resources_path/num_hist_list_entries"
+  echo -n 8192 > "$vf_resources_path/num_ldb_credits"
+  echo -n 4 > "$vf_resources_path/num_ldb_ports"
+  echo -n 32 > "$vf_resources_path/num_ldb_queues"
+  echo -n 32 > "$vf_resources_path/num_sched_domains"
+  echo -n 2 > "$vf_resources_path/num_sn0_slots"
+  echo -n 2 > "$vf_resources_path/num_sn1_slots"
+  # bind vf back to dlb2 driver
+  echo -n "$vf_pciid" > $dlb_pci_driver_path/bind
+
+  echo "$devpath configured"
+  # TODO: Due to unknown e2e-dlb (Simics) limitations, it is not possible to configure per VF resources based on values
+  # reported in /sys/bus/pci/devices/<dev_name>/total_resources/<resource_name>. Therefore, only one VF with 
+  # values known to work is enabled. This will be improved in the future to make the scrip more meaningful for real-world
+  # deployments (see #1145).
+}
+
+# use first dlb device to configure a vf
+DEVPATH=$(realpath /sys/bus/pci/drivers/dlb2/????:??:??\.0 | head -1)
+enable_and_configure_vfs "$DEVPATH"

deployments/dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-dlb-plugin
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: intel-dlb-initcontainer
+        image: intel/intel-dlb-initcontainer:devel
+        securityContext:
+          readOnlyRootFilesystem: true
+          privileged: true
+        volumeMounts:
+        - name: sysfs-driver-dlb2
+          mountPath: /sys/bus/pci/drivers/dlb2
+        - name: sysfs-devices
+          mountPath: /sys/devices
+      volumes:
+      - name: sysfs-driver-dlb2
+        hostPath:
+          path: /sys/bus/pci/drivers/dlb2
+      - name: sysfs-devices
+        hostPath:
+          path: /sys/devices

deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ../../base
+patches:
+- path: dlb_initcontainer.yaml

deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml

@@ -53,6 +53,10 @@ spec:
               image:
                 description: Image is a container image with DLB device plugin executable.
                 type: string
+              initImage:
+                description: InitImage is a container image with a script that initializes
+                  devices.
+                type: string
               logLevel:
                 description: LogLevel sets the plugin's log level.
                 minimum: 0

deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml

@@ -10,6 +10,7 @@ metadata:
   #   container.apparmor.security.beta.kubernetes.io/intel-dlb-plugin: unconfined
 spec:
   image: intel/intel-dlb-plugin:0.24.0
+  initImage: intel/intel-dlb-initcontainer:0.24.0
   logLevel: 4
   nodeSelector:
     intel.feature.node.kubernetes.io/dlb: 'true'

pkg/apis/deviceplugin/v1/dlbdeviceplugin_types.go

@@ -31,6 +31,9 @@ type DlbDevicePluginSpec struct {
 	// Image is a container image with DLB device plugin executable.
 	Image string `json:"image,omitempty"`
 
+	// InitImage is a container image with a script that initializes devices.
+	InitImage string `json:"initImage,omitempty"`
+
 	// LogLevel sets the plugin's log level.
 	// +kubebuilder:validation:Minimum=0
 	LogLevel int `json:"logLevel,omitempty"`

pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go

@@ -85,5 +85,11 @@ func (r *DlbDevicePlugin) ValidateDelete() error {
 }
 
 func (r *DlbDevicePlugin) validatePlugin() error {
+	if r.Spec.InitImage != "" {
+		if err := validatePluginImage(r.Spec.InitImage, "intel-dlb-initcontainer", dlbMinVersion); err != nil {
+			return err
+		}
+	}
+
 	return validatePluginImage(r.Spec.Image, "intel-dlb-plugin", dlbMinVersion)
 }

pkg/controllers/dlb/controller.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/reference"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -67,7 +68,7 @@ func (c *controller) CreateEmptyObject() client.Object {
 
 func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool {
 	dp := obj.(*devicepluginv1.DlbDevicePlugin)
-	return controllers.UpgradeImages(&dp.Spec.Image, nil)
+	return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage)
 }
 
 func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
@@ -82,17 +83,24 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.DlbDevicePlugin)
 
-	daemonSet := deployments.DLBPluginDaemonSet()
+	ds := deployments.DLBPluginDaemonSet()
 	if len(devicePlugin.Spec.NodeSelector) > 0 {
-		daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
+		ds.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector
 	}
 
-	daemonSet.ObjectMeta.Namespace = c.ns
+	if devicePlugin.Spec.InitImage == "" {
+		ds.Spec.Template.Spec.InitContainers = nil
+		ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs-devices", "sysfs-driver-dlb2")
+	} else {
+		setInitContainer(&ds.Spec.Template.Spec, devicePlugin.Spec)
+	}
 
-	daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
+	ds.ObjectMeta.Namespace = c.ns
-	daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
 
-	return daemonSet
+	ds.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin)
+	ds.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image
+
+	return ds
 }
 
 func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) {
@@ -103,6 +111,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	if dp.Spec.InitImage == "" {
+		if ds.Spec.Template.Spec.InitContainers != nil {
+			ds.Spec.Template.Spec.InitContainers = nil
+			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs-devices", "sysfs-driver-dlb2")
+			updated = true
+		}
+	} else {
+		setInitContainer(&ds.Spec.Template.Spec, dp.Spec)
+		updated = true
+	}
+
 	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
@@ -153,6 +172,70 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node
 	return updated, nil
 }
 
+func removeVolume(volumes []v1.Volume, names ...string) []v1.Volume {
+	newVolumes := []v1.Volume{}
+
+	for _, volume := range volumes {
+		for i, name := range names {
+			if volume.Name == name {
+				break
+			}
+
+			if i == len(names)-1 {
+				newVolumes = append(newVolumes, volume)
+			}
+		}
+	}
+
+	return newVolumes
+}
+
+func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.DlbDevicePluginSpec) {
+	yes := true
+
+	dsSpec.InitContainers = []v1.Container{
+		{
+			Image:           dpSpec.InitImage,
+			ImagePullPolicy: "IfNotPresent",
+			Name:            "enable-vfs",
+			SecurityContext: &v1.SecurityContext{
+				Privileged:             &yes,
+				ReadOnlyRootFilesystem: &yes,
+			},
+			VolumeMounts: []v1.VolumeMount{
+				{
+					Name:      "sysfs-devices",
+					MountPath: "/sys/devices",
+				},
+				{
+					Name:      "sysfs-driver-dlb2",
+					MountPath: "/sys/bus/pci/drivers/dlb2",
+				},
+			},
+		}}
+
+	addVolumeIfMissing(dsSpec, "sysfs-devices", "/sys/devices", v1.HostPathDirectoryOrCreate)
+	addVolumeIfMissing(dsSpec, "sysfs-driver-dlb2", "/sys/bus/pci/drivers/dlb2", v1.HostPathDirectoryOrCreate)
+}
+
+func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) {
+	for _, vol := range spec.Volumes {
+		if vol.Name == name {
+			return
+		}
+	}
+
+	spec.Volumes = append(spec.Volumes, v1.Volume{
+		Name: name,
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: path,
+				Type: &hpType,
+			},
+		},
+	})
+}
+
 func getPodArgs(gdp *devicepluginv1.DlbDevicePlugin) []string {
 	args := make([]string, 0, 4)
 	args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel))

test/e2e/dlb/dlb.go

@@ -30,7 +30,7 @@ import (
 )
 
 const (
-	kustomizationYaml = "deployments/dlb_plugin/kustomization.yaml"
+	kustomizationYaml = "deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml"
 	demoPFYaml        = "demo/dlb-libdlb-demo-pf-pod.yaml"
 	demoVFYaml        = "demo/dlb-libdlb-demo-vf-pod.yaml"
 )

test/envtest/dlbdeviceplugin_controller_test.go

@@ -37,6 +37,7 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 		It("should handle DlbDevicePlugin objects correctly", func() {
 			spec := devicepluginv1.DlbDevicePluginSpec{
 				Image:        "dlb-testimage",
+				InitImage:    "dlb-testinitimage",
 				NodeSelector: map[string]string{"feature.node.kubernetes.io/dlb": "true"},
 			}
 
@@ -65,13 +66,18 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 			ds := &apps.DaemonSet{}
 			_ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dlb-plugin"}, ds)
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image))
+			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector))
 
 			By("updating DlbDevicePlugin successfully")
 			updatedImage := "updated-dlb-testimage"
+			updatedInitImage := "updated-dlb-testinitimage"
 			updatedNodeSelector := map[string]string{"updated-dlb-nodeselector": "true"}
 			updatedLogLevel := 3
+
 			fetched.Spec.Image = updatedImage
+			fetched.Spec.InitImage = updatedInitImage
 			fetched.Spec.NodeSelector = updatedNodeSelector
 			fetched.Spec.LogLevel = updatedLogLevel
 
@@ -92,10 +98,15 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 			}
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage))
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))
+			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))
 
 			By("updating DlbDevicePlugin with different values successfully")
+			updatedInitImage = ""
 			updatedNodeSelector = map[string]string{}
+
+			fetched.Spec.InitImage = updatedInitImage
 			fetched.Spec.NodeSelector = updatedNodeSelector
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
@@ -122,11 +133,12 @@ var _ = Describe("DlbDevicePlugin Controller", func() {
 	It("upgrades", func() {
 		dp := &devicepluginv1.DlbDevicePlugin{}
 
-		var image string
+		var image, initimage string
 
-		testUpgrade("dlb", dp, &image, nil)
+		testUpgrade("dlb", dp, &image, &initimage)
 
 		Expect(dp.Spec.Image == image).To(BeTrue())
+		Expect(dp.Spec.InitImage == initimage).To(BeTrue())
 	})
 
 	var _ = AfterEach(func() {

test/envtest/suite_test.go

@@ -198,7 +198,8 @@ func makeDevicePlugin(name, image, initimage string) client.Object {
 	case "dlb":
 		obj = &devicepluginv1.DlbDevicePlugin{
 			Spec: devicepluginv1.DlbDevicePluginSpec{
-				Image: image,
+				Image:     image,
+				InitImage: initimage,
 			},
 		}
 	case "dsa":