diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f22e1a8f..1467a834 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -120,6 +120,7 @@ jobs: - intel-iaa-plugin - intel-idxd-config-initcontainer - intel-dlb-plugin + - intel-dlb-initcontainer # Demo images - crypto-perf diff --git a/.github/workflows/e2e-dlb.yml b/.github/workflows/e2e-dlb.yml index 49ba3746..67e619f6 100644 --- a/.github/workflows/e2e-dlb.yml +++ b/.github/workflows/e2e-dlb.yml @@ -13,7 +13,7 @@ on: - 'release-*' env: - IMAGES: 'intel-dlb-plugin dlb-libdlb-demo' + IMAGES: 'intel-dlb-plugin intel-dlb-initcontainer dlb-libdlb-demo' permissions: contents: read diff --git a/build/docker/intel-dlb-initcontainer.Dockerfile b/build/docker/intel-dlb-initcontainer.Dockerfile new file mode 100644 index 00000000..3f7f2c1f --- /dev/null +++ b/build/docker/intel-dlb-initcontainer.Dockerfile @@ -0,0 +1,61 @@ +## This is a generated file, do not edit directly. Edit build/docker/templates/intel-dlb-initcontainer.Dockerfile.in instead. +## +## Copyright 2022 Intel Corporation. All Rights Reserved. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +### +## FINAL_BASE can be used to configure the base image of the final image. +## +## This is used in two ways: +## 1) make BUILDER= +## 2) docker build ... -f .Dockerfile +## +## The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static +## (see build-image.sh). +## 2) and the default FINAL_BASE is primarily used to build Redhat Certified Openshift Operator container images that must be UBI based. +## The RedHat build tool does not allow additional image build parameters. +ARG FINAL_BASE=registry.access.redhat.com/ubi8-micro +### +## +## GOLANG_BASE can be used to make the build reproducible by choosing an +## image by its hash: +## GOLANG_BASE=golang@sha256:9d64369fd3c633df71d7465d67d43f63bb31192193e671742fa1c26ebc3a6210 +## +## This is used on release branches before tagging a stable version. +## The main branch defaults to using the latest Golang base image. +ARG GOLANG_BASE=golang:1.19-bullseye +### +FROM ${GOLANG_BASE} as builder +ARG DIR=/intel-device-plugins-for-kubernetes +WORKDIR $DIR +COPY . . +ARG TOYBOX_VERSION="0.8.7" +ARG TOYBOX_SHA256="b6f43d5738df54623ed21c32f430d1d5c5ac7ef465a6a883890f104b59d5d9e4" +ARG ROOT=/install_root +RUN apt update && apt -y install musl musl-tools musl-dev +RUN curl -SL https://github.com/landley/toybox/archive/refs/tags/$TOYBOX_VERSION.tar.gz -o toybox.tar.gz \ + && echo "$TOYBOX_SHA256 toybox.tar.gz" | sha256sum -c - \ + && tar -xzf toybox.tar.gz \ + && rm toybox.tar.gz \ + && cd toybox-$TOYBOX_VERSION \ + && KCONFIG_CONFIG=${DIR}/build/docker/toybox-config LDFLAGS="--static" CC=musl-gcc PREFIX=$ROOT V=2 make toybox install \ + && install -D LICENSE $ROOT/licenses/toybox \ + && cp -r /usr/share/doc/musl $ROOT/licenses/ +### +FROM ${FINAL_BASE} +LABEL vendor='Intel®' +LABEL version='devel' +LABEL release='1' +COPY --from=builder /install_root / +COPY demo/dlb-init.sh /usr/bin/ +ENTRYPOINT [ "/bin/bash", "dlb-init.sh"] diff --git a/build/docker/templates/intel-dlb-initcontainer.Dockerfile.in b/build/docker/templates/intel-dlb-initcontainer.Dockerfile.in new file mode 100644 index 00000000..0e5fb605 --- /dev/null +++ b/build/docker/templates/intel-dlb-initcontainer.Dockerfile.in @@ -0,0 +1,19 @@ +#include "final_base.docker" +#include "golang_base.docker" + +FROM ${GOLANG_BASE} as builder + +ARG DIR=/intel-device-plugins-for-kubernetes +WORKDIR $DIR +COPY . . + +#include "toybox_build.docker" + +FROM ${FINAL_BASE} + +#include "default_labels.docker" + +COPY --from=builder /install_root / + +COPY demo/dlb-init.sh /usr/bin/ +ENTRYPOINT [ "/bin/bash", "dlb-init.sh"] diff --git a/build/docker/toybox-config b/build/docker/toybox-config index bc1f288f..0014707e 100644 --- a/build/docker/toybox-config +++ b/build/docker/toybox-config @@ -1,7 +1,7 @@ # # Automatically generated make config: don't edit # ToyBox version: KCONFIG_VERSION -# Wed Jun 8 09:08:35 2022 +# Thu Sep 29 05:22:30 2022 # CONFIG_TOYBOX_CONTAINER=y CONFIG_TOYBOX_FIFREEZE=y @@ -19,7 +19,7 @@ CONFIG_TOYBOX_COPYFILERANGE=y # # Posix commands # -# CONFIG_BASENAME is not set +CONFIG_BASENAME=y # CONFIG_CAL is not set CONFIG_CAT=y # CONFIG_CHGRP is not set @@ -38,7 +38,7 @@ CONFIG_CP=y # CONFIG_DF is not set # CONFIG_DIRNAME is not set # CONFIG_DU is not set -# CONFIG_ECHO is not set +CONFIG_ECHO=y # CONFIG_ENV is not set # CONFIG_EXPAND is not set # CONFIG_FALSE is not set @@ -48,7 +48,7 @@ CONFIG_CP=y CONFIG_GREP=y CONFIG_EGREP=y CONFIG_FGREP=y -# CONFIG_HEAD is not set +CONFIG_HEAD=y # CONFIG_ICONV is not set # CONFIG_ID is not set # CONFIG_ID_Z is not set @@ -263,7 +263,7 @@ CONFIG_LSPCI=y # CONFIG_READAHEAD is not set # CONFIG_READELF is not set # CONFIG_READLINK is not set -# CONFIG_REALPATH is not set +CONFIG_REALPATH=y # CONFIG_REBOOT is not set # CONFIG_RESET is not set # CONFIG_REV is not set diff --git a/demo/dlb-init.sh b/demo/dlb-init.sh new file mode 100755 index 00000000..041ed171 --- /dev/null +++ b/demo/dlb-init.sh @@ -0,0 +1,49 @@ +#!/bin/sh -eu + +enable_and_configure_vfs() { + devpath=$1 + + sriov_numvfs_path="$devpath/sriov_numvfs" + if ! test -w "$sriov_numvfs_path"; then + echo "error: $sriov_numvfs_path is not found or not writable. Check if dlb driver module is loaded" + exit 1 + fi + if [ "$(cat "$sriov_numvfs_path")" -ne 0 ]; then + echo "$devpath already configured" + exit 0 + fi + + # enable sriov + echo -n 1 > "$sriov_numvfs_path" + + # configure vf + # unbind vf + vf_pciid=$(basename "$(realpath "$devpath/virtfn0")") + dlb_pci_driver_path=/sys/bus/pci/drivers/dlb2 + echo -n "$vf_pciid" > $dlb_pci_driver_path/unbind + + # assign resources to vf + vf_resources_path="$devpath/vf0_resources" + echo -n 2048 > "$vf_resources_path/num_atomic_inflights" + echo -n 2048 > "$vf_resources_path/num_dir_credits" + echo -n 8 > "$vf_resources_path/num_dir_ports" + echo -n 2048 > "$vf_resources_path/num_hist_list_entries" + echo -n 8192 > "$vf_resources_path/num_ldb_credits" + echo -n 4 > "$vf_resources_path/num_ldb_ports" + echo -n 32 > "$vf_resources_path/num_ldb_queues" + echo -n 32 > "$vf_resources_path/num_sched_domains" + echo -n 2 > "$vf_resources_path/num_sn0_slots" + echo -n 2 > "$vf_resources_path/num_sn1_slots" + # bind vf back to dlb2 driver + echo -n "$vf_pciid" > $dlb_pci_driver_path/bind + + echo "$devpath configured" + # TODO: Due to unknown e2e-dlb (Simics) limitations, it is not possible to configure per VF resources based on values + # reported in /sys/bus/pci/devices//total_resources/. Therefore, only one VF with + # values known to work is enabled. This will be improved in the future to make the scrip more meaningful for real-world + # deployments (see #1145). +} + +# use first dlb device to configure a vf +DEVPATH=$(realpath /sys/bus/pci/drivers/dlb2/????:??:??\.0 | head -1) +enable_and_configure_vfs "$DEVPATH" diff --git a/deployments/dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml b/deployments/dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml new file mode 100644 index 00000000..182f24b6 --- /dev/null +++ b/deployments/dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-dlb-plugin +spec: + template: + spec: + initContainers: + - name: intel-dlb-initcontainer + image: intel/intel-dlb-initcontainer:devel + securityContext: + readOnlyRootFilesystem: true + privileged: true + volumeMounts: + - name: sysfs-driver-dlb2 + mountPath: /sys/bus/pci/drivers/dlb2 + - name: sysfs-devices + mountPath: /sys/devices + volumes: + - name: sysfs-driver-dlb2 + hostPath: + path: /sys/bus/pci/drivers/dlb2 + - name: sysfs-devices + hostPath: + path: /sys/devices diff --git a/deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml b/deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml new file mode 100644 index 00000000..1dd41902 --- /dev/null +++ b/deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../../base +patches: +- path: dlb_initcontainer.yaml diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml index 9d8ec8f1..bb259754 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml @@ -53,6 +53,10 @@ spec: image: description: Image is a container image with DLB device plugin executable. type: string + initImage: + description: InitImage is a container image with a script that initializes + devices. + type: string logLevel: description: LogLevel sets the plugin's log level. minimum: 0 diff --git a/deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml index 0feab0f7..1c004f16 100644 --- a/deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml +++ b/deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml @@ -10,6 +10,7 @@ metadata: # container.apparmor.security.beta.kubernetes.io/intel-dlb-plugin: unconfined spec: image: intel/intel-dlb-plugin:0.24.0 + initImage: intel/intel-dlb-initcontainer:0.24.0 logLevel: 4 nodeSelector: intel.feature.node.kubernetes.io/dlb: 'true' diff --git a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_types.go index cd7fd70e..9b552c5a 100644 --- a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_types.go @@ -31,6 +31,9 @@ type DlbDevicePluginSpec struct { // Image is a container image with DLB device plugin executable. Image string `json:"image,omitempty"` + // InitImage is a container image with a script that initializes devices. + InitImage string `json:"initImage,omitempty"` + // LogLevel sets the plugin's log level. // +kubebuilder:validation:Minimum=0 LogLevel int `json:"logLevel,omitempty"` diff --git a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go index 4ac45166..1b35eb56 100644 --- a/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/dlbdeviceplugin_webhook.go @@ -85,5 +85,11 @@ func (r *DlbDevicePlugin) ValidateDelete() error { } func (r *DlbDevicePlugin) validatePlugin() error { + if r.Spec.InitImage != "" { + if err := validatePluginImage(r.Spec.InitImage, "intel-dlb-initcontainer", dlbMinVersion); err != nil { + return err + } + } + return validatePluginImage(r.Spec.Image, "intel-dlb-plugin", dlbMinVersion) } diff --git a/pkg/controllers/dlb/controller.go b/pkg/controllers/dlb/controller.go index e027a710..964bbc0b 100644 --- a/pkg/controllers/dlb/controller.go +++ b/pkg/controllers/dlb/controller.go @@ -22,6 +22,7 @@ import ( "strings" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" @@ -67,7 +68,7 @@ func (c *controller) CreateEmptyObject() client.Object { func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { dp := obj.(*devicepluginv1.DlbDevicePlugin) - return controllers.UpgradeImages(&dp.Spec.Image, nil) + return controllers.UpgradeImages(&dp.Spec.Image, &dp.Spec.InitImage) } func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { @@ -82,17 +83,24 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { devicePlugin := rawObj.(*devicepluginv1.DlbDevicePlugin) - daemonSet := deployments.DLBPluginDaemonSet() + ds := deployments.DLBPluginDaemonSet() if len(devicePlugin.Spec.NodeSelector) > 0 { - daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector + ds.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector } - daemonSet.ObjectMeta.Namespace = c.ns + if devicePlugin.Spec.InitImage == "" { + ds.Spec.Template.Spec.InitContainers = nil + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs-devices", "sysfs-driver-dlb2") + } else { + setInitContainer(&ds.Spec.Template.Spec, devicePlugin.Spec) + } - daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) - daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + ds.ObjectMeta.Namespace = c.ns - return daemonSet + ds.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + ds.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + return ds } func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { @@ -103,6 +111,17 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( updated = true } + if dp.Spec.InitImage == "" { + if ds.Spec.Template.Spec.InitContainers != nil { + ds.Spec.Template.Spec.InitContainers = nil + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs-devices", "sysfs-driver-dlb2") + updated = true + } + } else { + setInitContainer(&ds.Spec.Template.Spec, dp.Spec) + updated = true + } + if len(dp.Spec.NodeSelector) > 0 { if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector @@ -153,6 +172,70 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node return updated, nil } +func removeVolume(volumes []v1.Volume, names ...string) []v1.Volume { + newVolumes := []v1.Volume{} + + for _, volume := range volumes { + for i, name := range names { + if volume.Name == name { + break + } + + if i == len(names)-1 { + newVolumes = append(newVolumes, volume) + } + } + } + + return newVolumes +} + +func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.DlbDevicePluginSpec) { + yes := true + + dsSpec.InitContainers = []v1.Container{ + { + Image: dpSpec.InitImage, + ImagePullPolicy: "IfNotPresent", + Name: "enable-vfs", + SecurityContext: &v1.SecurityContext{ + Privileged: &yes, + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "sysfs-devices", + MountPath: "/sys/devices", + }, + { + Name: "sysfs-driver-dlb2", + MountPath: "/sys/bus/pci/drivers/dlb2", + }, + }, + }} + + addVolumeIfMissing(dsSpec, "sysfs-devices", "/sys/devices", v1.HostPathDirectoryOrCreate) + addVolumeIfMissing(dsSpec, "sysfs-driver-dlb2", "/sys/bus/pci/drivers/dlb2", v1.HostPathDirectoryOrCreate) +} + +func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) { + for _, vol := range spec.Volumes { + if vol.Name == name { + return + } + } + + spec.Volumes = append(spec.Volumes, v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: path, + Type: &hpType, + }, + }, + }) +} + func getPodArgs(gdp *devicepluginv1.DlbDevicePlugin) []string { args := make([]string, 0, 4) args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel)) diff --git a/test/e2e/dlb/dlb.go b/test/e2e/dlb/dlb.go index 6cfb6d2a..2637489a 100644 --- a/test/e2e/dlb/dlb.go +++ b/test/e2e/dlb/dlb.go @@ -30,7 +30,7 @@ import ( ) const ( - kustomizationYaml = "deployments/dlb_plugin/kustomization.yaml" + kustomizationYaml = "deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml" demoPFYaml = "demo/dlb-libdlb-demo-pf-pod.yaml" demoVFYaml = "demo/dlb-libdlb-demo-vf-pod.yaml" ) diff --git a/test/envtest/dlbdeviceplugin_controller_test.go b/test/envtest/dlbdeviceplugin_controller_test.go index af5eebbe..5ad8ad1b 100644 --- a/test/envtest/dlbdeviceplugin_controller_test.go +++ b/test/envtest/dlbdeviceplugin_controller_test.go @@ -37,6 +37,7 @@ var _ = Describe("DlbDevicePlugin Controller", func() { It("should handle DlbDevicePlugin objects correctly", func() { spec := devicepluginv1.DlbDevicePluginSpec{ Image: "dlb-testimage", + InitImage: "dlb-testinitimage", NodeSelector: map[string]string{"feature.node.kubernetes.io/dlb": "true"}, } @@ -65,13 +66,18 @@ var _ = Describe("DlbDevicePlugin Controller", func() { ds := &apps.DaemonSet{} _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-dlb-plugin"}, ds) Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(spec.Image)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(spec.InitImage)) Expect(ds.Spec.Template.Spec.NodeSelector).To(Equal(spec.NodeSelector)) By("updating DlbDevicePlugin successfully") updatedImage := "updated-dlb-testimage" + updatedInitImage := "updated-dlb-testinitimage" updatedNodeSelector := map[string]string{"updated-dlb-nodeselector": "true"} updatedLogLevel := 3 + fetched.Spec.Image = updatedImage + fetched.Spec.InitImage = updatedInitImage fetched.Spec.NodeSelector = updatedNodeSelector fetched.Spec.LogLevel = updatedLogLevel @@ -92,10 +98,15 @@ var _ = Describe("DlbDevicePlugin Controller", func() { } Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) By("updating DlbDevicePlugin with different values successfully") + updatedInitImage = "" updatedNodeSelector = map[string]string{} + + fetched.Spec.InitImage = updatedInitImage fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) @@ -122,11 +133,12 @@ var _ = Describe("DlbDevicePlugin Controller", func() { It("upgrades", func() { dp := &devicepluginv1.DlbDevicePlugin{} - var image string + var image, initimage string - testUpgrade("dlb", dp, &image, nil) + testUpgrade("dlb", dp, &image, &initimage) Expect(dp.Spec.Image == image).To(BeTrue()) + Expect(dp.Spec.InitImage == initimage).To(BeTrue()) }) var _ = AfterEach(func() { diff --git a/test/envtest/suite_test.go b/test/envtest/suite_test.go index a51483df..005409b8 100644 --- a/test/envtest/suite_test.go +++ b/test/envtest/suite_test.go @@ -198,7 +198,8 @@ func makeDevicePlugin(name, image, initimage string) client.Object { case "dlb": obj = &devicepluginv1.DlbDevicePlugin{ Spec: devicepluginv1.DlbDevicePluginSpec{ - Image: image, + Image: image, + InitImage: initimage, }, } case "dsa":