dsa: Add e2e tests

Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>
Signed-off-by: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>

.github/workflows/e2e-dsa.yml

@@ -0,0 +1,34 @@
+name: e2e-dsa
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 * * *'
+  pull_request:
+    branches:
+      - main
+      - 'release-*'
+
+env:
+  IMAGES: 'intel-dsa-plugin intel-idxd-config-initcontainer dsa-accel-config-demo'
+
+jobs:
+  e2e-dsa:
+    name: e2e-dsa
+    runs-on: [self-hosted, linux, x64, dsa]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Describe test environment
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          echo "Actor: ${{ github.actor }}"
+          echo "Ref: ${{ github.ref }}"
+          echo "SHA: ${{ github.sha }}"
+          echo "Images: $IMAGES"
+      - name: Wait for ready state
+        run: ../../../../bmetal/actions-bmetal-runstage.sh waitready
+      - name: Prepare test environment
+        run: ../../../../bmetal/actions-bmetal-runstage.sh prepare
+      - name: Run tests
+        run: ../../../../bmetal/actions-bmetal-runstage.sh test

Makefile

@@ -148,6 +148,9 @@ e2e-sgx:
 e2e-gpu:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "GPU" -delete-namespace-on-failure=false
 
+e2e-dsa:
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "DSA" -delete-namespace-on-failure=false
+
 terrascan:
 	@ls deployments/*/kustomization.yaml | while read f ; \
 	do \

demo/dsa-accel-config-demo-pod.yaml

@@ -2,15 +2,16 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: dsa-accel-config-demo
+  labels:
+    app: dsa-accel-config-demo
 spec:
   containers:
   - name: dsa-accel-config-demo
-    image: dsa-accel-config-demo:devel
+    image: intel/dsa-accel-config-demo:devel
     imagePullPolicy: IfNotPresent
     resources:
       limits:
         dsa.intel.com/wq-user-dedicated: 1
-        dsa.intel.com/wq-user-shared: 1
-        cpu: 1
-
+        # In kernels 5.13-5.17, ENQCMD is disabled (is to be reinstated in 5.18)
+        # dsa.intel.com/wq-user-shared: 1
   restartPolicy: Never

demo/dsa-accel-config-demo/Dockerfile

@@ -1,23 +1,28 @@
-FROM fedora:32 AS builder
+FROM fedora:35 AS builder
 
-RUN dnf install -y wget autoconf automake libtool m4 diffutils file make dnf-utils
-RUN dnf install -y libuuid-devel json-c-devel kmod-devel libudev-devel
+RUN dnf install -y \
+    wget autoconf automake libtool m4 diffutils file make dnf-utils patch \
+    libuuid-devel json-c-devel
 
-ARG ACCEL_CONFIG_VERSION=v2.8
+ARG ACCEL_CONFIG_VERSION=v3.4.6.3
 
 RUN wget -O- https://github.com/intel/idxd-config/archive/accel-config-$ACCEL_CONFIG_VERSION.tar.gz | tar -zx
 
+ADD idxd-reset.patch /
+ADD test_runner_disable_shared_queues.patch /
+
 RUN cd idxd-config-accel-config-$ACCEL_CONFIG_VERSION && \
+    patch -p1 < ../idxd-reset.patch && \
+    patch -p1 < ../test_runner_disable_shared_queues.patch && \
     mkdir m4 && \
     ./autogen.sh && \
     ./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib64 --enable-test=yes --disable-docs && \
     make && \
-    make check && \
     make install
 
-FROM fedora:32
+FROM fedora:35
 
-RUN dnf install -y libuuid json-c kmod udev
+RUN dnf install -y libuuid json-c
 
 COPY --from=builder /lib64/libaccel-config.so.1 /lib64/
 COPY --from=builder /lib64/libaccel-config.so.1.0.0 /lib64/
@@ -25,5 +30,6 @@ RUN ldconfig
 
 COPY --from=builder /usr/bin/accel-config /usr/bin/
 COPY --from=builder /usr/share/accel-config/test /test
+COPY --from=builder /idxd-reset.patch /usr/local/share/package-sources/
 
-ENTRYPOINT cd /test && sed '/_cleanup$/d;/start_dsa$/d;/enable_wqs$/d;/stop_dsa$/d;/disable_wqs$/d' dsa_user_test_runner.sh | sh
+ENTRYPOINT cd /test && /bin/bash -e ./dsa_user_test_runner.sh

demo/dsa-accel-config-demo/idxd-reset.patch

@@ -0,0 +1,26 @@
+From 07e1137a71021dc02df98b942a260b70ff852ca5 Mon Sep 17 00:00:00 2001
+From: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>
+Date: Tue, 3 May 2022 15:28:05 +0300
+Subject: [PATCH] idxd: Ignore reset device error
+
+Signed-off-by: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>
+---
+ accfg/lib/libaccfg.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/accfg/lib/libaccfg.c b/accfg/lib/libaccfg.c
+index be3be69..32eb71f 100644
+--- a/accfg/lib/libaccfg.c
++++ b/accfg/lib/libaccfg.c
+@@ -636,7 +636,7 @@ static void *add_device(void *parent, int id, const char *ctl_base,
+ 
+ 	rc = accfg_set_param(ctx, dfd, "cmd_status", "1", 1);
+ 	/* older drivers don't support writing to cmd_status */
+-	if (rc && rc != -EACCES) {
++	if (rc && rc != -EACCES && rc != -EROFS) {
+ 		err(ctx, "Failed resetting cmd status %d\n", rc);
+ 		close(dfd);
+ 		goto err_device;
+-- 
+2.36.1
+

demo/dsa-accel-config-demo/test_runner_disable_shared_queues.patch

@@ -0,0 +1,45 @@
+--- idxd-config-accel-config-v3.4.6.3/test/dsa_user_test_runner.sh.orig	2022-05-25 20:31:33.185073217 +0300
++++ idxd-config-accel-config-v3.4.6.3/test/dsa_user_test_runner.sh	2022-05-27 22:12:33.447091343 +0300
+@@ -6,7 +6,8 @@
+ 
+ rc="$EXIT_SKIP"
+ 
+-DSA=dsa0
++DEV=`ls /dev/dsa/ | sed -ne 's|wq\([^.]\+\)\(.*\)|dsa\1/wq\1\2|p'`
++DSA=`echo $DEV | cut -f1 -d/`
+ WQ0=wq0.0
+ WQ1=wq0.1
+ 
+@@ -70,7 +71,7 @@
+ 		for xfer_size in $SIZE_1 $SIZE_4K $SIZE_64K $SIZE_1M $SIZE_2M; do
+ 			echo "Testing $xfer_size bytes"
+ 			"$DSATEST" -w "$wq_mode_code" -l "$xfer_size" -o "$opcode" \
+-				-f "$flag" t200 -v
++				-f "$flag" t200 -d "$DEV"
+ 		done
+ 	done
+ }
+@@ -95,14 +96,11 @@
+ 		for xfer_size in $SIZE_1 $SIZE_4K $SIZE_64K $SIZE_1M $SIZE_2M; do
+ 			echo "Testing $xfer_size bytes"
+ 			"$DSATEST" -w "$wq_mode_code" -l "$xfer_size" -o 0x1 -b "$opcode" \
+-				-c 16 -f "$flag" t2000 -v
++				-c 16 -f "$flag" t2000 -d "$DEV"
+ 		done
+ 	done
+ }
+ 
+-_cleanup
+-start_dsa
+-enable_wqs
+ # shellcheck disable=SC2034
+ rc="$EXIT_FAILURE"
+ 
+@@ -120,7 +118,3 @@
+ 	test_op_batch $opcode $flag
+ done
+ 
+-disable_wqs
+-stop_dsa
+-_cleanup
+-exit 0

scripts/set-version.sh

@@ -15,6 +15,6 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
     exit 1
 fi
 
-for file in $(git grep -l '^TAG?*=\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
-    sed -i -e "s;\(^TAG?*=\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
+for file in $(git grep -l '^TAG?*=\|intel/dsa-accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*dsa*.yaml demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
+    sed -i -e "s;\(^TAG?*=\|intel/dsa-accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
 done

test/e2e/deviceplugins_suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import (
 	"github.com/onsi/gomega"
 
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/dlb"
+	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/dsa"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpga"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpgaadmissionwebhook"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu"

test/e2e/dsa/dsa.go

@@ -0,0 +1,93 @@
+// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dsa
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
+	"github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/kubectl"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	admissionapi "k8s.io/pod-security-admission/api"
+)
+
+const (
+	kustomizationYaml = "deployments/dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml"
+	configmapYaml     = "demo/dsa.conf"
+	demoYaml          = "demo/dsa-accel-config-demo-pod.yaml"
+	podName           = "dsa-accel-config-demo"
+)
+
+func init() {
+	ginkgo.Describe("DSA plugin", describe)
+}
+
+func describe() {
+	f := framework.NewDefaultFramework("dsaplugin")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
+
+	kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", kustomizationYaml, err)
+	}
+
+	configmap, err := utils.LocateRepoFile(configmapYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", configmapYaml, err)
+	}
+
+	demoPath, err := utils.LocateRepoFile(demoYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", demoYaml, err)
+	}
+
+	ginkgo.It("runs DSA plugin and a demo workload", func() {
+		ginkgo.By("deploying DSA plugin")
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configmap)
+
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
+
+		ginkgo.By("waiting for DSA plugin's availability")
+		if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
+			labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second); err != nil {
+			framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
+			kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
+			framework.Failf("unable to wait for all pods to be running and ready: %v", err)
+		}
+
+		ginkgo.By("checking the resource is allocatable")
+		if err := utils.WaitForNodesWithResource(f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second); err != nil {
+			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+		}
+
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-f", demoPath)
+
+		ginkgo.By("waiting for the DSA demo to succeed")
+		f.PodClient().WaitForSuccess(podName, 200*time.Second)
+
+		ginkgo.By("getting workload log")
+		log, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, podName)
+
+		if err != nil {
+			framework.Failf("unable to get log from pod: %v", err)
+		}
+
+		framework.Logf("log output: %s", log)
+	})
+}