iaa: Add e2e tests

Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>
Signed-off-by: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>

.github/workflows/e2e-iaa.yml

@@ -0,0 +1,34 @@
+name: e2e-iaa
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 * * *'
+  pull_request:
+    branches:
+      - main
+      - 'release-*'
+
+env:
+  IMAGES: 'intel-iaa-plugin intel-idxd-config-initcontainer accel-config-demo'
+
+jobs:
+  e2e-iaa:
+    name: e2e-iaa
+    runs-on: [self-hosted, linux, x64, iaa]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Describe test environment
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          echo "Actor: ${{ github.actor }}"
+          echo "Ref: ${{ github.ref }}"
+          echo "SHA: ${{ github.sha }}"
+          echo "Images: $IMAGES"
+      - name: Wait for ready state
+        run: ../../../../bmetal/actions-bmetal-runstage.sh waitready
+      - name: Prepare test environment
+        run: ../../../../bmetal/actions-bmetal-runstage.sh prepare
+      - name: Run tests
+        run: ../../../../bmetal/actions-bmetal-runstage.sh test

Makefile

@@ -154,6 +154,9 @@ e2e-gpu:
 e2e-dsa:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "DSA" -delete-namespace-on-failure=false
 
+e2e-iaa:
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "IAA" -delete-namespace-on-failure=false
+
 terrascan:
 	@ls deployments/*/kustomization.yaml | while read f ; \
 	do \

build/docker/intel-idxd-config-initcontainer.Dockerfile

@@ -30,7 +30,7 @@ COPY --from=builder /usr/bin/accel-config /usr/bin/
 COPY --from=builder /accel-config.tar.gz /licenses/accel-config/
 ADD demo/idxd-init.sh /usr/local/bin/
 ADD demo/dsa.conf /idxd-init/
-ADD demo/iaa.conf /idxd-init/
+ADD demo/iax.conf /idxd-init/
 RUN mkdir /idxd-init/scratch
 WORKDIR /idxd-init
 ENTRYPOINT bash /usr/local/bin/idxd-init.sh

build/docker/templates/intel-idxd-config-initcontainer.Dockerfile.in

@@ -33,7 +33,7 @@ COPY --from=builder /accel-config.tar.gz /licenses/accel-config/
 
 ADD demo/idxd-init.sh /usr/local/bin/
 ADD demo/dsa.conf /idxd-init/
-ADD demo/iaa.conf /idxd-init/
+ADD demo/iax.conf /idxd-init/
 
 RUN mkdir /idxd-init/scratch
 

cmd/iaa_plugin/iaa_plugin.go

@@ -30,7 +30,7 @@ const (
 	// Device directories.
 	devDir = "/dev/iax"
 	// Glob pattern for the state sysfs entry.
-	statePattern = "/sys/bus/iax/devices/iax*/wq*/state"
+	statePattern = "/sys/bus/dsa/devices/iax*/wq*/state"
 )
 
 func main() {

demo/accel-config-demo/test_runner_disable_shared_queues.patch

@@ -43,3 +43,44 @@
 -stop_dsa
 -_cleanup
 -exit 0
+--- idxd-config-accel-config-v3.4.6.4/test/iaa_user_test_runner.sh.orig	2022-06-07 20:52:39.582379032 +0300
++++ idxd-config-accel-config-v3.4.6.4/test/iaa_user_test_runner.sh	2022-06-07 20:57:14.972589001 +0300
+@@ -6,7 +6,8 @@
+ 
+ rc="$EXIT_SKIP"
+ 
+-IAX=iax1
++DEV=`ls /dev/iax/ | sed -ne 's|wq\([^.]\+\)\(.*\)|iax\1/wq\1\2|p'`
++IAX=`echo $DEV | cut -f1 -d/`
+ WQ0=wq1.4
+ WQ1=wq1.1
+ 
+@@ -72,18 +73,15 @@
+ 			if [ "$extra_flag" != "" ]
+ 			then
+ 				"$IAXTEST" -w "$wq_mode_code" -l "$xfer_size" -o "$opcode" \
+-					-f "$flag" -e "$extra_flag" -t 5000 -v
++					-f "$flag" -e "$extra_flag" -t 5000 -d "$DEV"
+ 			else
+ 				"$IAXTEST" -w "$wq_mode_code" -l "$xfer_size" -o "$opcode" \
+-					-f "$flag" -t 5000 -v
++					-f "$flag" -t 5000 -d "$DEV"
+ 			fi
+ 		done
+ 	done
+ }
+ 
+-_cleanup
+-start_iax
+-enable_wqs
+ # shellcheck disable=SC2034
+ rc="$EXIT_FAILURE"
+ 
+@@ -99,7 +97,3 @@
+ 	test_op $opcode $flag
+ done
+ 
+-disable_wqs
+-stop_iax
+-_cleanup
+-exit 0

demo/iaa-accel-config-demo-pod.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: iaa-accel-config-demo
+  labels:
+    app: iaa-accel-config-demo
+spec:
+  containers:
+  - name: iaa-accel-config-demo
+    image: intel/accel-config-demo:devel
+    command: [ "/bin/bash", "-c", "cd /test && /bin/bash -e ./iaa_user_test_runner.sh" ]
+    imagePullPolicy: IfNotPresent
+    resources:
+      limits:
+        iaa.intel.com/wq-user-dedicated: 1
+        # In kernels 5.13-5.17, ENQCMD is disabled (is to be reinstated in 5.18)
+        # iaa.intel.com/wq-user-shared: 1
+  restartPolicy: Never

deployments/iaa_plugin/base/intel-iaa-plugin.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /dev/char
           readOnly: true
         - name: sysfs
-          mountPath: /sys/bus/iax
+          mountPath: /sys/bus/dsa/devices
           readOnly: true
         - name: kubeletsockets
           mountPath: /var/lib/kubelet/device-plugins
@@ -46,7 +46,7 @@ spec:
           path: /dev/char
       - name: sysfs
         hostPath:
-          path: /sys/bus/iax
+          path: /sys/bus/dsa/devices
       - name: kubeletsockets
         hostPath:
           path: /var/lib/kubelet/device-plugins

deployments/iaa_plugin/overlays/iaa_initcontainer/iaa_initcontainer.yaml

@@ -13,7 +13,7 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
           - name: IDXD_DEVICE_TYPE
-            value: "iaa"
+            value: "iax"
         image: intel/intel-idxd-config-initcontainer:devel
         securityContext:
           readOnlyRootFilesystem: true

scripts/set-version.sh

@@ -15,6 +15,6 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
     exit 1
 fi
 
-for file in $(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*dsa*.yaml demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
+for file in $(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
     sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
 done

test/e2e/deviceplugins_suite_test.go

@@ -27,6 +27,7 @@ import (
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpga"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpgaadmissionwebhook"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu"
+	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/iaa"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/qat"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgx"
 	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgxadmissionwebhook"

test/e2e/iaa/iaa.go

@@ -0,0 +1,93 @@
+// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iaa
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
+	"github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/kubectl"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	admissionapi "k8s.io/pod-security-admission/api"
+)
+
+const (
+	kustomizationYaml = "deployments/iaa_plugin/overlays/iaa_initcontainer/iaa_initcontainer.yaml"
+	configmapYaml     = "demo/iax.conf"
+	demoYaml          = "demo/iaa-accel-config-demo-pod.yaml"
+	podName           = "iaa-accel-config-demo"
+)
+
+func init() {
+	ginkgo.Describe("IAA plugin", describe)
+}
+
+func describe() {
+	f := framework.NewDefaultFramework("iaaplugin")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
+
+	kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", kustomizationYaml, err)
+	}
+
+	configmap, err := utils.LocateRepoFile(configmapYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", configmapYaml, err)
+	}
+
+	demoPath, err := utils.LocateRepoFile(demoYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", demoYaml, err)
+	}
+
+	ginkgo.It("runs IAA plugin and a demo workload", func() {
+		ginkgo.By("deploying IAA plugin")
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "create", "configmap", "intel-iaa-config", "--from-file="+configmap)
+
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
+
+		ginkgo.By("waiting for IAA plugin's availability")
+		if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
+			labels.Set{"app": "intel-iaa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second); err != nil {
+			framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
+			kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
+			framework.Failf("unable to wait for all pods to be running and ready: %v", err)
+		}
+
+		ginkgo.By("checking the resource is allocatable")
+		if err := utils.WaitForNodesWithResource(f.ClientSet, "iaa.intel.com/wq-user-dedicated", 300*time.Second); err != nil {
+			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+		}
+
+		framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-f", demoPath)
+
+		ginkgo.By("waiting for the IAA demo to succeed")
+		f.PodClient().WaitForSuccess(podName, 200*time.Second)
+
+		ginkgo.By("getting workload log")
+		log, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, podName)
+
+		if err != nil {
+			framework.Failf("unable to get log from pod: %v", err)
+		}
+
+		framework.Logf("log output: %s", log)
+	})
+}