From 710aaaee2315995b71fa55f2924c69b19d2c1fe8 Mon Sep 17 00:00:00 2001 From: Oleg Zhurakivskyy Date: Mon, 30 May 2022 19:26:50 +0300 Subject: [PATCH] dsa: Add e2e tests Signed-off-by: Ed Bartosh Signed-off-by: Oleg Zhurakivskyy --- .github/workflows/e2e-dsa.yml | 34 +++++++ Makefile | 3 + demo/dsa-accel-config-demo-pod.yaml | 9 +- demo/dsa-accel-config-demo/Dockerfile | 22 +++-- demo/dsa-accel-config-demo/idxd-reset.patch | 26 ++++++ .../test_runner_disable_shared_queues.patch | 45 +++++++++ scripts/set-version.sh | 4 +- test/e2e/deviceplugins_suite_test.go | 3 +- test/e2e/dsa/dsa.go | 93 +++++++++++++++++++ 9 files changed, 224 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/e2e-dsa.yml create mode 100644 demo/dsa-accel-config-demo/idxd-reset.patch create mode 100644 demo/dsa-accel-config-demo/test_runner_disable_shared_queues.patch create mode 100644 test/e2e/dsa/dsa.go diff --git a/.github/workflows/e2e-dsa.yml b/.github/workflows/e2e-dsa.yml new file mode 100644 index 00000000..188360f6 --- /dev/null +++ b/.github/workflows/e2e-dsa.yml @@ -0,0 +1,34 @@ +name: e2e-dsa +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * *' + pull_request: + branches: + - main + - 'release-*' + +env: + IMAGES: 'intel-dsa-plugin intel-idxd-config-initcontainer dsa-accel-config-demo' + +jobs: + e2e-dsa: + name: e2e-dsa + runs-on: [self-hosted, linux, x64, dsa] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Describe test environment + run: | + echo "Event name: ${{ github.event_name }}" + echo "Actor: ${{ github.actor }}" + echo "Ref: ${{ github.ref }}" + echo "SHA: ${{ github.sha }}" + echo "Images: $IMAGES" + - name: Wait for ready state + run: ../../../../bmetal/actions-bmetal-runstage.sh waitready + - name: Prepare test environment + run: ../../../../bmetal/actions-bmetal-runstage.sh prepare + - name: Run tests + run: ../../../../bmetal/actions-bmetal-runstage.sh test diff --git a/Makefile b/Makefile index 51122860..dbe6cc68 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,9 @@ e2e-sgx: e2e-gpu: @$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "GPU" -delete-namespace-on-failure=false +e2e-dsa: + @$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "DSA" -delete-namespace-on-failure=false + terrascan: @ls deployments/*/kustomization.yaml | while read f ; \ do \ diff --git a/demo/dsa-accel-config-demo-pod.yaml b/demo/dsa-accel-config-demo-pod.yaml index 51e439d9..d070019b 100644 --- a/demo/dsa-accel-config-demo-pod.yaml +++ b/demo/dsa-accel-config-demo-pod.yaml @@ -2,15 +2,16 @@ apiVersion: v1 kind: Pod metadata: name: dsa-accel-config-demo + labels: + app: dsa-accel-config-demo spec: containers: - name: dsa-accel-config-demo - image: dsa-accel-config-demo:devel + image: intel/dsa-accel-config-demo:devel imagePullPolicy: IfNotPresent resources: limits: dsa.intel.com/wq-user-dedicated: 1 - dsa.intel.com/wq-user-shared: 1 - cpu: 1 - + # In kernels 5.13-5.17, ENQCMD is disabled (is to be reinstated in 5.18) + # dsa.intel.com/wq-user-shared: 1 restartPolicy: Never diff --git a/demo/dsa-accel-config-demo/Dockerfile b/demo/dsa-accel-config-demo/Dockerfile index 7fe64b81..04eea978 100644 --- a/demo/dsa-accel-config-demo/Dockerfile +++ b/demo/dsa-accel-config-demo/Dockerfile @@ -1,23 +1,28 @@ -FROM fedora:32 AS builder +FROM fedora:35 AS builder -RUN dnf install -y wget autoconf automake libtool m4 diffutils file make dnf-utils -RUN dnf install -y libuuid-devel json-c-devel kmod-devel libudev-devel +RUN dnf install -y \ + wget autoconf automake libtool m4 diffutils file make dnf-utils patch \ + libuuid-devel json-c-devel -ARG ACCEL_CONFIG_VERSION=v2.8 +ARG ACCEL_CONFIG_VERSION=v3.4.6.3 RUN wget -O- https://github.com/intel/idxd-config/archive/accel-config-$ACCEL_CONFIG_VERSION.tar.gz | tar -zx +ADD idxd-reset.patch / +ADD test_runner_disable_shared_queues.patch / + RUN cd idxd-config-accel-config-$ACCEL_CONFIG_VERSION && \ + patch -p1 < ../idxd-reset.patch && \ + patch -p1 < ../test_runner_disable_shared_queues.patch && \ mkdir m4 && \ ./autogen.sh && \ ./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib64 --enable-test=yes --disable-docs && \ make && \ - make check && \ make install -FROM fedora:32 +FROM fedora:35 -RUN dnf install -y libuuid json-c kmod udev +RUN dnf install -y libuuid json-c COPY --from=builder /lib64/libaccel-config.so.1 /lib64/ COPY --from=builder /lib64/libaccel-config.so.1.0.0 /lib64/ @@ -25,5 +30,6 @@ RUN ldconfig COPY --from=builder /usr/bin/accel-config /usr/bin/ COPY --from=builder /usr/share/accel-config/test /test +COPY --from=builder /idxd-reset.patch /usr/local/share/package-sources/ -ENTRYPOINT cd /test && sed '/_cleanup$/d;/start_dsa$/d;/enable_wqs$/d;/stop_dsa$/d;/disable_wqs$/d' dsa_user_test_runner.sh | sh +ENTRYPOINT cd /test && /bin/bash -e ./dsa_user_test_runner.sh diff --git a/demo/dsa-accel-config-demo/idxd-reset.patch b/demo/dsa-accel-config-demo/idxd-reset.patch new file mode 100644 index 00000000..3097c3b7 --- /dev/null +++ b/demo/dsa-accel-config-demo/idxd-reset.patch @@ -0,0 +1,26 @@ +From 07e1137a71021dc02df98b942a260b70ff852ca5 Mon Sep 17 00:00:00 2001 +From: Oleg Zhurakivskyy +Date: Tue, 3 May 2022 15:28:05 +0300 +Subject: [PATCH] idxd: Ignore reset device error + +Signed-off-by: Oleg Zhurakivskyy +--- + accfg/lib/libaccfg.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/accfg/lib/libaccfg.c b/accfg/lib/libaccfg.c +index be3be69..32eb71f 100644 +--- a/accfg/lib/libaccfg.c ++++ b/accfg/lib/libaccfg.c +@@ -636,7 +636,7 @@ static void *add_device(void *parent, int id, const char *ctl_base, + + rc = accfg_set_param(ctx, dfd, "cmd_status", "1", 1); + /* older drivers don't support writing to cmd_status */ +- if (rc && rc != -EACCES) { ++ if (rc && rc != -EACCES && rc != -EROFS) { + err(ctx, "Failed resetting cmd status %d\n", rc); + close(dfd); + goto err_device; +-- +2.36.1 + diff --git a/demo/dsa-accel-config-demo/test_runner_disable_shared_queues.patch b/demo/dsa-accel-config-demo/test_runner_disable_shared_queues.patch new file mode 100644 index 00000000..d4010702 --- /dev/null +++ b/demo/dsa-accel-config-demo/test_runner_disable_shared_queues.patch @@ -0,0 +1,45 @@ +--- idxd-config-accel-config-v3.4.6.3/test/dsa_user_test_runner.sh.orig 2022-05-25 20:31:33.185073217 +0300 ++++ idxd-config-accel-config-v3.4.6.3/test/dsa_user_test_runner.sh 2022-05-27 22:12:33.447091343 +0300 +@@ -6,7 +6,8 @@ + + rc="$EXIT_SKIP" + +-DSA=dsa0 ++DEV=`ls /dev/dsa/ | sed -ne 's|wq\([^.]\+\)\(.*\)|dsa\1/wq\1\2|p'` ++DSA=`echo $DEV | cut -f1 -d/` + WQ0=wq0.0 + WQ1=wq0.1 + +@@ -70,7 +71,7 @@ + for xfer_size in $SIZE_1 $SIZE_4K $SIZE_64K $SIZE_1M $SIZE_2M; do + echo "Testing $xfer_size bytes" + "$DSATEST" -w "$wq_mode_code" -l "$xfer_size" -o "$opcode" \ +- -f "$flag" t200 -v ++ -f "$flag" t200 -d "$DEV" + done + done + } +@@ -95,14 +96,11 @@ + for xfer_size in $SIZE_1 $SIZE_4K $SIZE_64K $SIZE_1M $SIZE_2M; do + echo "Testing $xfer_size bytes" + "$DSATEST" -w "$wq_mode_code" -l "$xfer_size" -o 0x1 -b "$opcode" \ +- -c 16 -f "$flag" t2000 -v ++ -c 16 -f "$flag" t2000 -d "$DEV" + done + done + } + +-_cleanup +-start_dsa +-enable_wqs + # shellcheck disable=SC2034 + rc="$EXIT_FAILURE" + +@@ -120,7 +118,3 @@ + test_op_batch $opcode $flag + done + +-disable_wqs +-stop_dsa +-_cleanup +-exit 0 diff --git a/scripts/set-version.sh b/scripts/set-version.sh index 21baeb85..947ddb8c 100755 --- a/scripts/set-version.sh +++ b/scripts/set-version.sh @@ -15,6 +15,6 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then exit 1 fi -for file in $(git grep -l '^TAG?*=\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do - sed -i -e "s;\(^TAG?*=\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file"; +for file in $(git grep -l '^TAG?*=\|intel/dsa-accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*dsa*.yaml demo/*fpga*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do + sed -i -e "s;\(^TAG?*=\|intel/dsa-accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file"; done diff --git a/test/e2e/deviceplugins_suite_test.go b/test/e2e/deviceplugins_suite_test.go index 5ef85e41..015f5b6d 100644 --- a/test/e2e/deviceplugins_suite_test.go +++ b/test/e2e/deviceplugins_suite_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import ( "github.com/onsi/gomega" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/dlb" + _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/dsa" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpga" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpgaadmissionwebhook" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu" diff --git a/test/e2e/dsa/dsa.go b/test/e2e/dsa/dsa.go new file mode 100644 index 00000000..7e1e794c --- /dev/null +++ b/test/e2e/dsa/dsa.go @@ -0,0 +1,93 @@ +// Copyright 2021-2022 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dsa + +import ( + "path/filepath" + "time" + + "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils" + "github.com/onsi/ginkgo" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/kubectl" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + admissionapi "k8s.io/pod-security-admission/api" +) + +const ( + kustomizationYaml = "deployments/dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml" + configmapYaml = "demo/dsa.conf" + demoYaml = "demo/dsa-accel-config-demo-pod.yaml" + podName = "dsa-accel-config-demo" +) + +func init() { + ginkgo.Describe("DSA plugin", describe) +} + +func describe() { + f := framework.NewDefaultFramework("dsaplugin") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml) + if err != nil { + framework.Failf("unable to locate %q: %v", kustomizationYaml, err) + } + + configmap, err := utils.LocateRepoFile(configmapYaml) + if err != nil { + framework.Failf("unable to locate %q: %v", configmapYaml, err) + } + + demoPath, err := utils.LocateRepoFile(demoYaml) + if err != nil { + framework.Failf("unable to locate %q: %v", demoYaml, err) + } + + ginkgo.It("runs DSA plugin and a demo workload", func() { + ginkgo.By("deploying DSA plugin") + framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configmap) + + framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath)) + + ginkgo.By("waiting for DSA plugin's availability") + if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name, + labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second); err != nil { + framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) + kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) + framework.Failf("unable to wait for all pods to be running and ready: %v", err) + } + + ginkgo.By("checking the resource is allocatable") + if err := utils.WaitForNodesWithResource(f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second); err != nil { + framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err) + } + + framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-f", demoPath) + + ginkgo.By("waiting for the DSA demo to succeed") + f.PodClient().WaitForSuccess(podName, 200*time.Second) + + ginkgo.By("getting workload log") + log, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, podName) + + if err != nil { + framework.Failf("unable to get log from pod: %v", err) + } + + framework.Logf("log output: %s", log) + }) +}