From 71505e6d8db0e431572473c565ef845a76a15ec1 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Thu, 6 Feb 2025 12:11:55 +0200 Subject: [PATCH] dsa: dpdk example workload and use it in e2e Signed-off-by: Tuomas Katila --- .github/workflows/lib-build.yaml | 1 + .github/workflows/lib-e2e.yaml | 3 ++- README.md | 1 + cmd/dsa_plugin/README.md | 22 ++++++++++++++++++++++ demo/dsa-dpdk-dmadevtest.yaml | 28 ++++++++++++++++++++++++++++ demo/dsa-dpdk-dmadevtest/Dockerfile | 28 ++++++++++++++++++++++++++++ demo/dsa.conf | 8 ++++---- go.mod | 1 - go.sum | 4 ++-- scripts/set-version.sh | 4 ++-- test/e2e/dsa/dsa.go | 15 +++++++++++++++ 11 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 demo/dsa-dpdk-dmadevtest.yaml create mode 100644 demo/dsa-dpdk-dmadevtest/Dockerfile diff --git a/.github/workflows/lib-build.yaml b/.github/workflows/lib-build.yaml index d8a7dc32..caadebd0 100644 --- a/.github/workflows/lib-build.yaml +++ b/.github/workflows/lib-build.yaml @@ -35,6 +35,7 @@ jobs: - openssl-qat-engine - sgx-sdk-demo - sgx-aesmd-demo + - dsa-dpdk-dmadevtest builder: [buildah, docker] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 diff --git a/.github/workflows/lib-e2e.yaml b/.github/workflows/lib-e2e.yaml index 7fb9055a..f1551f49 100644 --- a/.github/workflows/lib-e2e.yaml +++ b/.github/workflows/lib-e2e.yaml @@ -12,7 +12,7 @@ jobs: matrix: include: - name: e2e-dsa-gnr - targetjob: e2e-dsa + targetjob: e2e-dsa SKIP="(App:dpdk-test)" runner: simics-gnr images: - intel-dsa-plugin @@ -44,6 +44,7 @@ jobs: - intel-dsa-plugin - intel-idxd-config-initcontainer - accel-config-demo + - dsa-dpdk-dmadevtest - intel-deviceplugin-operator - intel-iaa-plugin - crypto-perf diff --git a/README.md b/README.md index 00611dbd..31ddfe29 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ The summary of resources available via plugins in this repository is given in th * [dlb-libdlb-demo-pod.yaml](demo/dlb-libdlb-demo-pod.yaml) * `dsa.intel.com` : `wq-user-[shared or dedicated]` * [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml) + * [dsa-dpdk-dmadevtest.yaml](demo/dsa-dpdk-dmadevtest.yaml) * `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings) * [intelfpga-job.yaml](demo/intelfpga-job.yaml) * `gpu.intel.com` : `i915`, `i915_monitoring`, `xe` or `xe_monitoring` diff --git a/cmd/dsa_plugin/README.md b/cmd/dsa_plugin/README.md index 416ddf9c..5fe74533 100644 --- a/cmd/dsa_plugin/README.md +++ b/cmd/dsa_plugin/README.md @@ -7,6 +7,7 @@ Table of Contents * [Pre-built Images](#pre-built-images) * [Verify Plugin Registration](#verify-plugin-registration) * [Testing and Demos](#testing-and-demos) + * [DPDK application](#dpdk-application) ## Introduction @@ -133,3 +134,24 @@ We can test the plugin is working by deploying the provided example accel-config ---- ------ ---- ---- ------- Warning FailedScheduling 2m26s default-scheduler 0/1 nodes are available: 1 Insufficient dsa.intel.com/wq-user-dedicated, 1 Insufficient dsa.intel.com/wq-user-shared. ``` +### DPDK application + +DPDK use requires a couple of changes for the DSA configuration. The workqueue mode has to be `dedicated` and the name has to have a prefix of `dpdk_`. + +e.g. +``` +{ + "dev":"wqX.0", + "mode":"dedicated", + "size":16, + "group_id":0, + "priority":10, + "block_on_fault":1, + "type":"user", + "name":"dpdk_X0", + "driver_name":"user", + "threshold":15 +} +``` + +For testing DPDK, one can use the test container in the [demo directory](../../demo/dsa-dpdk-dmadevtest/) and its deployment [file](../../demo/dsa-dpdk-dmadevtest.yaml). diff --git a/demo/dsa-dpdk-dmadevtest.yaml b/demo/dsa-dpdk-dmadevtest.yaml new file mode 100644 index 00000000..ebd8ad04 --- /dev/null +++ b/demo/dsa-dpdk-dmadevtest.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dpdk +spec: + restartPolicy: Never + containers: + - name: dpdk + image: intel/dsa-dpdk-dmadevtest:devel + securityContext: + capabilities: + add: ["SYS_RAWIO"] + volumeMounts: + - mountPath: /mnt/hugepage + name: hugepage + resources: + requests: + hugepages-2Mi: 64Mi + memory: 128Mi + dsa.intel.com/wq-user-dedicated: 1 + limits: + hugepages-2Mi: 64Mi + memory: 128Mi + dsa.intel.com/wq-user-dedicated: 1 + volumes: + - name: hugepage + emptyDir: + medium: HugePages diff --git a/demo/dsa-dpdk-dmadevtest/Dockerfile b/demo/dsa-dpdk-dmadevtest/Dockerfile new file mode 100644 index 00000000..45ffd86d --- /dev/null +++ b/demo/dsa-dpdk-dmadevtest/Dockerfile @@ -0,0 +1,28 @@ +FROM debian:sid-slim AS builder + +ARG DIR=/dpdk-build +WORKDIR $DIR + +RUN apt-get update && apt-get install -y --no-install-recommends wget build-essential meson ninja-build python3-pyelftools libnuma-dev python3-pip pkg-config dpkg-dev libipsec-mb-dev + +# Download & unpack DPDK tarball +ARG DPDK_TARBALL=dpdk-24.11.1.tar.xz +ARG DPDK_TARBALL_SHA256="bcae7d42c449fc456dfb279feabcbe0599a29bebb2fe2905761e187339d96b8e" + +RUN wget -q https://fast.dpdk.org/rel/$DPDK_TARBALL \ + && echo "$DPDK_TARBALL_SHA256 $DPDK_TARBALL" | sha256sum -c - \ + && tar -xf $DPDK_TARBALL && rm $DPDK_TARBALL + +RUN cd dpdk-* && meson setup \ + "-Dplatform=generic" \ + "-Denable_drivers=dma/idxd" \ + "-Denable_apps=test" \ + --prefix $(pwd)/installdir \ + builddir +RUN cd dpdk-* && ninja -C builddir install && install -D installdir/bin/dpdk-test /install_root/usr/bin/dpdk-test + +FROM debian:sid-slim +RUN apt-get update && apt-get install -y --no-install-recommends libipsec-mb2 libnuma1 libatomic1 && ldconfig -v +COPY --from=builder /install_root/usr/bin/dpdk-test /usr/bin/dpdk-test + +ENTRYPOINT ["dpdk-test", "dmadev_autotest"] diff --git a/demo/dsa.conf b/demo/dsa.conf index f7db4b9a..3ae80a68 100644 --- a/demo/dsa.conf +++ b/demo/dsa.conf @@ -17,7 +17,7 @@ "priority":10, "block_on_fault":1, "type":"user", - "name":"appX0", + "name":"dpdk_X0", "driver_name":"user", "threshold":15 } @@ -43,7 +43,7 @@ "priority":10, "block_on_fault":1, "type":"user", - "name":"appX1", + "name":"dpdk_X1", "driver_name":"user", "threshold":15 } @@ -69,7 +69,7 @@ "priority":10, "block_on_fault":1, "type":"user", - "name":"appX2", + "name":"dpdk_X2", "driver_name":"user", "threshold":15 } @@ -95,7 +95,7 @@ "priority":10, "block_on_fault":1, "type":"user", - "name":"appX3", + "name":"dpdk_X3", "driver_name":"user", "threshold":15 } diff --git a/go.mod b/go.mod index c43a8f15..0e5e19c6 100644 --- a/go.mod +++ b/go.mod @@ -130,7 +130,6 @@ require ( golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index 3992172a..1ba4a625 100644 --- a/go.sum +++ b/go.sum @@ -344,8 +344,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a h1:OAiGFfOiA0v9MRYsSidp3ubZaBnteRUyn3xB2ZQ5G/E= google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= diff --git a/scripts/set-version.sh b/scripts/set-version.sh index e2b24ed9..e9eed9d0 100755 --- a/scripts/set-version.sh +++ b/scripts/set-version.sh @@ -15,8 +15,8 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then exit 1 fi -files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go) +files=$(git grep -l '^TAG?*=\|intel/dsa-dpdk-dmadevtest:\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml demo/dsa-dpdk-dmadev*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go) for file in $files; do - sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file"; + sed -i -e "s;\(^TAG?*=\|intel/dsa-dpdk-dmadevtest:\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file"; done diff --git a/test/e2e/dsa/dsa.go b/test/e2e/dsa/dsa.go index 322783dc..c5eaf23e 100644 --- a/test/e2e/dsa/dsa.go +++ b/test/e2e/dsa/dsa.go @@ -36,7 +36,9 @@ const ( kustomizationYaml = "deployments/dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml" configmapYaml = "demo/dsa.conf" demoYaml = "demo/dsa-accel-config-demo-pod.yaml" + dpdkDemoYaml = "demo/dsa-dpdk-dmadevtest.yaml" podName = "dsa-accel-config-demo" + dpdkPodName = "dpdk" ) func init() { @@ -62,6 +64,11 @@ func describe() { framework.Failf("unable to locate %q: %v", demoYaml, errFailedToLocateRepoFile) } + demoDpdkPath, errFailedToLocateRepoFile := utils.LocateRepoFile(dpdkDemoYaml) + if errFailedToLocateRepoFile != nil { + framework.Failf("unable to locate %q: %v", dpdkDemoYaml, errFailedToLocateRepoFile) + } + var dpPodName string ginkgo.BeforeEach(func(ctx context.Context) { @@ -110,6 +117,14 @@ func describe() { gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, podName, podName)) }) + ginkgo.It("deploys a demo app [App:dpdk-test]", func(ctx context.Context) { + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoDpdkPath) + + ginkgo.By("waiting for the DSA DPDK demo to succeed") + err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, dpdkPodName, f.Namespace.Name, 200*time.Second) + gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, dpdkPodName, dpdkPodName)) + }) + ginkgo.When("there is no app to run [App:noapp]", func() { ginkgo.It("does nothing", func() {}) })