qat_dpdk_app: drop generic

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>
This commit is contained in:
Hyeongju Johannes Lee 2024-04-11 12:13:06 -07:00 committed by Hyeongju Johannes Lee
parent 18d6e2d9e5
commit 2af37fd4cb
29 changed files with 194 additions and 236 deletions

View File

@ -12,7 +12,6 @@ misconfigurations:
paths:
- dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml
- dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml
- qat_dpdk_app/patches/crypto-perf/env_replace_testcmd.yaml
- iaa_plugin/overlays/iaa_initcontainer/iaa_initcontainer.yaml
- qat_plugin/base/intel-qat-kernel-plugin.yaml
- qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml
@ -46,10 +45,7 @@ misconfigurations:
- xpumanager_sidecar/kustom/kustom_xpumanager.yaml
- operator/default/manager_auth_proxy_patch.yaml
- operator/default/manager_webhook_patch.yaml
- qat_dpdk_app/patches/compress-perf/env_replace_testcmd.yaml
- qat_dpdk_app/patches/compress-perf/volume_add_configmap.yaml
- qat_plugin/overlays/debug/add-args.yaml
- qat_plugin/overlays/e2e/add-args.yaml
- qat_plugin/overlays/debug/add-args.yaml
- qat_dpdk_app/patches/crypto-perf/env_replace_testcmd.yaml
- sgx_admissionwebhook/base/manager_webhook_patch.yaml

View File

@ -219,7 +219,6 @@ go test -v ./test/e2e/... -args -ginkgo.focus <labels in regex> -ginkgo.skip <la
| `iaa` |- | `dedicated` | `accel-config` |
| `qat` | `dpdk` | `dc` | `openssl` |
| `qat` | `dpdk` | `cy` | `openssl`, `crypto-perf` |
| `qat` | `dpdk` | `generic` | `crypto-perf`, `compress-perf` |
| `qat` | `kernel` | `cy1_dc0` | `busybox` |
| `sgx` |- | | `sgx-sdk-demo` |
@ -240,9 +239,6 @@ go test -v ./test/e2e/... -args -ginkgo.focus "Device:gpu" -ginkgo.skip "App:bus
# QAT for qat4 cy resource with openssl app running
go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:cy.*App:openssl"
# QAT with dpdk mode for qat2 generic resource with all apps running
go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:generic.*App:(crypto-perf|compress-perf)"
# SGX without running sgx-sdk-demo app
go test -v ./test/e2e/... -args -ginkgo.focus "Device:sgx" -ginkgo.skip "App:sgx-sdk-demo"

View File

@ -234,7 +234,8 @@ The summary of resources available via plugins in this repository is given in th
* `iaa.intel.com` : `wq-user-[shared or dedicated]`
* [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml)
* `qat.intel.com` : `generic` or `cy`/`dc`/`asym-dc`/`sym-dc`
* [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml)
* [compress-perf-dpdk-pod-requesting-qat-dc.yaml](deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml)
* [crypto-perf-dpdk-pod-requesting-qat-cy.yaml](deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml)
* `sgx.intel.com` : `epc`
* [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml)

View File

@ -13,8 +13,6 @@ Table of Contents
* [DPDK QAT Demos](#dpdk-qat-demos)
* [DPDK Prerequisites](#dpdk-prerequisites)
* [Deploy the pod](#deploy-the-pod)
* [Manual test run](#manual-test-run)
* [Automated test run](#automated-test-run)
* [OpenSSL QAT Demo](#openssl-qat-demo)
* [Checking for Hardware](#checking-for-hardware)
@ -180,61 +178,42 @@ website links.
#### DPDK Prerequisites
For the DPDK QAT demos to work, the DPDK drivers must be loaded and configured.
- For the DPDK QAT demos to work, the DPDK drivers must be loaded and configured.
For more information, refer to:
[DPDK Getting Started Guide for Linux](https://doc.dpdk.org/guides/linux_gsg/index.html) and
[DPDK Getting Started Guide, Linux Drivers section](http://dpdk.org/doc/guides/linux_gsg/linux_drivers.html)
#### Deploy the pod
- You may need to add hugepage request and limit.
In the pod specification file, add container resource request and limit.
For example, `qat.intel.com/generic: <number of devices>` for a container requesting QAT devices.
For a DPDK-based workload, you may need to add hugepage request and limit.
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/base/
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
qat-dpdk 1/1 Running 0 27m
intel-qat-plugin-5zgvb 1/1 Running 0 3h
```
- The cluster must enable [Kubernetes CPU manager's](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) `static` policy.
> **Note**: If the `igb_uio` VF driver is used with the QAT device plugin,
> the workload be deployed with `SYS_ADMIN` capabilities added.
#### Manual Test Run
Manually execute the `dpdk-test-crypto-perf` application to review the logs:
#### Deploy the pod
```bash
$ kubectl exec -it qat-dpdk bash
$ dpdk-test-crypto-perf -l 6-7 -w $QAT1 \
-d /usr/lib64/librte_mempool_ring.so.1.1 \
-d /usr/lib64/librte_pmd_qat.so.1.1 -- \
--ptest throughput --devtype crypto_qat \
--optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt \
--cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
qat-dpdk-test-crypto-perf 1/1 Running 0 27m
qat-dpdk-test-compress-perf 1/1 Running 0 27m
intel-qat-plugin-5zgvb 1/1 Running 0 3h
```
> **Note**: Adapt the `.so` versions to what the DPDK version in the container provides.
Both pods for [crypto-perf](../..//deployments/qat_dpdk_app/crypto-perf) and [compress-perf](../../deployments/qat_dpdk_app/compress-perf) are generated by default using `kustomize`.
#### Automated Test Run
It is also possible to deploy and run `crypto-perf` using the following
`kustomize` overlays:
It is also possible to deploy and run `crypto-perf` or `compress-perf` only.
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/test-crypto1
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/test-compress1
$ kubectl logs qat-dpdk-test-crypto-perf-tc1
$ kubectl logs qat-dpdk-test-compress-perf-tc1
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/crypto-perf
$ kubectl logs qat-dpdk-test-crypto-perf
```
> **Note**: for `test-crypto1` and `test-compress1` to work, the cluster must enable
[Kubernetes CPU manager's](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) `static` policy.
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/compress-perf
$ kubectl logs qat-dpdk-test-compress-perf
```
### OpenSSL QAT Demo

View File

@ -79,11 +79,11 @@
screen5()
{
clear
out "7. View pod specification file for pod requesting QAT Virtual Functions:"
command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/base"
out "7. View pod specification file for pod requesting QAT Virtual Functions (crypto):"
command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/crypto-perf"
sleep 5
out "8. Create Pod requesting QAT Virtual Functions:"
command "kubectl apply -k ../deployments/qat_dpdk_app/base"
out "8. Create Pod requesting QAT Virtual Functions (crypto):"
command "kubectl apply -k ../deployments/qat_dpdk_app/crypto-perf"
sleep 2
command "kubectl get pods"
sleep 2
@ -91,10 +91,13 @@
screen6()
{
clear
out "9. Get a shell to the running container and run a DPDK application using QAT device"
out "* \"export\": Lists environment variables - note QAT0, QAT1, QAT2... etc"
out "* \"./dpdk-test-crypto-perf -l 6-7 -w \$QAT0 -- --ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64\" : Manually executes the dpdk-test-crypto-perf application to review the logs"
command "kubectl exec -it qat-dpdk bash"
out "9. View pod specification file for pod requesting QAT Virtual Functions (compress):"
command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/compress-perf"
sleep 5
out "10. Create Pod requesting QAT Virtual Functions (compress):"
command "kubectl apply -k ../deployments/qat_dpdk_app/compress-perf"
sleep 2
command "kubectl get pods"
sleep 10
}
if [ "$1" == 'play' ] ; then

View File

@ -1,2 +0,0 @@
resources:
- crypto-perf-dpdk-pod-requesting-qat.yaml

View File

@ -0,0 +1,49 @@
kind: Pod
apiVersion: v1
metadata:
name: qat-dpdk-test-compress-perf
spec:
containers:
- name: compress-perf
image: intel/crypto-perf:devel
imagePullPolicy: IfNotPresent
env:
- name: TESTCMD
value: "compress"
- name: PTEST
value: "--driver-name compress_qat --input-file /var/data/file.txt --seg-sz 8192 --compress-level 1:1:9 --num-iter 10 --extended-input-sz 1048576 --max-num-sgl-segs 16 --huffman-enc fixed"
volumeMounts:
- mountPath: /dev/hugepages
name: hugepage
- mountPath: /var/run/dpdk
name: dpdk-runtime
- mountPath: /var/data/
name: testfile
resources:
requests:
cpu: "3"
memory: "128Mi"
qat.intel.com/dc: '4'
hugepages-2Mi: "128Mi"
limits:
cpu: "3"
memory: "128Mi"
qat.intel.com/dc: '4'
hugepages-2Mi: "128Mi"
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
add:
["IPC_LOCK"]
restartPolicy: Never
volumes:
- name: dpdk-runtime
emptyDir:
medium: Memory
- name: hugepage
emptyDir:
medium: HugePages
- name: testfile
configMap:
name: test-data

View File

@ -0,0 +1,49 @@
kind: Pod
apiVersion: v1
metadata:
name: qat-dpdk-test-compress-perf
spec:
containers:
- name: compress-perf
image: intel/crypto-perf:devel
imagePullPolicy: IfNotPresent
env:
- name: TESTCMD
value: "compress"
- name: PTEST
value: "--driver-name compress_qat --input-file /var/data/file.txt --seg-sz 8192 --compress-level 1:1:9 --num-iter 10 --extended-input-sz 1048576 --max-num-sgl-segs 16 --huffman-enc fixed"
volumeMounts:
- mountPath: /dev/hugepages
name: hugepage
- mountPath: /var/run/dpdk
name: dpdk-runtime
- mountPath: /var/data/
name: testfile
resources:
requests:
cpu: "3"
memory: "128Mi"
qat.intel.com/dc: '4'
hugepages-2Mi: "128Mi"
limits:
cpu: "3"
memory: "128Mi"
qat.intel.com/dc: '4'
hugepages-2Mi: "128Mi"
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
add:
["IPC_LOCK"]
restartPolicy: Never
volumes:
- name: dpdk-runtime
emptyDir:
medium: Memory
- name: hugepage
emptyDir:
medium: HugePages
- name: testfile
configMap:
name: test-data

View File

@ -0,0 +1,7 @@
configMapGenerator:
- files:
- file.txt
name: test-data
resources:
- compress-perf-dpdk-pod-requesting-qat-dc.yaml

View File

@ -0,0 +1,44 @@
kind: Pod
apiVersion: v1
metadata:
name: qat-dpdk-test-crypto-perf
spec:
containers:
- name: crypto-perf
image: intel/crypto-perf:devel
imagePullPolicy: IfNotPresent
env:
- name: TESTCMD
value: "crypto"
- name: PTEST
value: "--ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64"
volumeMounts:
- mountPath: /dev/hugepages
name: hugepage
- mountPath: /var/run/dpdk
name: dpdk-runtime
resources:
requests:
cpu: "3"
memory: "128Mi"
qat.intel.com/cy: '4'
hugepages-2Mi: "128Mi"
limits:
cpu: "3"
memory: "128Mi"
qat.intel.com/cy: '4'
hugepages-2Mi: "128Mi"
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
add:
["IPC_LOCK"]
restartPolicy: Never
volumes:
- name: dpdk-runtime
emptyDir:
medium: Memory
- name: hugepage
emptyDir:
medium: HugePages

View File

@ -1,14 +1,17 @@
kind: Pod
apiVersion: v1
metadata:
name: qat-dpdk
name: qat-dpdk-test-crypto-perf
spec:
containers:
- name: crypto-perf
image: intel/crypto-perf:devel
imagePullPolicy: IfNotPresent
command: [ "/bin/bash", "-c", "--" ]
args: [ "while true; do sleep 300000; done;" ]
env:
- name: TESTCMD
value: "crypto"
- name: PTEST
value: "--ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64"
volumeMounts:
- mountPath: /dev/hugepages
name: hugepage

View File

@ -0,0 +1,2 @@
resources:
- crypto-perf-dpdk-pod-requesting-qat-cy.yaml

View File

@ -1,2 +1,3 @@
resources:
- base
- compress-perf
- crypto-perf

View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: qat-dpdk
spec:
containers:
- name: crypto-perf
env:
- name: TESTCMD
value: "compress"

View File

@ -1,12 +0,0 @@
nameSuffix: -compress-perf
resources:
- ../dpdk-test/
configMapGenerator:
- files:
- file.txt
name: test-data
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: volume_add_configmap.yaml
- path: env_replace_testcmd.yaml

View File

@ -1,14 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: qat-dpdk
spec:
containers:
- name: crypto-perf
volumeMounts:
- name: testfile
mountPath: /var/data/
volumes:
- name: testfile
configMap:
name: test-data

View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: qat-dpdk-test
spec:
containers:
- name: crypto-perf
env:
- name: TESTCMD
value: "crypto"

View File

@ -1,11 +0,0 @@
nameSuffix: -crypto-perf
resources:
- ../dpdk-test/
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: test.json
target:
kind: Pod
name: qat-dpdk
version: v1

View File

@ -1,3 +0,0 @@
[
{"op": "replace", "path": "/spec/containers/0/env/0/value", "value": "crypto"}
]

View File

@ -1,5 +0,0 @@
[
{"op": "remove", "path": "/spec/containers/0/args"},
{"op": "replace", "path": "/spec/containers/0/command", "value": ["/usr/bin/run-dpdk-test"]},
{"op": "add", "path": "/spec/containers/0/env", "value": [{"name": "TESTCMD", "value": "dummy"}, {"name": "PTEST", "value": "dummy"}]}
]

View File

@ -1,11 +0,0 @@
nameSuffix: -test
resources:
- ../../base/
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: containers_patch_command_args.json
target:
kind: Pod
name: qat-dpdk
version: v1

View File

@ -1,11 +0,0 @@
nameSuffix: -tc1
resources:
- ../patches/compress-perf
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: tc1.json
target:
kind: Pod
name: qat-dpdk
version: v1

View File

@ -1,3 +0,0 @@
- op: replace
path: /spec/containers/0/env/1/value
value: "--driver-name compress_qat --input-file /var/data/file.txt --seg-sz 8192 --compress-level 1:1:9 --num-iter 10 --extended-input-sz 1048576 --max-num-sgl-segs 16 --huffman-enc fixed"

View File

@ -1,9 +0,0 @@
nameSuffix: -gen4
resources:
- ../test-crypto1
patches:
- target:
version: v1
kind: Pod
name: qat-dpdk
path: tc1-gen4.yaml

View File

@ -1,13 +0,0 @@
- op: replace
path: /spec/containers/0/resources
value:
requests:
cpu: "3"
memory: "128Mi"
qat.intel.com/cy: '4'
hugepages-2Mi: "128Mi"
limits:
cpu: "3"
memory: "128Mi"
qat.intel.com/cy: '4'
hugepages-2Mi: "128Mi"

View File

@ -1,11 +0,0 @@
nameSuffix: -tc1
resources:
- ../patches/crypto-perf
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: tc1.json
target:
kind: Pod
name: qat-dpdk
version: v1

View File

@ -1,3 +0,0 @@
- op: replace
path: /spec/containers/0/env/1/value
value: "--ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64"

View File

@ -36,10 +36,7 @@ import (
const (
qatPluginKustomizationYaml = "deployments/qat_plugin/overlays/e2e/kustomization.yaml"
compressTestYaml = "deployments/qat_dpdk_app/test-compress1/kustomization.yaml"
cryptoTestYaml = "deployments/qat_dpdk_app/test-crypto1/kustomization.yaml"
cryptoTestGen4Yaml = "deployments/qat_dpdk_app/test-crypto1-gen4/kustomization.yaml"
demoPodContainerName = "crypto-perf"
cryptoTestYaml = "deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml"
)
const (
@ -66,21 +63,11 @@ func describeQatDpdkPlugin() {
framework.Failf("unable to locate %q: %v", qatPluginKustomizationYaml, errFailedToLocateRepoFile)
}
compressTestYamlPath, errFailedToLocateRepoFile := utils.LocateRepoFile(compressTestYaml)
if errFailedToLocateRepoFile != nil {
framework.Failf("unable to locate %q: %v", compressTestYaml, errFailedToLocateRepoFile)
}
cryptoTestYamlPath, errFailedToLocateRepoFile := utils.LocateRepoFile(cryptoTestYaml)
if errFailedToLocateRepoFile != nil {
framework.Failf("unable to locate %q: %v", cryptoTestYaml, errFailedToLocateRepoFile)
}
cryptoTestGen4YamlPath, errFailedToLocateRepoFile := utils.LocateRepoFile(cryptoTestGen4Yaml)
if errFailedToLocateRepoFile != nil {
framework.Failf("unable to locate %q: %v", cryptoTestGen4Yaml, errFailedToLocateRepoFile)
}
var dpPodName string
var resourceName v1.ResourceName
@ -134,11 +121,11 @@ func describeQatDpdkPlugin() {
ginkgo.It("deploys a crypto pod (dpdk crypto-perf) requesting QAT resources [App:crypto-perf]", func(ctx context.Context) {
ginkgo.By("submitting a crypto pod requesting QAT resources")
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(cryptoTestGen4YamlPath))
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(cryptoTestYamlPath))
ginkgo.By("waiting the crypto pod to finish successfully")
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, "qat-dpdk-test-crypto-perf-tc1-gen4", f.Namespace.Name, 300*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, "qat-dpdk-test-crypto-perf-tc1-gen4", "crypto-perf"))
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, "qat-dpdk-test-crypto-perf", f.Namespace.Name, 300*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, "qat-dpdk-test-crypto-perf", "crypto-perf"))
})
ginkgo.When("there is no app to run [App:noapp]", func() {
@ -163,37 +150,6 @@ func describeQatDpdkPlugin() {
ginkgo.It("does nothing", func() {})
})
})
ginkgo.Context("When QAT resources are available [Resource:generic]", func() {
ginkgo.BeforeEach(func() {
ginkgo.By("setting resourceName for generic resources")
resourceName = "qat.intel.com/generic"
})
ginkgo.It("deploys a crypto pod requesting QAT resources [App:crypto-perf]", func(ctx context.Context) {
ginkgo.By("submitting a crypto pod requesting QAT resources")
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(cryptoTestYamlPath))
ginkgo.By("waiting the crypto pod to finish successfully")
demoPodName := "qat-dpdk-test-crypto-perf-tc1"
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, demoPodName, f.Namespace.Name, 60*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, demoPodName, demoPodContainerName))
})
ginkgo.It("deploys a compress pod requesting QAT resources [App:compress-perf]", func(ctx context.Context) {
ginkgo.By("submitting a compress pod requesting QAT resources")
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(compressTestYamlPath))
ginkgo.By("waiting the compress pod to finish successfully")
demoPodName := "qat-dpdk-test-compress-perf-tc1"
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, demoPodName, f.Namespace.Name, 60*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, demoPodName, demoPodContainerName))
})
ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.It("does nothing", func() {})
})
})
}
func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int, resourceName v1.ResourceName) {