qat: Add e2e tests

Closes #950

Signed-off-by: Oleg Zhurakivskyy <oleg.zhurakivskyy@intel.com>
Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

.github/workflows/e2e-qat4.yml

@@ -0,0 +1,41 @@
+name: e2e-qat4
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '20 3 * * *'
+  pull_request:
+    branches:
+      - main
+      - 'release-*'
+  push:
+    branches:
+      - main
+      - 'release-*'
+
+env:
+  IMAGES: 'intel-qat-plugin intel-qat-initcontainer openssl-qat-engine'
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-qat4:
+    name: e2e-qat4
+    runs-on: [self-hosted, linux, x64, dsa]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Describe test environment
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          echo "Actor: ${{ github.actor }}"
+          echo "Ref: ${{ github.ref }}"
+          echo "SHA: ${{ github.sha }}"
+          echo "Images: $IMAGES"
+      - name: Wait for ready state
+        run: ../../../../bmetal/actions-bmetal-runstage.sh waitready
+      - name: Prepare test environment
+        run: ../../../../bmetal/actions-bmetal-runstage.sh prepare
+      - name: Run tests
+        run: ../../../../bmetal/actions-bmetal-runstage.sh test

Makefile

@@ -147,6 +147,9 @@ e2e-fpga:
 e2e-qat:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "QAT plugin in DPDK mode" -delete-namespace-on-failure=false
 
+e2e-qat4:
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "QAT Gen4" -delete-namespace-on-failure=false
+
 e2e-sgx:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.progress -ginkgo.focus "SGX" -delete-namespace-on-failure=false
 

demo/openssl-qat-engine-cpa-sample-pod.yaml

@@ -0,0 +1,20 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: openssl-qat-engine
+  labels:
+    app: openssl-qat-engine
+spec:
+  containers:
+  - name: openssl-qat-engine
+    image: intel/openssl-qat-engine:devel
+    imagePullPolicy: IfNotPresent
+    command: [ "cpa_sample_code", "runTests=4", "signOfLife=1" ]
+    securityContext:
+      capabilities:
+        add:
+          ["IPC_LOCK"]
+    resources:
+      limits:
+        qat.intel.com/cy: 1
+  restartPolicy: Never

scripts/set-version.sh

@@ -15,6 +15,6 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
     exit 1
 fi
 
-for file in $(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
-    sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
+for file in $(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile charts test/e2e/*/*.go); do
+    sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
 done

test/e2e/qat/qatplugin_qatgen4_crypto.go

@@ -0,0 +1,86 @@
+// Copyright 2022 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qat
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
+	"github.com/onsi/ginkgo/v2"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/kubectl"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	admissionapi "k8s.io/pod-security-admission/api"
+)
+
+const (
+	qatPluginKustomizationYaml = "deployments/qat_plugin/overlays/e2e/kustomization.yaml"
+	opensslTestYaml            = "demo/openssl-qat-engine-cpa-sample-pod.yaml"
+)
+
+func init() {
+	ginkgo.Describe("QAT Gen4 plugin in Crypto mode", describeQatPluginCy)
+}
+
+func describeQatPluginCy() {
+	f := framework.NewDefaultFramework("qatplugincy")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
+
+	kustomizationPath, err := utils.LocateRepoFile(qatPluginKustomizationYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", qatPluginKustomizationYaml, err)
+	}
+
+	opensslTestYamlPath, err := utils.LocateRepoFile(opensslTestYaml)
+	if err != nil {
+		framework.Failf("unable to locate %q: %v", opensslTestYaml, err)
+	}
+
+	ginkgo.It("measures performance of QAT Cy Services", func() {
+		ginkgo.By("deploying QAT plugin in DPDK mode")
+		framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
+
+		ginkgo.By("waiting for QAT plugin's availability")
+		podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
+			labels.Set{"app": "intel-qat-plugin"}.AsSelector(), 1 /* one replica */, 100*time.Second)
+		if err != nil {
+			framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
+			kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
+			framework.Failf("unable to wait for all pods to be running and ready: %v", err)
+		}
+
+		ginkgo.By("checking QAT plugin's securityContext")
+		if err := utils.TestPodsFileSystemInfo(podList.Items); err != nil {
+			framework.Failf("container filesystem info checks failed: %v", err)
+		}
+
+		ginkgo.By("checking the resource is allocatable")
+		if err := utils.WaitForNodesWithResource(f.ClientSet, "qat.intel.com/cy", 30*time.Second); err != nil {
+			framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+		}
+
+		ginkgo.By("submitting a crypto pod requesting QAT resources")
+		framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", opensslTestYamlPath)
+
+		ginkgo.By("waiting the crypto pod to finish successfully")
+		f.PodClient().WaitForSuccess("openssl-qat-engine", 300*time.Second)
+
+		output, _ := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, "openssl-qat-engine", "openssl-qat-engine")
+
+		framework.Logf("cpa_sample_code output:\n %s", output)
+	})
+}