Merge pull request #1685 from hj-johannes-lee/PR-2024-001

qat: improve qat_dpdk_app, openssl-qat-engine

Author: tkatila

.github/workflows/lib-e2e.yaml

@@ -28,19 +28,12 @@ jobs:
               - accel-config-demo
               - intel-deviceplugin-operator
           - name: e2e-qat-gnrd
-            targetjob: e2e-qat FOCUS="Mode:dpdk.*Resource:(cy|dc)" SKIP=App:crypto-perf
+            targetjob: e2e-qat FOCUS="Mode:dpdk.*Resource:(cy|dc)" SKIP="App:(crypto-perf|compress-perf)"
             runner: simics-gnrd
             images:
               - intel-qat-plugin
               - intel-qat-initcontainer
               - openssl-qat-engine
-          - name: e2e-qat
-            targetjob: e2e-qat FOCUS=Resource:generic
-            runner: qat
-            images:
-              - intel-qat-plugin
-              - intel-qat-initcontainer
-              - crypto-perf
           - name: e2e-fpga
             runner: fpga
             images:
@@ -49,6 +42,7 @@ jobs:
               - intel-fpga-admissionwebhook
               - opae-nlb-demo
           - name: e2e-spr
+            targetjob: e2e-spr SKIP="App:compress-perf"
             runner: spr
             images:
               - intel-qat-plugin

.trivyignore.yaml

@@ -12,7 +12,6 @@ misconfigurations:
     paths:
       - dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml
       - dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml
-      - qat_dpdk_app/patches/crypto-perf/env_replace_testcmd.yaml
       - iaa_plugin/overlays/iaa_initcontainer/iaa_initcontainer.yaml
       - qat_plugin/base/intel-qat-kernel-plugin.yaml
       - qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml
@@ -46,10 +45,7 @@ misconfigurations:
       - xpumanager_sidecar/kustom/kustom_xpumanager.yaml
       - operator/default/manager_auth_proxy_patch.yaml
       - operator/default/manager_webhook_patch.yaml
-      - qat_dpdk_app/patches/compress-perf/env_replace_testcmd.yaml
-      - qat_dpdk_app/patches/compress-perf/volume_add_configmap.yaml
       - qat_plugin/overlays/debug/add-args.yaml
       - qat_plugin/overlays/e2e/add-args.yaml
       - qat_plugin/overlays/debug/add-args.yaml
-      - qat_dpdk_app/patches/crypto-perf/env_replace_testcmd.yaml
       - sgx_admissionwebhook/base/manager_webhook_patch.yaml

DEVEL.md

@@ -217,9 +217,8 @@ go test -v ./test/e2e/... -args -ginkgo.focus <labels in regex> -ginkgo.skip <la
 | `fpga` | `af`, `region`   |             | `opae-nlb-demo`                |
 | `gpu`  |-                 | `i915`      | `busybox`, `tensorflow`        |
 | `iaa`  |-                 | `dedicated` | `accel-config`                 |
-| `qat`  | `dpdk`           | `dc`        | `openssl`                      |
+| `qat`  | `dpdk`           | `dc`        | `openssl`, `compress-perf`     |
 | `qat`  | `dpdk`           | `cy`        | `openssl`, `crypto-perf`       |
-| `qat`  | `dpdk`           | `generic`   | `crypto-perf`, `compress-perf` |
 | `qat`  | `kernel`         | `cy1_dc0`   | `busybox`                      |
 | `sgx`  |-                 |             | `sgx-sdk-demo`                 |
 
@@ -240,9 +239,6 @@ go test -v ./test/e2e/... -args -ginkgo.focus "Device:gpu" -ginkgo.skip "App:bus
 # QAT for qat4 cy resource with openssl app running
 go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:cy.*App:openssl"
 
-# QAT with dpdk mode for qat2 generic resource with all apps running
-go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:generic.*App:(crypto-perf|compress-perf)"
-
 # SGX without running sgx-sdk-demo app
 go test -v ./test/e2e/... -args -ginkgo.focus "Device:sgx" -ginkgo.skip "App:sgx-sdk-demo"
 

README.md

@@ -234,7 +234,8 @@ The summary of resources available via plugins in this repository is given in th
  * `iaa.intel.com`  : `wq-user-[shared or dedicated]`
    * [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml)
  * `qat.intel.com`  : `generic` or `cy`/`dc`/`asym-dc`/`sym-dc`
-   * [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml)
+   * [compress-perf-dpdk-pod-requesting-qat-dc.yaml](deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml)
+   * [crypto-perf-dpdk-pod-requesting-qat-cy.yaml](deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml)
  * `sgx.intel.com`  : `epc`
    * [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml)
 

cmd/qat_plugin/README.md

@@ -13,8 +13,6 @@ Table of Contents
     * [DPDK QAT Demos](#dpdk-qat-demos)
         * [DPDK Prerequisites](#dpdk-prerequisites)
         * [Deploy the pod](#deploy-the-pod)
-        * [Manual test run](#manual-test-run)
-        * [Automated test run](#automated-test-run)
     * [OpenSSL QAT Demo](#openssl-qat-demo)
 * [Checking for Hardware](#checking-for-hardware)
 
@@ -180,61 +178,42 @@ website links.
 
 #### DPDK Prerequisites
 
-For the DPDK QAT demos to work, the DPDK drivers must be loaded and configured.
+- For the DPDK QAT demos to work, the DPDK drivers must be loaded and configured.
 For more information, refer to:
 [DPDK Getting Started Guide for Linux](https://doc.dpdk.org/guides/linux_gsg/index.html) and
 [DPDK Getting Started Guide, Linux Drivers section](http://dpdk.org/doc/guides/linux_gsg/linux_drivers.html)
 
-#### Deploy the pod
-
-In the pod specification file, add container resource request and limit.
-For example, `qat.intel.com/generic: <number of devices>` for a container requesting QAT devices.
+- You may need to add hugepage request and limit.
 
-For a DPDK-based workload, you may need to add hugepage request and limit.
-
-```bash
-$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/base/
-$ kubectl get pods
-  NAME                     READY     STATUS    RESTARTS   AGE
-  qat-dpdk                 1/1       Running   0          27m
-  intel-qat-plugin-5zgvb   1/1       Running   0          3h
-
-```
+- The cluster must enable [Kubernetes CPU manager's](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) `static` policy.
 
 > **Note**: If the `igb_uio` VF driver is used with the QAT device plugin,
 > the workload be deployed with `SYS_ADMIN` capabilities added.
 
-#### Manual Test Run
-
-Manually execute the `dpdk-test-crypto-perf` application to review the logs:
+#### Deploy the pod
 
 ```bash
-$ kubectl exec -it qat-dpdk bash
-
-$ dpdk-test-crypto-perf -l 6-7 -w $QAT1 \
--d /usr/lib64/librte_mempool_ring.so.1.1 \
--d /usr/lib64/librte_pmd_qat.so.1.1 -- \
---ptest throughput --devtype crypto_qat \
---optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt \
---cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/
+$ kubectl get pods
+  NAME                          READY     STATUS    RESTARTS   AGE
+  qat-dpdk-test-crypto-perf     1/1       Running   0          27m
+  qat-dpdk-test-compress-perf   1/1       Running   0          27m
+  intel-qat-plugin-5zgvb        1/1       Running   0          3h
 ```
 
-> **Note**: Adapt the `.so` versions to what the DPDK version in the container provides.
+Both pods for [crypto-perf](../..//deployments/qat_dpdk_app/crypto-perf) and [compress-perf](../../deployments/qat_dpdk_app/compress-perf) are generated by default using `kustomize`.
 
-#### Automated Test Run
-
-It is also possible to deploy and run `crypto-perf` using the following
-`kustomize` overlays:
+It is also possible to deploy and run `crypto-perf` or `compress-perf` only.
 
 ```bash
-$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/test-crypto1
-$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/test-compress1
-$ kubectl logs qat-dpdk-test-crypto-perf-tc1
-$ kubectl logs qat-dpdk-test-compress-perf-tc1
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/crypto-perf
+$ kubectl logs qat-dpdk-test-crypto-perf
 ```
 
-> **Note**: for `test-crypto1` and `test-compress1` to work, the cluster must enable
-[Kubernetes CPU manager's](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) `static` policy.
+```bash
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/qat_dpdk_app/compress-perf
+$ kubectl logs qat-dpdk-test-compress-perf
+```
 
 ### OpenSSL QAT Demo
 

demo/openssl-qat-engine/Dockerfile

@@ -1,4 +1,4 @@
 FROM ubuntu:24.04
 
 RUN apt update && \
-    apt install --no-install-recommends -y qatengine qatlib-examples qatzip openssl
+    apt install --no-install-recommends -y qatengine qatlib-examples qatzip openssl

demo/screencast-qat-dpdk.sh

@@ -79,22 +79,25 @@
  screen5()
 {
   clear
-  out "7. View pod specification file for pod requesting QAT Virtual Functions:"
-  command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/base"
+  out "7. View pod specification file for pod requesting QAT Virtual Functions (crypto):"
+  command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/crypto-perf"
   sleep 5 
-  out "8. Create Pod requesting QAT Virtual Functions:"
-  command "kubectl apply -k ../deployments/qat_dpdk_app/base"
+  out "8. Create Pod requesting QAT Virtual Functions (crypto):"
+  command "kubectl apply -k ../deployments/qat_dpdk_app/crypto-perf"
   sleep 2
   command "kubectl get pods"
   sleep 2
 }
  screen6()
 {
   clear
-  out "9. Get a shell to the running container and run a DPDK application using QAT device"
-  out "* \"export\": Lists environment variables - note QAT0, QAT1, QAT2... etc"
-  out "* \"./dpdk-test-crypto-perf -l 6-7 -w \$QAT0 -- --ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64\" : Manually executes the dpdk-test-crypto-perf application to review the logs"
-   command "kubectl exec -it qat-dpdk bash"
+  out "9. View pod specification file for pod requesting QAT Virtual Functions (compress):"
+  command "kubectl apply --dry-run -o yaml -k ../deployments/qat_dpdk_app/compress-perf"
+  sleep 5 
+  out "10. Create Pod requesting QAT Virtual Functions (compress):"
+  command "kubectl apply -k ../deployments/qat_dpdk_app/compress-perf"
+  sleep 2
+  command "kubectl get pods"
   sleep 10
 }
  if [ "$1" == 'play' ] ; then

deployments/qat_dpdk_app/base/kustomization.yaml

@@ -0,0 +1,49 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: qat-dpdk-test-compress-perf
+spec:
+  containers:
+  - name: compress-perf
+    image: intel/crypto-perf:devel
+    imagePullPolicy: IfNotPresent
+    env:
+    - name: TESTCMD
+      value: "compress"
+    - name: PTEST
+      value: "--driver-name compress_qat --input-file /var/data/file.txt --seg-sz 8192 --compress-level 1:1:9 --num-iter 10 --extended-input-sz 1048576 --max-num-sgl-segs 16 --huffman-enc fixed"
+    volumeMounts:
+    - mountPath: /dev/hugepages
+      name: hugepage
+    - mountPath: /var/run/dpdk
+      name: dpdk-runtime
+    - mountPath: /var/data/
+      name: testfile
+    resources:
+      requests:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/dc: '4'
+        hugepages-2Mi: "128Mi"
+      limits:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/dc: '4'
+        hugepages-2Mi: "128Mi"
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+      capabilities:
+        add:
+          ["IPC_LOCK"]
+  restartPolicy: Never
+  volumes:
+  - name: dpdk-runtime
+    emptyDir:
+      medium: Memory
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
+  - name: testfile
+    configMap:
+      name: test-data

deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml

@@ -0,0 +1,49 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: qat-dpdk-test-compress-perf
+spec:
+  containers:
+  - name: compress-perf
+    image: intel/crypto-perf:devel
+    imagePullPolicy: IfNotPresent
+    env:
+    - name: TESTCMD
+      value: "compress"
+    - name: PTEST
+      value: "--driver-name compress_qat --input-file /var/data/file.txt --seg-sz 8192 --compress-level 1:1:9 --num-iter 10 --extended-input-sz 1048576 --max-num-sgl-segs 16 --huffman-enc fixed"
+    volumeMounts:
+    - mountPath: /dev/hugepages
+      name: hugepage
+    - mountPath: /var/run/dpdk
+      name: dpdk-runtime
+    - mountPath: /var/data/
+      name: testfile
+    resources:
+      requests:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/dc: '4'
+        hugepages-2Mi: "128Mi"
+      limits:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/dc: '4'
+        hugepages-2Mi: "128Mi"
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+      capabilities:
+        add:
+          ["IPC_LOCK"]
+  restartPolicy: Never
+  volumes:
+  - name: dpdk-runtime
+    emptyDir:
+      medium: Memory
+  - name: hugepage
+    emptyDir:
+      medium: HugePages
+  - name: testfile
+    configMap:
+      name: test-data

deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-generic.yaml

@@ -0,0 +1,7 @@
+configMapGenerator:
+- files:
+  - file.txt
+  name: test-data
+
+resources:
+- compress-perf-dpdk-pod-requesting-qat-dc.yaml

deployments/qat_dpdk_app/patches/compress-perf/file.txt renamed to deployments/qat_dpdk_app/compress-perf/file.txt

@@ -0,0 +1,44 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: qat-dpdk-test-crypto-perf
+spec:
+  containers:
+  - name: crypto-perf
+    image: intel/crypto-perf:devel
+    imagePullPolicy: IfNotPresent
+    env:
+    - name: TESTCMD
+      value: "crypto"
+    - name: PTEST
+      value: "--ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64"
+    volumeMounts:
+    - mountPath: /dev/hugepages
+      name: hugepage
+    - mountPath: /var/run/dpdk
+      name: dpdk-runtime
+    resources:
+      requests:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/cy: '4'
+        hugepages-2Mi: "128Mi"
+      limits:
+        cpu: "3"
+        memory: "128Mi"
+        qat.intel.com/cy: '4'
+        hugepages-2Mi: "128Mi"
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+      capabilities:
+        add:
+          ["IPC_LOCK"]
+  restartPolicy: Never
+  volumes:
+  - name: dpdk-runtime
+    emptyDir:
+      medium: Memory
+  - name: hugepage
+    emptyDir:
+      medium: HugePages

deployments/qat_dpdk_app/compress-perf/kustomization.yaml

@@ -1,14 +1,17 @@
 kind: Pod
 apiVersion: v1
 metadata:
-  name: qat-dpdk
+  name: qat-dpdk-test-crypto-perf
 spec:
   containers:
   - name: crypto-perf
     image: intel/crypto-perf:devel
     imagePullPolicy: IfNotPresent
-    command: [ "/bin/bash", "-c", "--" ]
-    args: [ "while true; do sleep 300000; done;" ]
+    env:
+    - name: TESTCMD
+      value: "crypto"
+    - name: PTEST
+      value: "--ptest throughput --devtype crypto_qat --optype cipher-only --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --total-ops 10000000 --burst-sz 32 --buffer-sz 64"
     volumeMounts:
     - mountPath: /dev/hugepages
       name: hugepage

deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml

@@ -0,0 +1,2 @@
+resources:
+- crypto-perf-dpdk-pod-requesting-qat-cy.yaml

deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml renamed to deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-generic.yaml

@@ -1,2 +1,3 @@
 resources:
-  - base
+  - compress-perf
+  - crypto-perf