From 3a4c0e574f90ab4c913595dffdf4657909410540 Mon Sep 17 00:00:00 2001 From: Mikko Ylinen Date: Tue, 25 Apr 2023 11:06:35 +0300 Subject: [PATCH] sgx: stop using local source hooks for EPC registration Signed-off-by: Mikko Ylinen --- cmd/operator/README.md | 8 +++---- cmd/sgx_plugin/README.md | 12 +++++----- deployments/nfd/base/kustomization.yaml | 2 +- .../nfd/components/sgx/kustomization.yaml | 16 -------------- .../nfd/components/sgx/master-args.yaml | 3 --- .../nfd/components/sgx/master-rbac.yaml | 3 --- .../node-feature-rules-openshift.yaml | 6 +++-- .../node-feature-rules.yaml | 2 ++ .../nfd/overlays/sgx/kustomization.yaml | 7 ------ .../deviceplugin_v1_sgxdeviceplugin.yaml | 1 - .../epc-nfd/add-epc-nfd-initcontainer.yaml | 22 ------------------- .../overlays/epc-nfd/kustomization.yaml | 2 -- .../v1/sgxdeviceplugin_webhook.go | 8 +++---- test/e2e/deviceplugins_suite_test.go | 2 +- test/e2e/sgx/sgx.go | 2 +- 15 files changed, 22 insertions(+), 74 deletions(-) delete mode 100644 deployments/nfd/components/sgx/kustomization.yaml delete mode 100644 deployments/nfd/components/sgx/master-args.yaml delete mode 100644 deployments/nfd/components/sgx/master-rbac.yaml delete mode 100644 deployments/nfd/overlays/sgx/kustomization.yaml delete mode 100644 deployments/sgx_plugin/overlays/epc-nfd/add-epc-nfd-initcontainer.yaml diff --git a/cmd/operator/README.md b/cmd/operator/README.md index 02c4a9eb..22da7511 100644 --- a/cmd/operator/README.md +++ b/cmd/operator/README.md @@ -23,14 +23,12 @@ The default operator deployment depends on NFD and cert-manager. Those component ### NFD -Install NFD (if it's not already installed) and node labelling rules (requires NFD v0.10+): +Install NFD (if it's not already installed) and node labelling rules (requires NFD v0.13+): ``` -# either with default NFD installation +# deploy NFD $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=' -# or when setting up with SGX -$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=' -# and finally, NodeFeatureRules +# deploy NodeFeatureRules $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=' ``` Make sure both NFD master and worker pods are running: diff --git a/cmd/sgx_plugin/README.md b/cmd/sgx_plugin/README.md index 025b597d..0bdd93e8 100644 --- a/cmd/sgx_plugin/README.md +++ b/cmd/sgx_plugin/README.md @@ -84,7 +84,7 @@ Where `` needs to be substituted with the desired [release tag] First, deploy `node-feature-discovery`: ```bash -$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=' +$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/?ref=' $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=' ``` @@ -110,13 +110,13 @@ $ kubectl apply -f 'https://raw.githubusercontent.com/intel/intel-device-plugins There are two alternative ways to deploy SGX device plugin using `kubectl`. The first approach involves deployment of the [SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml) -and [node-feature-discovery](/deployments/nfd/overlays/sgx/kustomization.yaml) +and [node-feature-discovery](/deployments/nfd/kustomization.yaml) with the necessary configuration. The following kustomizations are needed for deploying everything: ```bash # first, deploy NFD and the necessary NodeFeatureRules -$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx' +$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd' $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules' # and then, deploy SGX plugin $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_plugin/overlays/epc-nfd/' @@ -150,9 +150,9 @@ $ kubectl describe node | grep sgx.intel.com sgx.intel.com/enclave: 20 sgx.intel.com/epc: 98566144 sgx.intel.com/provision: 20 - sgx.intel.com/enclave 1 1 - sgx.intel.com/epc 400 400 - sgx.intel.com/provision 1 1 + sgx.intel.com/enclave 0 0 + sgx.intel.com/epc 0 0 + sgx.intel.com/provision 0 0 ``` ## Testing and Demos diff --git a/deployments/nfd/base/kustomization.yaml b/deployments/nfd/base/kustomization.yaml index 1eaaf9f1..bcc3c245 100644 --- a/deployments/nfd/base/kustomization.yaml +++ b/deployments/nfd/base/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization bases: -- "https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.12.1" +- "https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.13.1" diff --git a/deployments/nfd/components/sgx/kustomization.yaml b/deployments/nfd/components/sgx/kustomization.yaml deleted file mode 100644 index c4eee0c6..00000000 --- a/deployments/nfd/components/sgx/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component - -patchesJson6902: -- target: - group: apps - version: v1 - kind: Deployment - name: nfd-master - path: master-args.yaml -- target: - group: rbac.authorization.k8s.io - version: v1 - kind: ClusterRole - name: nfd-master - path: master-rbac.yaml diff --git a/deployments/nfd/components/sgx/master-args.yaml b/deployments/nfd/components/sgx/master-args.yaml deleted file mode 100644 index d1113c98..00000000 --- a/deployments/nfd/components/sgx/master-args.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- op: add - path: /spec/template/spec/containers/0/args - value: ["--resource-labels=sgx.intel.com/epc", "--extra-label-ns=sgx.intel.com"] diff --git a/deployments/nfd/components/sgx/master-rbac.yaml b/deployments/nfd/components/sgx/master-rbac.yaml deleted file mode 100644 index 62fa8272..00000000 --- a/deployments/nfd/components/sgx/master-rbac.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- op: add - path: /rules/0/resources - value: ["nodes", "nodes/status"] diff --git a/deployments/nfd/overlays/node-feature-rules/node-feature-rules-openshift.yaml b/deployments/nfd/overlays/node-feature-rules/node-feature-rules-openshift.yaml index 156282ca..278af79b 100755 --- a/deployments/nfd/overlays/node-feature-rules/node-feature-rules-openshift.yaml +++ b/deployments/nfd/overlays/node-feature-rules/node-feature-rules-openshift.yaml @@ -86,14 +86,16 @@ spec: - name: "intel.sgx" labels: "intel.feature.node.kubernetes.io/sgx": "true" + extendedResources: + sgx.intel.com/epc: "@cpu.security.sgx.epc" matchFeatures: - feature: cpu.cpuid matchExpressions: SGX: {op: Exists} SGXLC: {op: Exists} - - feature: cpu.sgx + - feature: cpu.security matchExpressions: - enabled: {op: IsTrue} + sgx.enabled: {op: IsTrue} - feature: kernel.config matchExpressions: X86_SGX: {op: Exists} diff --git a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml index fb64bd24..5e9dff04 100644 --- a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml +++ b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml @@ -90,6 +90,8 @@ spec: - name: "intel.sgx" labels: "intel.feature.node.kubernetes.io/sgx": "true" + extendedResources: + sgx.intel.com/epc: "@cpu.security.sgx.epc" matchFeatures: - feature: cpu.cpuid matchExpressions: diff --git a/deployments/nfd/overlays/sgx/kustomization.yaml b/deployments/nfd/overlays/sgx/kustomization.yaml deleted file mode 100644 index 35c5037f..00000000 --- a/deployments/nfd/overlays/sgx/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: -- ../../base - -components: -- ../../components/sgx diff --git a/deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml index 5e00d56e..3048434c 100644 --- a/deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml +++ b/deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml @@ -4,7 +4,6 @@ metadata: name: sgxdeviceplugin-sample spec: image: intel/intel-sgx-plugin:0.26.0 - initImage: intel/intel-sgx-initcontainer:0.26.0 enclaveLimit: 110 provisionLimit: 110 logLevel: 4 diff --git a/deployments/sgx_plugin/overlays/epc-nfd/add-epc-nfd-initcontainer.yaml b/deployments/sgx_plugin/overlays/epc-nfd/add-epc-nfd-initcontainer.yaml deleted file mode 100644 index 168fa94d..00000000 --- a/deployments/sgx_plugin/overlays/epc-nfd/add-epc-nfd-initcontainer.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-sgx-plugin -spec: - template: - spec: - initContainers: - - name: intel-sgx-initcontainer - image: intel/intel-sgx-initcontainer:devel - imagePullPolicy: IfNotPresent - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/node-feature-discovery/source.d/ - name: nfd-features - volumes: - - name: nfd-features - hostPath: - path: /etc/kubernetes/node-feature-discovery/source.d/ - type: DirectoryOrCreate diff --git a/deployments/sgx_plugin/overlays/epc-nfd/kustomization.yaml b/deployments/sgx_plugin/overlays/epc-nfd/kustomization.yaml index 418917ed..de7caa37 100644 --- a/deployments/sgx_plugin/overlays/epc-nfd/kustomization.yaml +++ b/deployments/sgx_plugin/overlays/epc-nfd/kustomization.yaml @@ -1,5 +1,3 @@ bases: - ../../base - ../../../sgx_admissionwebhook/overlays/default-with-certmanager -patches: - - path: add-epc-nfd-initcontainer.yaml diff --git a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go index af82a393..83eef2fa 100644 --- a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go @@ -53,10 +53,6 @@ func (r *SgxDevicePlugin) Default() { if len(r.Spec.Image) == 0 { r.Spec.Image = "intel/intel-sgx-plugin:" + sgxMinVersion.String() } - - if len(r.Spec.InitImage) == 0 { - r.Spec.InitImage = "intel/intel-sgx-initcontainer:" + sgxMinVersion.String() - } } // +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,versions=v1,name=vsgxdeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1 @@ -93,5 +89,9 @@ func (r *SgxDevicePlugin) validatePlugin() error { return err } + if r.Spec.InitImage == "" { + return nil + } + return validatePluginImage(r.Spec.InitImage, "intel-sgx-initcontainer", sgxMinVersion) } diff --git a/test/e2e/deviceplugins_suite_test.go b/test/e2e/deviceplugins_suite_test.go index 4d2e7bc1..b73beee6 100644 --- a/test/e2e/deviceplugins_suite_test.go +++ b/test/e2e/deviceplugins_suite_test.go @@ -103,7 +103,7 @@ func setupFirstNode() []byte { framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion) } - utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/overlays/sgx/kustomization.yaml") + utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/kustomization.yaml") utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/overlays/node-feature-rules/kustomization.yaml") diff --git a/test/e2e/sgx/sgx.go b/test/e2e/sgx/sgx.go index 50bc4b41..408c1bc8 100644 --- a/test/e2e/sgx/sgx.go +++ b/test/e2e/sgx/sgx.go @@ -37,7 +37,7 @@ const ( ns = "inteldeviceplugins-system" timeout = time.Second * 120 kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml" - kustomizationPlugin = "deployments/sgx_plugin/overlays/epc-hook-initcontainer/kustomization.yaml" + kustomizationPlugin = "deployments/sgx_plugin/base/kustomization.yaml" ) func init() {