Merge pull request #1392 from mythi/PR-2023-019

sgx: stop using local source hooks for EPC registration

cmd/operator/README.md

@@ -23,14 +23,12 @@ The default operator deployment depends on NFD and cert-manager. Those component
 
 ### NFD
 
-Install NFD (if it's not already installed) and node labelling rules (requires NFD v0.10+):
+Install NFD (if it's not already installed) and node labelling rules (requires NFD v0.13+):
 
 ```
-# either with default NFD installation
+# deploy NFD
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=<RELEASE_VERSION>'
-# or when setting up with SGX
+# deploy NodeFeatureRules
-$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=<RELEASE_VERSION>'
-# and finally, NodeFeatureRules
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=<RELEASE_VERSION>'
 ```
 Make sure both NFD master and worker pods are running:

cmd/sgx_plugin/README.md

@@ -84,7 +84,7 @@ Where `<RELEASE_VERSION>` needs to be substituted with the desired [release tag]
 First, deploy `node-feature-discovery`:
 
 ```bash
-$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=<RELEASE_VERSION>'
+$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/?ref=<RELEASE_VERSION>'
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=<RELEASE_VERSION>'
 ```
 
@@ -110,13 +110,13 @@ $ kubectl apply -f 'https://raw.githubusercontent.com/intel/intel-device-plugins
 There are two alternative ways to deploy SGX device plugin using `kubectl`.
 
 The first approach involves deployment of the [SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml)
-and [node-feature-discovery](/deployments/nfd/overlays/sgx/kustomization.yaml)
+and [node-feature-discovery](/deployments/nfd/kustomization.yaml)
 with the necessary configuration.
 
 The following kustomizations are needed for deploying everything:
 ```bash
 # first, deploy NFD and the necessary NodeFeatureRules
-$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx'
+$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd'
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules'
 # and then, deploy SGX plugin
 $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_plugin/overlays/epc-nfd/'
@@ -150,9 +150,9 @@ $ kubectl describe node <node name> | grep sgx.intel.com
  sgx.intel.com/enclave:    20
  sgx.intel.com/epc:        98566144
  sgx.intel.com/provision:  20
- sgx.intel.com/enclave    1           1
+ sgx.intel.com/enclave    0           0
- sgx.intel.com/epc        400         400
+ sgx.intel.com/epc        0           0
- sgx.intel.com/provision  1           1
+ sgx.intel.com/provision  0           0
 ```
 
 ## Testing and Demos

deployments/nfd/base/kustomization.yaml

@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- "https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.12.1"
+- "https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.13.1"

deployments/nfd/components/sgx/kustomization.yaml

@@ -1,16 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1alpha1
-kind: Component
-
-patchesJson6902:
-- target:
-    group: apps
-    version: v1
-    kind: Deployment
-    name: nfd-master
-  path: master-args.yaml
-- target:
-    group: rbac.authorization.k8s.io
-    version: v1
-    kind: ClusterRole
-    name: nfd-master
-  path: master-rbac.yaml

deployments/nfd/components/sgx/master-args.yaml

@@ -1,3 +0,0 @@
-- op: add
-  path: /spec/template/spec/containers/0/args
-  value: ["--resource-labels=sgx.intel.com/epc", "--extra-label-ns=sgx.intel.com"]

deployments/nfd/components/sgx/master-rbac.yaml

@@ -1,3 +0,0 @@
-- op: add
-  path: /rules/0/resources
-  value: ["nodes", "nodes/status"]

deployments/nfd/overlays/node-feature-rules/node-feature-rules-openshift.yaml

@@ -86,14 +86,16 @@ spec:
     - name: "intel.sgx"
       labels:
         "intel.feature.node.kubernetes.io/sgx": "true"
+      extendedResources:
+        sgx.intel.com/epc: "@cpu.security.sgx.epc"
       matchFeatures:
         - feature: cpu.cpuid
           matchExpressions:
             SGX: {op: Exists}
             SGXLC: {op: Exists}
-        - feature: cpu.sgx
+        - feature: cpu.security
           matchExpressions:
-            enabled: {op: IsTrue}
+            sgx.enabled: {op: IsTrue}
         - feature: kernel.config
           matchExpressions:
             X86_SGX: {op: Exists}

deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml

@@ -90,6 +90,8 @@ spec:
     - name: "intel.sgx"
       labels:
         "intel.feature.node.kubernetes.io/sgx": "true"
+      extendedResources:
+        sgx.intel.com/epc: "@cpu.security.sgx.epc"
       matchFeatures:
         - feature: cpu.cpuid
           matchExpressions:

deployments/nfd/overlays/sgx/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-bases:
-- ../../base
-
-components:
-- ../../components/sgx

deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml

@@ -4,7 +4,6 @@ metadata:
   name: sgxdeviceplugin-sample
 spec:
   image: intel/intel-sgx-plugin:0.26.0
-  initImage: intel/intel-sgx-initcontainer:0.26.0
   enclaveLimit: 110
   provisionLimit: 110
   logLevel: 4

deployments/sgx_plugin/overlays/epc-nfd/add-epc-nfd-initcontainer.yaml

@@ -1,22 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: intel-sgx-plugin
-spec:
-  template:
-    spec:
-      initContainers:
-      - name: intel-sgx-initcontainer
-        image: intel/intel-sgx-initcontainer:devel
-        imagePullPolicy: IfNotPresent
-        securityContext:
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
-        volumeMounts:
-        - mountPath: /etc/kubernetes/node-feature-discovery/source.d/
-          name: nfd-features
-      volumes:
-      - name: nfd-features
-        hostPath:
-          path: /etc/kubernetes/node-feature-discovery/source.d/
-          type: DirectoryOrCreate

deployments/sgx_plugin/overlays/epc-nfd/kustomization.yaml

@@ -1,5 +1,3 @@
 bases:
   - ../../base
   - ../../../sgx_admissionwebhook/overlays/default-with-certmanager
-patches:
-  - path: add-epc-nfd-initcontainer.yaml

pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go

@@ -53,10 +53,6 @@ func (r *SgxDevicePlugin) Default() {
 	if len(r.Spec.Image) == 0 {
 		r.Spec.Image = "intel/intel-sgx-plugin:" + sgxMinVersion.String()
 	}
-
-	if len(r.Spec.InitImage) == 0 {
-		r.Spec.InitImage = "intel/intel-sgx-initcontainer:" + sgxMinVersion.String()
-	}
 }
 
 // +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,versions=v1,name=vsgxdeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1
@@ -93,5 +89,9 @@ func (r *SgxDevicePlugin) validatePlugin() error {
 		return err
 	}
 
+	if r.Spec.InitImage == "" {
+		return nil
+	}
+
 	return validatePluginImage(r.Spec.InitImage, "intel-sgx-initcontainer", sgxMinVersion)
 }

test/e2e/deviceplugins_suite_test.go

@@ -103,7 +103,7 @@ func setupFirstNode() []byte {
 		framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
 	}
 
-	utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/overlays/sgx/kustomization.yaml")
+	utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/kustomization.yaml")
 
 	utils.Kubectl("node-feature-discovery", "apply", "-k", "deployments/nfd/overlays/node-feature-rules/kustomization.yaml")
 

test/e2e/sgx/sgx.go

@@ -37,7 +37,7 @@ const (
 	ns                   = "inteldeviceplugins-system"
 	timeout              = time.Second * 120
 	kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
-	kustomizationPlugin  = "deployments/sgx_plugin/overlays/epc-hook-initcontainer/kustomization.yaml"
+	kustomizationPlugin  = "deployments/sgx_plugin/base/kustomization.yaml"
 )
 
 func init() {