Merge pull request #597 from dmitsh/ds-node-label

option to add a node label if epc memory is present

cmd/sgx_epchook/main.go

@@ -20,6 +20,8 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/klauspost/cpuid/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -30,23 +32,27 @@ import (
 )
 
 const (
-	namespace  = "sgx.intel.com"
+	namespace = "sgx.intel.com"
-	epc        = "epc"
+	epc       = "epc"
-	pathPrefix = "/status/capacity"
+	capable   = "capable"
 )
 
-type patchExtendedResource struct {
+type patchNodeOp struct {
-	Op    string `json:"op"`
+	Op    string      `json:"op"`
-	Path  string `json:"path"`
+	Path  string      `json:"path"`
-	Value uint64 `json:"value"`
+	Value interface{} `json:"value"`
 }
 
 func main() {
-	var register, affirm bool
+	var register, affirm, label, daemon bool
 	flag.BoolVar(&register, "register", false, "register EPC as extended resource")
 	flag.BoolVar(&affirm, "affirm", false, "return error if EPC is not available")
+	flag.BoolVar(&label, "node-label", false, "create node label")
+	flag.BoolVar(&daemon, "daemon", false, "run as a daemon")
 	flag.Parse()
 
+	klog.Infof("starting sgx_epchook")
+
 	// get the EPC size
 	var epcSize uint64
 	if cpuid.CPU.SGX.Available {
@@ -54,21 +60,55 @@ func main() {
 			epcSize += s.EPCSize
 		}
 	}
+	klog.Infof("epc capacity: %d bytes", epcSize)
 
 	if epcSize == 0 && affirm {
 		klog.Fatal("SGX EPC is not available")
 	}
 
-	if register {
+	if err := updateNode(epcSize, register, label); err != nil {
-		if err := registerExtendedResource(epcSize); err != nil {
+		klog.Fatal(err.Error())
-			klog.Fatal(err.Error())
+	}
-		}
+
-	} else {
+	// if the "register" flag is FALSE, we assume that sgx_epchook is used as NFD hook
+	if !register {
 		fmt.Printf("%s/%s=%d", namespace, epc, epcSize)
 	}
+
+	if daemon {
+		klog.Info("waiting for termination signal")
+		term := make(chan os.Signal, 1)
+		signal.Notify(term, os.Interrupt, syscall.SIGTERM)
+		<-term
+	}
 }
 
-func registerExtendedResource(epcSize uint64) error {
+func updateNode(epcSize uint64, register, label bool) error {
+	// create patch payload
+	payload := []patchNodeOp{}
+	if register {
+		payload = append(payload, patchNodeOp{
+			Op:    "add",
+			Path:  fmt.Sprintf("/status/capacity/%s~1%s", namespace, epc),
+			Value: epcSize,
+		})
+	}
+	if label && epcSize > 0 {
+		payload = append(payload, patchNodeOp{
+			Op:    "add",
+			Path:  fmt.Sprintf("/metadata/labels/%s~1%s", namespace, capable),
+			Value: "true",
+		})
+	}
+	if len(payload) == 0 {
+		return nil
+	}
+
+	payloadBytes, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+
 	// create the in-cluster config
 	config, err := rest.InClusterConfig()
 	if err != nil {
@@ -87,19 +127,7 @@ func registerExtendedResource(epcSize uint64) error {
 		return err
 	}
 
-	// create and send patch request
+	// patch the node
-	payload := []patchExtendedResource{{
-		Op:    "add",
-		Path:  fmt.Sprintf("%s/%s~1%s", pathPrefix, namespace, epc),
-		Value: epcSize,
-	}}
-	payloadBytes, err := json.Marshal(payload)
-	if err != nil {
-		return err
-	}
 	_, err = clientset.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}, "status")
-	if err != nil {
+	return err
-		return err
-	}
-	return nil
 }

cmd/sgx_plugin/README.md

@@ -174,8 +174,9 @@ Successfully tagged intel/intel-sgx-initcontainer:devel
 
 #### Deploy the DaemonSet
 
-Deploying the plugin involves the deployment of the
+There are two alternative ways to deploy SGX device plugin.
-[SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml)
+
+The first approach involves deployment of the [SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml)
 and [node-feature-discovery](/deployments/sgx_nfd/kustomization.yaml)
 with the necessary configuration.
 
@@ -184,6 +185,13 @@ There is a kustomization for deploying everything:
 $ kubectl apply -k ${INTEL_DEVICE_PLUGINS_SRC}/deployments/sgx_plugin/overlays/epc-nfd/
 ```
 
+The second approach has a lesser deployment footprint. It does not deploy NFD, but a helper daemonset that creates `sgx.intel.com/capable='true'` node label and advertises EPC capacity to the API server.
+
+The following kustomization is used for this approach:
+```bash
+$ kubectl apply -k ${INTEL_DEVICE_PLUGINS_SRC}/deployments/sgx_plugin/overlays/epc-register/
+```
+
 #### Verify SGX device plugin is registered:
 
 Verification of the plugin deployment and detection of SGX hardware can be confirmed by

deployments/sgx_plugin/overlays/epc-register/add-epc-register-initcontainer.yaml

@@ -1,20 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: intel-sgx-plugin
-spec:
-  template:
-    spec:
-      serviceAccountName: sgx-epc-extres
-      initContainers:
-      - name: intel-sgx-initcontainer
-        image: intel/intel-sgx-initcontainer:devel
-        imagePullPolicy: IfNotPresent
-        command:
-          - /usr/local/bin/sgx-sw/intel-sgx-epchook
-          - -register
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName

deployments/sgx_plugin/overlays/epc-register/add-node-selector.yaml

@@ -0,0 +1,9 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-sgx-plugin
+spec:
+  template:
+    spec:
+      nodeSelector:
+        sgx.intel.com/capable: 'true'

deployments/sgx_plugin/overlays/epc-register/init-daemonset.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sgx-node-init
+  labels:
+    app: sgx-node-init
+spec:
+  selector:
+    matchLabels:
+      app: sgx-node-init
+  template:
+    metadata:
+      labels:
+        app: sgx-node-init
+    spec:
+      serviceAccountName: sgx-plugin
+      containers:
+      - name: sgx-node-init
+        image: intel/intel-sgx-initcontainer:devel
+        imagePullPolicy: IfNotPresent
+        command:
+          - /usr/local/bin/sgx-sw/intel-sgx-epchook
+          - -register
+          - -node-label
+          - -daemon
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL

deployments/sgx_plugin/overlays/epc-register/kustomization.yaml

@@ -3,5 +3,6 @@ bases:
 namespace: kube-system
 resources:
   - service-account.yaml
+  - init-daemonset.yaml
 patches:
-  - add-epc-register-initcontainer.yaml
+  - add-node-selector.yaml

deployments/sgx_plugin/overlays/epc-register/service-account.yaml

@@ -1,13 +1,13 @@
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: sgx-epc-extres
+  name: sgx-plugin
   namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: sgx-epc-extres-rd
+  name: sgx-plugin
 rules:
 - apiGroups:
   - ""
@@ -22,12 +22,12 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: sgx-epc-extres-rd
+  name: sgx-plugin
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: sgx-epc-extres-rd
+  name: sgx-plugin
 subjects:
 - kind: ServiceAccount
-  name: sgx-epc-extres
+  name: sgx-plugin
   namespace: kube-system