From c8b5dce2476bc6cdda2b42050f727131e1ab0d18 Mon Sep 17 00:00:00 2001 From: Dmitry Shmulevich Date: Tue, 9 Mar 2021 20:45:42 -0800 Subject: [PATCH] added an option to create a node label if epc memory is present updated README for SGX device plugin Signed-off-by: Dmitry Shmulevich --- cmd/sgx_epchook/main.go | 84 ++++++++++++------- cmd/sgx_plugin/README.md | 12 ++- .../add-epc-register-initcontainer.yaml | 20 ----- .../epc-register/add-node-selector.yaml | 9 ++ .../overlays/epc-register/init-daemonset.yaml | 35 ++++++++ .../overlays/epc-register/kustomization.yaml | 3 +- .../epc-register/service-account.yaml | 10 +-- 7 files changed, 117 insertions(+), 56 deletions(-) delete mode 100644 deployments/sgx_plugin/overlays/epc-register/add-epc-register-initcontainer.yaml create mode 100644 deployments/sgx_plugin/overlays/epc-register/add-node-selector.yaml create mode 100644 deployments/sgx_plugin/overlays/epc-register/init-daemonset.yaml diff --git a/cmd/sgx_epchook/main.go b/cmd/sgx_epchook/main.go index de15551a..52d25d80 100644 --- a/cmd/sgx_epchook/main.go +++ b/cmd/sgx_epchook/main.go @@ -20,6 +20,8 @@ import ( "flag" "fmt" "os" + "os/signal" + "syscall" "github.com/klauspost/cpuid/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,23 +32,27 @@ import ( ) const ( - namespace = "sgx.intel.com" - epc = "epc" - pathPrefix = "/status/capacity" + namespace = "sgx.intel.com" + epc = "epc" + capable = "capable" ) -type patchExtendedResource struct { - Op string `json:"op"` - Path string `json:"path"` - Value uint64 `json:"value"` +type patchNodeOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value"` } func main() { - var register, affirm bool + var register, affirm, label, daemon bool flag.BoolVar(®ister, "register", false, "register EPC as extended resource") flag.BoolVar(&affirm, "affirm", false, "return error if EPC is not available") + flag.BoolVar(&label, "node-label", false, "create node label") + flag.BoolVar(&daemon, "daemon", false, "run as a daemon") flag.Parse() + klog.Infof("starting sgx_epchook") + // get the EPC size var epcSize uint64 if cpuid.CPU.SGX.Available { @@ -54,21 +60,55 @@ func main() { epcSize += s.EPCSize } } + klog.Infof("epc capacity: %d bytes", epcSize) if epcSize == 0 && affirm { klog.Fatal("SGX EPC is not available") } - if register { - if err := registerExtendedResource(epcSize); err != nil { - klog.Fatal(err.Error()) - } - } else { + if err := updateNode(epcSize, register, label); err != nil { + klog.Fatal(err.Error()) + } + + // if the "register" flag is FALSE, we assume that sgx_epchook is used as NFD hook + if !register { fmt.Printf("%s/%s=%d", namespace, epc, epcSize) } + + if daemon { + klog.Info("waiting for termination signal") + term := make(chan os.Signal, 1) + signal.Notify(term, os.Interrupt, syscall.SIGTERM) + <-term + } } -func registerExtendedResource(epcSize uint64) error { +func updateNode(epcSize uint64, register, label bool) error { + // create patch payload + payload := []patchNodeOp{} + if register { + payload = append(payload, patchNodeOp{ + Op: "add", + Path: fmt.Sprintf("/status/capacity/%s~1%s", namespace, epc), + Value: epcSize, + }) + } + if label && epcSize > 0 { + payload = append(payload, patchNodeOp{ + Op: "add", + Path: fmt.Sprintf("/metadata/labels/%s~1%s", namespace, capable), + Value: "true", + }) + } + if len(payload) == 0 { + return nil + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + return err + } + // create the in-cluster config config, err := rest.InClusterConfig() if err != nil { @@ -87,19 +127,7 @@ func registerExtendedResource(epcSize uint64) error { return err } - // create and send patch request - payload := []patchExtendedResource{{ - Op: "add", - Path: fmt.Sprintf("%s/%s~1%s", pathPrefix, namespace, epc), - Value: epcSize, - }} - payloadBytes, err := json.Marshal(payload) - if err != nil { - return err - } + // patch the node _, err = clientset.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}, "status") - if err != nil { - return err - } - return nil + return err } diff --git a/cmd/sgx_plugin/README.md b/cmd/sgx_plugin/README.md index 7daad690..e3e3841f 100644 --- a/cmd/sgx_plugin/README.md +++ b/cmd/sgx_plugin/README.md @@ -174,8 +174,9 @@ Successfully tagged intel/intel-sgx-initcontainer:devel #### Deploy the DaemonSet -Deploying the plugin involves the deployment of the -[SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml) +There are two alternative ways to deploy SGX device plugin. + +The first approach involves deployment of the [SGX DaemonSet YAML](/deployments/sgx_plugin/base/intel-sgx-plugin.yaml) and [node-feature-discovery](/deployments/sgx_nfd/kustomization.yaml) with the necessary configuration. @@ -184,6 +185,13 @@ There is a kustomization for deploying everything: $ kubectl apply -k ${INTEL_DEVICE_PLUGINS_SRC}/deployments/sgx_plugin/overlays/epc-nfd/ ``` +The second approach has a lesser deployment footprint. It does not deploy NFD, but a helper daemonset that creates `sgx.intel.com/capable='true'` node label and advertises EPC capacity to the API server. + +The following kustomization is used for this approach: +```bash +$ kubectl apply -k ${INTEL_DEVICE_PLUGINS_SRC}/deployments/sgx_plugin/overlays/epc-register/ +``` + #### Verify SGX device plugin is registered: Verification of the plugin deployment and detection of SGX hardware can be confirmed by diff --git a/deployments/sgx_plugin/overlays/epc-register/add-epc-register-initcontainer.yaml b/deployments/sgx_plugin/overlays/epc-register/add-epc-register-initcontainer.yaml deleted file mode 100644 index 2dbb11e3..00000000 --- a/deployments/sgx_plugin/overlays/epc-register/add-epc-register-initcontainer.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-sgx-plugin -spec: - template: - spec: - serviceAccountName: sgx-epc-extres - initContainers: - - name: intel-sgx-initcontainer - image: intel/intel-sgx-initcontainer:devel - imagePullPolicy: IfNotPresent - command: - - /usr/local/bin/sgx-sw/intel-sgx-epchook - - -register - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName diff --git a/deployments/sgx_plugin/overlays/epc-register/add-node-selector.yaml b/deployments/sgx_plugin/overlays/epc-register/add-node-selector.yaml new file mode 100644 index 00000000..5c0334b4 --- /dev/null +++ b/deployments/sgx_plugin/overlays/epc-register/add-node-selector.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-sgx-plugin +spec: + template: + spec: + nodeSelector: + sgx.intel.com/capable: 'true' diff --git a/deployments/sgx_plugin/overlays/epc-register/init-daemonset.yaml b/deployments/sgx_plugin/overlays/epc-register/init-daemonset.yaml new file mode 100644 index 00000000..e513bc79 --- /dev/null +++ b/deployments/sgx_plugin/overlays/epc-register/init-daemonset.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sgx-node-init + labels: + app: sgx-node-init +spec: + selector: + matchLabels: + app: sgx-node-init + template: + metadata: + labels: + app: sgx-node-init + spec: + serviceAccountName: sgx-plugin + containers: + - name: sgx-node-init + image: intel/intel-sgx-initcontainer:devel + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/sgx-sw/intel-sgx-epchook + - -register + - -node-label + - -daemon + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/deployments/sgx_plugin/overlays/epc-register/kustomization.yaml b/deployments/sgx_plugin/overlays/epc-register/kustomization.yaml index 11366a20..6727bfae 100644 --- a/deployments/sgx_plugin/overlays/epc-register/kustomization.yaml +++ b/deployments/sgx_plugin/overlays/epc-register/kustomization.yaml @@ -3,5 +3,6 @@ bases: namespace: kube-system resources: - service-account.yaml + - init-daemonset.yaml patches: - - add-epc-register-initcontainer.yaml + - add-node-selector.yaml diff --git a/deployments/sgx_plugin/overlays/epc-register/service-account.yaml b/deployments/sgx_plugin/overlays/epc-register/service-account.yaml index f6b39479..ce5b3217 100644 --- a/deployments/sgx_plugin/overlays/epc-register/service-account.yaml +++ b/deployments/sgx_plugin/overlays/epc-register/service-account.yaml @@ -1,13 +1,13 @@ kind: ServiceAccount apiVersion: v1 metadata: - name: sgx-epc-extres + name: sgx-plugin namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: sgx-epc-extres-rd + name: sgx-plugin rules: - apiGroups: - "" @@ -22,12 +22,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: sgx-epc-extres-rd + name: sgx-plugin roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: sgx-epc-extres-rd + name: sgx-plugin subjects: - kind: ServiceAccount - name: sgx-epc-extres + name: sgx-plugin namespace: kube-system