sgx: move to RFC v4x device API

The SGX device nodes have changed from /dev/sgx/[enclave|provision]
to /dev/sgx_[enclave|provision] in v4x RFC patches according to the
LKML feedback.

This changes moves to use the new device nodes. Backwards compatibility
is provided by adding /dev/sgx directory mount to containers. This
assumes the cluster admin has installed the udev rules provided in the
README to make the old device nodes as symlinks to the new device nodes.

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

cmd/sgx_plugin/README.md

@@ -5,6 +5,7 @@ Contents
 * [Introduction](#introduction)
 * [Installation](#installation)
     * [Prerequisites](#prerequisites)
+        * [Backwards compatiblity note](#backwards-compatibility-note)
     * [Pre-built images](#pre-built-images)
     * [Getting the source code](#getting-the-source-code)
     * [Verify node kubelet config](#verify-node-kubelet-config)
@@ -63,11 +64,44 @@ The component has the same basic dependancies as the
 [generic plugin framework dependencies](../../README.md#about).
 
 The SGX device plugin requires Linux Kernel SGX drivers to be available. These drivers
-are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git/tag/?h=v39).
-RFC *v39* was used to validate what is written in this document.
+are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git).
+RFC *v41* was used to validate what is written in this document.
 
 The hardware platform must support SGX Flexible Launch Control.
 
+#### Backwards compatibility note
+
+The SGX device nodes have changed from `/dev/sgx/[enclave|provision]`
+to `/dev/sgx_[enclave|provision]` in v4x RFC patches according to the
+LKML feedback.
+
+Backwards compatibility is provided by adding `/dev/sgx` directory volume
+mount to containers. This assumes the cluster admin has installed the
+udev rules provided below to make the old device nodes as symlinks to the
+new device nodes.
+
+**Note:** the symlinks become visible in all containers requesting SGX
+resources but are potentially dangling links if the device the corresponding
+device resource is not requested.
+
+```bash
+$ cat /etc/udev/rules/9*.rules
+SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666"
+SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave"
+SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666"
+SUBSYSTEM=="misc",KERNEL=="provision",MODE="0660"
+SUBSYSTEM=="misc",KERNEL=="sgx_provision",SYMLINK+="sgx/provision",MODE="0660"
+SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660"
+$ sudo udevadm trigger
+$ ls -la /dev/sgx/*
+lrwxrwxrwx 1 root root 14 Nov 18 01:01 /dev/sgx/enclave -> ../sgx_enclave
+lrwxrwxrwx 1 root root 16 Nov 18 01:01 /dev/sgx/provision -> ../sgx_provision
+```
+
+The backwards compatibility will be removed in the next release (v0.20) and
+from the main development branch once the SGX SDK and DCAP releases default to
+the new devices.
+
 ### Pre-built images
 
 [Pre-built images](https://hub.docker.com/u/intel/)

cmd/sgx_plugin/sgx_plugin.go

@@ -68,9 +68,9 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 	devTree := dpapi.NewDeviceTree()
 
-	// Assume that both /dev/sgx/enclave and /dev/sgx/provision must be present.
-	sgxEnclavePath := path.Join(dp.devfsDir, "sgx", "enclave")
-	sgxProvisionPath := path.Join(dp.devfsDir, "sgx", "provision")
+	// Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present.
+	sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave")
+	sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision")
 	if _, err := os.Stat(sgxEnclavePath); err != nil {
 		klog.Error("No SGX enclave file available: ", err)
 		return devTree, nil
@@ -80,15 +80,22 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 		return devTree, nil
 	}
 
+	deprecatedMounts := []pluginapi.Mount{
+		{
+			HostPath:      "/dev/sgx",
+			ContainerPath: "/dev/sgx",
+		},
+	}
+
 	for i := uint(0); i < dp.nEnclave; i++ {
 		devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
 		nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
+		devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
 	}
 	for i := uint(0); i < dp.nProvision; i++ {
 		devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
 		nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
+		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
 	}
 	return devTree, nil
 }

cmd/sgx_plugin/sgx_plugin_test.go

@@ -103,22 +103,22 @@ func TestScan(t *testing.T) {
 		},
 		{
 			name:                  "only enclave file",
-			enclaveDevice:         "enclave",
+			enclaveDevice:         "sgx_enclave",
 			requestedEnclaveDevs:  1,
 			expectedEnclaveDevs:   0,
 			expectedProvisionDevs: 0,
 		},
 		{
 			name:                   "only provision file",
-			provisionDevice:        "provision",
+			provisionDevice:        "sgx_provision",
 			requestedProvisionDevs: 1,
 			expectedEnclaveDevs:    0,
 			expectedProvisionDevs:  0,
 		},
 		{
 			name:                   "one device",
-			enclaveDevice:          "enclave",
-			provisionDevice:        "provision",
+			enclaveDevice:          "sgx_enclave",
+			provisionDevice:        "sgx_provision",
 			requestedEnclaveDevs:   1,
 			expectedEnclaveDevs:    1,
 			requestedProvisionDevs: 1,
@@ -126,8 +126,8 @@ func TestScan(t *testing.T) {
 		},
 		{
 			name:                   "one device",
-			enclaveDevice:          "enclave",
-			provisionDevice:        "provision",
+			enclaveDevice:          "sgx_enclave",
+			provisionDevice:        "sgx_provision",
 			requestedEnclaveDevs:   10,
 			expectedEnclaveDevs:    10,
 			requestedProvisionDevs: 20,
@@ -144,20 +144,20 @@ func TestScan(t *testing.T) {
 			defer func() { _ = os.RemoveAll(root) }()
 
 			devfs := path.Join(root, "dev")
-			err = os.MkdirAll(path.Join(devfs, "sgx"), 0750)
+			err = os.MkdirAll(devfs, 0750)
 			if err != nil {
 				t.Fatalf("Failed to create fake device directory: %+v", err)
 			}
 			if tc.enclaveDevice != "" {
-				err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.enclaveDevice), []byte{}, 0600)
+				err = ioutil.WriteFile(path.Join(devfs, tc.enclaveDevice), []byte{}, 0600)
 				if err != nil {
-					t.Fatalf("Failed to create fake vendor file: %+v", err)
+					t.Fatalf("Failed to create fake enclave file: %+v", err)
 				}
 			}
 			if tc.provisionDevice != "" {
-				err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.provisionDevice), []byte{}, 0600)
+				err = ioutil.WriteFile(path.Join(devfs, tc.provisionDevice), []byte{}, 0600)
 				if err != nil {
-					t.Fatalf("Failed to create fake vendor file: %+v", err)
+					t.Fatalf("Failed to create fake provision file: %+v", err)
 				}
 			}
 

deployments/sgx_plugin/base/intel-sgx-plugin.yaml

@@ -34,6 +34,12 @@ spec:
         - name: sgxdevices
           mountPath: /dev/sgx
           readOnly: true
+        - name: sgx-enclave
+          mountPath: /dev/sgx_enclave
+          readOnly: true
+        - name: sgx-provision
+          mountPath: /dev/sgx_provision
+          readOnly: true
       volumes:
       - name: kubeletsockets
         hostPath:
@@ -41,6 +47,15 @@ spec:
       - name: sgxdevices
         hostPath:
           path: /dev/sgx
+          type: DirectoryOrCreate
+      - name: sgx-enclave
+        hostPath:
+          path: /dev/sgx_enclave
+          type: CharDevice
+      - name: sgx-provision
+        hostPath:
+          path: /dev/sgx_provision
+          type: CharDevice
       - name: nfd-source-hooks
         hostPath:
           path: /etc/kubernetes/node-feature-discovery/source.d/

pkg/controllers/sgx/controller.go

@@ -81,6 +81,7 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	}
 
 	yes := true
+	charDevice := v1.HostPathCharDev
 	directoryOrCreate := v1.HostPathDirectoryOrCreate
 	return &apps.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
@@ -134,6 +135,16 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 									MountPath: "/dev/sgx",
 									ReadOnly:  true,
 								},
+								{
+									Name:      "sgx-enclave",
+									MountPath: "/dev/sgx_enclave",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "sgx-provision",
+									MountPath: "/dev/sgx_provision",
+									ReadOnly:  true,
+								},
 								{
 									Name:      "kubeletsockets",
 									MountPath: "/var/lib/kubelet/device-plugins",
@@ -148,6 +159,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 							VolumeSource: v1.VolumeSource{
 								HostPath: &v1.HostPathVolumeSource{
 									Path: "/dev/sgx",
+									Type: &directoryOrCreate,
+								},
+							},
+						},
+						{
+							Name: "sgx-enclave",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev/sgx_enclave",
+									Type: &charDevice,
+								},
+							},
+						},
+						{
+							Name: "sgx-provision",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev/sgx_provision",
+									Type: &charDevice,
 								},
 							},
 						},