diff --git a/cmd/sgx_plugin/README.md b/cmd/sgx_plugin/README.md index dfb01f94..25484a94 100644 --- a/cmd/sgx_plugin/README.md +++ b/cmd/sgx_plugin/README.md @@ -5,6 +5,7 @@ Contents * [Introduction](#introduction) * [Installation](#installation) * [Prerequisites](#prerequisites) + * [Backwards compatiblity note](#backwards-compatibility-note) * [Pre-built images](#pre-built-images) * [Getting the source code](#getting-the-source-code) * [Verify node kubelet config](#verify-node-kubelet-config) @@ -63,11 +64,44 @@ The component has the same basic dependancies as the [generic plugin framework dependencies](../../README.md#about). The SGX device plugin requires Linux Kernel SGX drivers to be available. These drivers -are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git/tag/?h=v39). -RFC *v39* was used to validate what is written in this document. +are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git). +RFC *v41* was used to validate what is written in this document. The hardware platform must support SGX Flexible Launch Control. +#### Backwards compatibility note + +The SGX device nodes have changed from `/dev/sgx/[enclave|provision]` +to `/dev/sgx_[enclave|provision]` in v4x RFC patches according to the +LKML feedback. + +Backwards compatibility is provided by adding `/dev/sgx` directory volume +mount to containers. This assumes the cluster admin has installed the +udev rules provided below to make the old device nodes as symlinks to the +new device nodes. + +**Note:** the symlinks become visible in all containers requesting SGX +resources but are potentially dangling links if the device the corresponding +device resource is not requested. + +```bash +$ cat /etc/udev/rules/9*.rules +SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666" +SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave" +SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666" +SUBSYSTEM=="misc",KERNEL=="provision",MODE="0660" +SUBSYSTEM=="misc",KERNEL=="sgx_provision",SYMLINK+="sgx/provision",MODE="0660" +SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660" +$ sudo udevadm trigger +$ ls -la /dev/sgx/* +lrwxrwxrwx 1 root root 14 Nov 18 01:01 /dev/sgx/enclave -> ../sgx_enclave +lrwxrwxrwx 1 root root 16 Nov 18 01:01 /dev/sgx/provision -> ../sgx_provision +``` + +The backwards compatibility will be removed in the next release (v0.20) and +from the main development branch once the SGX SDK and DCAP releases default to +the new devices. + ### Pre-built images [Pre-built images](https://hub.docker.com/u/intel/) diff --git a/cmd/sgx_plugin/sgx_plugin.go b/cmd/sgx_plugin/sgx_plugin.go index 0f4c0793..2c3b6249 100644 --- a/cmd/sgx_plugin/sgx_plugin.go +++ b/cmd/sgx_plugin/sgx_plugin.go @@ -68,9 +68,9 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { devTree := dpapi.NewDeviceTree() - // Assume that both /dev/sgx/enclave and /dev/sgx/provision must be present. - sgxEnclavePath := path.Join(dp.devfsDir, "sgx", "enclave") - sgxProvisionPath := path.Join(dp.devfsDir, "sgx", "provision") + // Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present. + sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave") + sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision") if _, err := os.Stat(sgxEnclavePath); err != nil { klog.Error("No SGX enclave file available: ", err) return devTree, nil @@ -80,15 +80,22 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { return devTree, nil } + deprecatedMounts := []pluginapi.Mount{ + { + HostPath: "/dev/sgx", + ContainerPath: "/dev/sgx", + }, + } + for i := uint(0); i < dp.nEnclave; i++ { devID := fmt.Sprintf("%s-%d", "sgx-enclave", i) nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}} - devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil)) } for i := uint(0); i < dp.nProvision; i++ { devID := fmt.Sprintf("%s-%d", "sgx-provision", i) nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}} - devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil)) } return devTree, nil } diff --git a/cmd/sgx_plugin/sgx_plugin_test.go b/cmd/sgx_plugin/sgx_plugin_test.go index 2b8d13d6..894ff883 100644 --- a/cmd/sgx_plugin/sgx_plugin_test.go +++ b/cmd/sgx_plugin/sgx_plugin_test.go @@ -103,22 +103,22 @@ func TestScan(t *testing.T) { }, { name: "only enclave file", - enclaveDevice: "enclave", + enclaveDevice: "sgx_enclave", requestedEnclaveDevs: 1, expectedEnclaveDevs: 0, expectedProvisionDevs: 0, }, { name: "only provision file", - provisionDevice: "provision", + provisionDevice: "sgx_provision", requestedProvisionDevs: 1, expectedEnclaveDevs: 0, expectedProvisionDevs: 0, }, { name: "one device", - enclaveDevice: "enclave", - provisionDevice: "provision", + enclaveDevice: "sgx_enclave", + provisionDevice: "sgx_provision", requestedEnclaveDevs: 1, expectedEnclaveDevs: 1, requestedProvisionDevs: 1, @@ -126,8 +126,8 @@ func TestScan(t *testing.T) { }, { name: "one device", - enclaveDevice: "enclave", - provisionDevice: "provision", + enclaveDevice: "sgx_enclave", + provisionDevice: "sgx_provision", requestedEnclaveDevs: 10, expectedEnclaveDevs: 10, requestedProvisionDevs: 20, @@ -144,20 +144,20 @@ func TestScan(t *testing.T) { defer func() { _ = os.RemoveAll(root) }() devfs := path.Join(root, "dev") - err = os.MkdirAll(path.Join(devfs, "sgx"), 0750) + err = os.MkdirAll(devfs, 0750) if err != nil { t.Fatalf("Failed to create fake device directory: %+v", err) } if tc.enclaveDevice != "" { - err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.enclaveDevice), []byte{}, 0600) + err = ioutil.WriteFile(path.Join(devfs, tc.enclaveDevice), []byte{}, 0600) if err != nil { - t.Fatalf("Failed to create fake vendor file: %+v", err) + t.Fatalf("Failed to create fake enclave file: %+v", err) } } if tc.provisionDevice != "" { - err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.provisionDevice), []byte{}, 0600) + err = ioutil.WriteFile(path.Join(devfs, tc.provisionDevice), []byte{}, 0600) if err != nil { - t.Fatalf("Failed to create fake vendor file: %+v", err) + t.Fatalf("Failed to create fake provision file: %+v", err) } } diff --git a/deployments/sgx_plugin/base/intel-sgx-plugin.yaml b/deployments/sgx_plugin/base/intel-sgx-plugin.yaml index 1e24f9e9..ac204fb0 100644 --- a/deployments/sgx_plugin/base/intel-sgx-plugin.yaml +++ b/deployments/sgx_plugin/base/intel-sgx-plugin.yaml @@ -34,6 +34,12 @@ spec: - name: sgxdevices mountPath: /dev/sgx readOnly: true + - name: sgx-enclave + mountPath: /dev/sgx_enclave + readOnly: true + - name: sgx-provision + mountPath: /dev/sgx_provision + readOnly: true volumes: - name: kubeletsockets hostPath: @@ -41,6 +47,15 @@ spec: - name: sgxdevices hostPath: path: /dev/sgx + type: DirectoryOrCreate + - name: sgx-enclave + hostPath: + path: /dev/sgx_enclave + type: CharDevice + - name: sgx-provision + hostPath: + path: /dev/sgx_provision + type: CharDevice - name: nfd-source-hooks hostPath: path: /etc/kubernetes/node-feature-discovery/source.d/ diff --git a/pkg/controllers/sgx/controller.go b/pkg/controllers/sgx/controller.go index 4259e699..ced52563 100644 --- a/pkg/controllers/sgx/controller.go +++ b/pkg/controllers/sgx/controller.go @@ -81,6 +81,7 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { } yes := true + charDevice := v1.HostPathCharDev directoryOrCreate := v1.HostPathDirectoryOrCreate return &apps.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ @@ -134,6 +135,16 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { MountPath: "/dev/sgx", ReadOnly: true, }, + { + Name: "sgx-enclave", + MountPath: "/dev/sgx_enclave", + ReadOnly: true, + }, + { + Name: "sgx-provision", + MountPath: "/dev/sgx_provision", + ReadOnly: true, + }, { Name: "kubeletsockets", MountPath: "/var/lib/kubelet/device-plugins", @@ -148,6 +159,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { VolumeSource: v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ Path: "/dev/sgx", + Type: &directoryOrCreate, + }, + }, + }, + { + Name: "sgx-enclave", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/sgx_enclave", + Type: &charDevice, + }, + }, + }, + { + Name: "sgx-provision", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/sgx_provision", + Type: &charDevice, }, }, },