Merge pull request #433 from mythi/sgx

sgx: add SgxDevicePlugin CRD and admission webhook

cmd/operator/main.go

@@ -30,8 +30,10 @@ import (
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat"
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/sgx"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpgacontroller"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpgacontroller/patcher"
+	sgxwebhook "github.com/intel/intel-device-plugins-for-kubernetes/pkg/webhooks/sgx"
 )
 
 var (
@@ -90,12 +92,25 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = sgx.SetupReconciler(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "SgxDevicePlugin")
+		os.Exit(1)
+	}
+	if err = (&devicepluginv1.SgxDevicePlugin{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "SgxDevicePlugin")
+		os.Exit(1)
+	}
+
 	pm := patcher.NewPatcherManager(mgr.GetLogger().WithName("webhooks").WithName("Fpga"))
 
 	mgr.GetWebhookServer().Register("/pods", &webhook.Admission{
 		Handler: admission.HandlerFunc(pm.GetPodMutator()),
 	})
 
+	mgr.GetWebhookServer().Register("/pods-sgx", &webhook.Admission{
+		Handler: &sgxwebhook.SgxMutator{Client: mgr.GetClient()},
+	})
+
 	if err = fpga.SetupReconciler(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FpgaDevicePlugin")
 		os.Exit(1)

deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml

@@ -0,0 +1,151 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.3.0
+  creationTimestamp: null
+  name: sgxdeviceplugins.deviceplugin.intel.com
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.desiredNumberScheduled
+    name: Desired
+    type: integer
+  - JSONPath: .status.numberReady
+    name: Ready
+    type: integer
+  - JSONPath: .spec.nodeSelector
+    name: Node Selector
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: deviceplugin.intel.com
+  names:
+    kind: SgxDevicePlugin
+    listKind: SgxDevicePluginList
+    plural: sgxdeviceplugins
+    singular: sgxdeviceplugin
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: SgxDevicePlugin is the Schema for the sgxdeviceplugins API.
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: SgxDevicePluginSpec defines the desired state of SgxDevicePlugin.
+          properties:
+            enclaveLimit:
+              description: EnclaveLimit is a number of containers that can share the
+                same SGX enclave device.
+              minimum: 1
+              type: integer
+            image:
+              description: Image is a container image with SGX device plugin executable.
+              type: string
+            initImage:
+              description: InitImage is a container image with tools (e.g., SGX NFD
+                source hook) installed on each node.
+              type: string
+            logLevel:
+              description: LogLevel sets the plugin's log level.
+              minimum: 0
+              type: integer
+            nodeSelector:
+              additionalProperties:
+                type: string
+              description: NodeSelector provides a simple way to constrain device
+                plugin pods to nodes with particular labels.
+              type: object
+            provisionLimit:
+              description: ProvisionLimit is a number of containers that can share
+                the same SGX provision device.
+              minimum: 1
+              type: integer
+          type: object
+        status:
+          description: SgxDevicePluginStatus defines the observed state of SgxDevicePlugin.
+          properties:
+            controlledDaemonSet:
+              description: ControlledDaemoSet references the DaemonSet controlled
+                by the operator.
+              properties:
+                apiVersion:
+                  description: API version of the referent.
+                  type: string
+                fieldPath:
+                  description: 'If referring to a piece of an object instead of an
+                    entire object, this string should contain a valid JSON/Go field
+                    access statement, such as desiredState.manifest.containers[2].
+                    For example, if the object reference is to a container within
+                    a pod, this would take on a value like: "spec.containers{name}"
+                    (where "name" refers to the name of the container that triggered
+                    the event) or if no container name is specified "spec.containers[2]"
+                    (container with index 2 in this pod). This syntax is chosen only
+                    to have some well-defined way of referencing a part of an object.
+                    TODO: this design is not final and this field is subject to change
+                    in the future.'
+                  type: string
+                kind:
+                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                namespace:
+                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                  type: string
+                resourceVersion:
+                  description: 'Specific resourceVersion to which this reference is
+                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                  type: string
+                uid:
+                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                  type: string
+              type: object
+            desiredNumberScheduled:
+              description: The total number of nodes that should be running the device
+                plugin pod (including nodes correctly running the device plugin pod).
+              format: int32
+              type: integer
+            nodeNames:
+              description: The list of Node names where the device plugin pods are
+                running.
+              items:
+                type: string
+              type: array
+            numberReady:
+              description: The number of nodes that should be running the device plugin
+                pod and have one or more of the device plugin pod running and ready.
+              format: int32
+              type: integer
+          required:
+          - desiredNumberScheduled
+          - numberReady
+          type: object
+      type: object
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

deployments/operator/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/deviceplugin.intel.com_gpudeviceplugins.yaml
 - bases/deviceplugin.intel.com_qatdeviceplugins.yaml
 - bases/deviceplugin.intel.com_fpgadeviceplugins.yaml
+- bases/deviceplugin.intel.com_sgxdeviceplugins.yaml
 - bases/fpga.intel.com_acceleratorfunctions.yaml
 - bases/fpga.intel.com_fpgaregions.yaml
 # +kubebuilder:scaffold:crdkustomizeresource

deployments/operator/rbac/role.yaml

@@ -86,6 +86,26 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - deviceplugin.intel.com
+  resources:
+  - sgxdeviceplugins
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - deviceplugin.intel.com
+  resources:
+  - sgxdeviceplugins/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - fpga.intel.com
   resources:

deployments/operator/webhook/manifests.yaml

@@ -60,6 +60,25 @@ webhooks:
     - UPDATE
     resources:
     - qatdeviceplugins
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-deviceplugin-intel-com-v1-sgxdeviceplugin
+  failurePolicy: Fail
+  name: msgxdeviceplugin.kb.io
+  rules:
+  - apiGroups:
+    - deviceplugin.intel.com
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - sgxdeviceplugins
+  sideEffects: None
 - clientConfig:
     caBundle: Cg==
     service:
@@ -78,6 +97,25 @@ webhooks:
     - UPDATE
     resources:
     - pods
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /pods-sgx
+  failurePolicy: Ignore
+  name: sgx.mutator.webhooks.intel.com
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: NoneOnDryRun
 
 ---
 apiVersion: admissionregistration.k8s.io/v1beta1
@@ -140,3 +178,22 @@ webhooks:
     - UPDATE
     resources:
     - qatdeviceplugins
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-deviceplugin-intel-com-v1-sgxdeviceplugin
+  failurePolicy: Fail
+  name: vsgxdeviceplugin.kb.io
+  rules:
+  - apiGroups:
+    - deviceplugin.intel.com
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - sgxdeviceplugins
+  sideEffects: NoneOnDryRun

pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go

@@ -0,0 +1,99 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// SgxDevicePluginSpec defines the desired state of SgxDevicePlugin.
+type SgxDevicePluginSpec struct {
+	// Important: Run "make generate" to regenerate code after modifying this file
+
+	// Image is a container image with SGX device plugin executable.
+	Image string `json:"image,omitempty"`
+
+	// InitImage is a container image with tools (e.g., SGX NFD source hook) installed on each node.
+	InitImage string `json:"initImage,omitempty"`
+
+	// EnclaveLimit is a number of containers that can share the same SGX enclave device.
+	// +kubebuilder:validation:Minimum=1
+	EnclaveLimit int `json:"enclaveLimit,omitempty"`
+
+	// ProvisionLimit is a number of containers that can share the same SGX provision device.
+	// +kubebuilder:validation:Minimum=1
+	ProvisionLimit int `json:"provisionLimit,omitempty"`
+
+	// LogLevel sets the plugin's log level.
+	// +kubebuilder:validation:Minimum=0
+	LogLevel int `json:"logLevel,omitempty"`
+
+	// NodeSelector provides a simple way to constrain device plugin pods to nodes with particular labels.
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+}
+
+// SgxDevicePluginStatus defines the observed state of SgxDevicePlugin.
+type SgxDevicePluginStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make generate" to regenerate code after modifying this file
+
+	// ControlledDaemoSet references the DaemonSet controlled by the operator.
+	// +optional
+	ControlledDaemonSet v1.ObjectReference `json:"controlledDaemonSet,omitempty"`
+
+	// The total number of nodes that should be running the device plugin
+	// pod (including nodes correctly running the device plugin pod).
+	DesiredNumberScheduled int32 `json:"desiredNumberScheduled"`
+
+	// The number of nodes that should be running the device plugin pod and have one
+	// or more of the device plugin pod running and ready.
+	NumberReady int32 `json:"numberReady"`
+
+	// The list of Node names where the device plugin pods are running.
+	// +optional
+	NodeNames []string `json:"nodeNames,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.status.desiredNumberScheduled`
+// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.numberReady`
+// +kubebuilder:printcolumn:name="Node Selector",type=string,JSONPath=`.spec.nodeSelector`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+
+// SgxDevicePlugin is the Schema for the sgxdeviceplugins API.
+type SgxDevicePlugin struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   SgxDevicePluginSpec   `json:"spec,omitempty"`
+	Status SgxDevicePluginStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// SgxDevicePluginList contains a list of SgxDevicePlugin.
+type SgxDevicePluginList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []SgxDevicePlugin `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&SgxDevicePlugin{}, &SgxDevicePluginList{})
+}

pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go

@@ -0,0 +1,98 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/version"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
+)
+
+const (
+	sgxPluginKind = "SgxDevicePlugin"
+)
+
+var (
+	// sgxdevicepluginlog is for logging in this package.
+	sgxdevicepluginlog = logf.Log.WithName("sgxdeviceplugin-resource")
+
+	sgxMinVersion = version.MustParseSemantic("0.19.0")
+)
+
+// SetupWebhookWithManager sets up a webhook for SgxDevicePlugin custom resources.
+func (r *SgxDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/mutate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=true,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=create;update,versions=v1,name=msgxdeviceplugin.kb.io,sideEffects=None
+
+var _ webhook.Defaulter = &SgxDevicePlugin{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type.
+func (r *SgxDevicePlugin) Default() {
+	sgxdevicepluginlog.Info("default", "name", r.Name)
+
+	if len(r.Spec.Image) == 0 {
+		r.Spec.Image = "intel/intel-sgx-plugin:0.19.0"
+	}
+
+	if len(r.Spec.InitImage) == 0 {
+		r.Spec.Image = "intel/intel-sgx-initcontainer:0.19.0"
+	}
+}
+
+// +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,versions=v1,name=vsgxdeviceplugin.kb.io,sideEffects=NoneOnDryRun
+
+var _ webhook.Validator = &SgxDevicePlugin{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
+func (r *SgxDevicePlugin) ValidateCreate() error {
+	sgxdevicepluginlog.Info("validate create", "name", r.Name)
+
+	if controllers.GetDevicePluginCount(sgxPluginKind) > 0 {
+		return errors.Errorf("an instance of %q already exists in the cluster", sgxPluginKind)
+	}
+
+	return r.validatePlugin()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
+func (r *SgxDevicePlugin) ValidateUpdate(old runtime.Object) error {
+	sgxdevicepluginlog.Info("validate update", "name", r.Name)
+
+	return r.validatePlugin()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
+func (r *SgxDevicePlugin) ValidateDelete() error {
+	sgxdevicepluginlog.Info("validate delete", "name", r.Name)
+
+	return nil
+}
+
+func (r *SgxDevicePlugin) validatePlugin() error {
+	if err := validatePluginImage(r.Spec.Image, "intel-sgx-plugin", sgxMinVersion); err != nil {
+		return err
+	}
+
+	return validatePluginImage(r.Spec.InitImage, "intel-sgx-initcontainer", sgxMinVersion)
+}

pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go

@@ -332,3 +332,105 @@ func (in *QatDevicePluginStatus) DeepCopy() *QatDevicePluginStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SgxDevicePlugin) DeepCopyInto(out *SgxDevicePlugin) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePlugin.
+func (in *SgxDevicePlugin) DeepCopy() *SgxDevicePlugin {
+	if in == nil {
+		return nil
+	}
+	out := new(SgxDevicePlugin)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *SgxDevicePlugin) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SgxDevicePluginList) DeepCopyInto(out *SgxDevicePluginList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]SgxDevicePlugin, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginList.
+func (in *SgxDevicePluginList) DeepCopy() *SgxDevicePluginList {
+	if in == nil {
+		return nil
+	}
+	out := new(SgxDevicePluginList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *SgxDevicePluginList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SgxDevicePluginSpec) DeepCopyInto(out *SgxDevicePluginSpec) {
+	*out = *in
+	if in.NodeSelector != nil {
+		in, out := &in.NodeSelector, &out.NodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginSpec.
+func (in *SgxDevicePluginSpec) DeepCopy() *SgxDevicePluginSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(SgxDevicePluginSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SgxDevicePluginStatus) DeepCopyInto(out *SgxDevicePluginStatus) {
+	*out = *in
+	out.ControlledDaemonSet = in.ControlledDaemonSet
+	if in.NodeNames != nil {
+		in, out := &in.NodeNames, &out.NodeNames
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginStatus.
+func (in *SgxDevicePluginStatus) DeepCopy() *SgxDevicePluginStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SgxDevicePluginStatus)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/controllers/sgx/controller.go

@@ -0,0 +1,253 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sgx contains SGX specific reconciliation logic.
+package sgx
+
+import (
+	"context"
+	"reflect"
+	"strconv"
+	"strings"
+
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/reference"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1"
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
+	"github.com/pkg/errors"
+)
+
+const (
+	ownerKey = ".metadata.controller.sgx"
+	appLabel = "intel-sgx-plugin"
+)
+
+// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins/status,verbs=get;update;patch
+
+// SetupReconciler creates a new reconciler for SgxDevicePlugin objects.
+func SetupReconciler(mgr ctrl.Manager) error {
+	c := &controller{scheme: mgr.GetScheme()}
+	return controllers.SetupWithManager(mgr, c, devicepluginv1.GroupVersion.String(), "SgxDevicePlugin", ownerKey)
+}
+
+type controller struct {
+	scheme *runtime.Scheme
+}
+
+func (c *controller) CreateEmptyObject() runtime.Object {
+	return &devicepluginv1.SgxDevicePlugin{}
+}
+
+func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
+	var list devicepluginv1.SgxDevicePluginList
+	if err := clnt.List(ctx, &list); err != nil {
+		return 0, err
+	}
+
+	return len(list.Items), nil
+}
+
+func (c *controller) NewDaemonSet(rawObj runtime.Object) *apps.DaemonSet {
+	devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
+
+	var nodeSelector map[string]string
+	dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
+	if dpNodeSelectorSize > 0 {
+		nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
+		for k, v := range devicePlugin.Spec.NodeSelector {
+			nodeSelector[k] = v
+		}
+		nodeSelector["kubernetes.io/arch"] = "amd64"
+	} else {
+		nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
+	}
+
+	yes := true
+	directoryOrCreate := v1.HostPathDirectoryOrCreate
+	return &apps.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:    devicePlugin.Namespace,
+			GenerateName: devicePlugin.Name + "-",
+			Labels: map[string]string{
+				"app": appLabel,
+			},
+		},
+		Spec: apps.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": appLabel,
+				},
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": appLabel,
+					},
+				},
+				Spec: v1.PodSpec{
+					InitContainers: []v1.Container{
+						{
+							Image:           devicePlugin.Spec.InitImage,
+							ImagePullPolicy: "IfNotPresent",
+							Name:            "intel-sgx-initcontainer",
+							SecurityContext: &v1.SecurityContext{
+								ReadOnlyRootFilesystem: &yes,
+							},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									MountPath: "/etc/kubernetes/node-feature-discovery/source.d/",
+									Name:      "nfd-source-hooks",
+								},
+							},
+						},
+					},
+					Containers: []v1.Container{
+						{
+							Name:            appLabel,
+							Args:            getPodArgs(devicePlugin),
+							Image:           devicePlugin.Spec.Image,
+							ImagePullPolicy: "IfNotPresent",
+							SecurityContext: &v1.SecurityContext{
+								ReadOnlyRootFilesystem: &yes,
+							},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "sgxdevices",
+									MountPath: "/dev/sgx",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "kubeletsockets",
+									MountPath: "/var/lib/kubelet/device-plugins",
+								},
+							},
+						},
+					},
+					NodeSelector: nodeSelector,
+					Volumes: []v1.Volume{
+						{
+							Name: "sgxdevices",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev/sgx",
+								},
+							},
+						},
+						{
+							Name: "kubeletsockets",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/var/lib/kubelet/device-plugins",
+								},
+							},
+						},
+						{
+							Name: "nfd-source-hooks",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/etc/kubernetes/node-feature-discovery/source.d/",
+									Type: &directoryOrCreate,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func (c *controller) UpdateDaemonSet(rawObj runtime.Object, ds *apps.DaemonSet) (updated bool) {
+	dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
+
+	if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image {
+		ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image
+		updated = true
+	}
+
+	if dp.Spec.NodeSelector == nil {
+		dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
+	} else {
+		dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64"
+	}
+	if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
+		ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
+		updated = true
+	}
+
+	newargs := getPodArgs(dp)
+	if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {
+		ds.Spec.Template.Spec.Containers[0].Args = newargs
+		updated = true
+	}
+
+	return updated
+}
+
+func (c *controller) UpdateStatus(rawObj runtime.Object, ds *apps.DaemonSet, nodeNames []string) (updated bool, err error) {
+	dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
+
+	dsRef, err := reference.GetReference(c.scheme, ds)
+	if err != nil {
+		return false, errors.Wrap(err, "unable to make reference to controlled daemon set")
+	}
+
+	if dp.Status.ControlledDaemonSet.UID != dsRef.UID {
+		dp.Status.ControlledDaemonSet = *dsRef
+		updated = true
+	}
+
+	if dp.Status.DesiredNumberScheduled != ds.Status.DesiredNumberScheduled {
+		dp.Status.DesiredNumberScheduled = ds.Status.DesiredNumberScheduled
+		updated = true
+	}
+
+	if dp.Status.NumberReady != ds.Status.NumberReady {
+		dp.Status.NumberReady = ds.Status.NumberReady
+		updated = true
+	}
+
+	if strings.Join(dp.Status.NodeNames, ",") != strings.Join(nodeNames, ",") {
+		dp.Status.NodeNames = nodeNames
+		updated = true
+	}
+
+	return updated, nil
+}
+
+func getPodArgs(sdp *devicepluginv1.SgxDevicePlugin) []string {
+	args := make([]string, 0, 4)
+	args = append(args, "-v", strconv.Itoa(sdp.Spec.LogLevel))
+
+	if sdp.Spec.EnclaveLimit > 0 {
+		args = append(args, "-enclave-limit", strconv.Itoa(sdp.Spec.EnclaveLimit))
+	} else {
+		args = append(args, "-enclave-limit", "1")
+	}
+
+	if sdp.Spec.ProvisionLimit > 0 {
+		args = append(args, "-provision-limit", strconv.Itoa(sdp.Spec.ProvisionLimit))
+	} else {
+		args = append(args, "-provision-limit", "1")
+	}
+
+	return args
+}

pkg/fpgacontroller/patcher/patcher.go

@@ -29,6 +29,7 @@ import (
 
 	fpgav2 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/fpga.intel.com/v2"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga"
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/internal/containers"
 )
 
 const (
@@ -133,57 +134,21 @@ func (p *patcher) RemoveRegion(name string) {
 	delete(p.resourceModeMap, namespace+"/"+name)
 }
 
-// getRequestedResources validates the container's requirements first, then returns them as a map.
+func validateContainer(container corev1.Container) error {
-func getRequestedResources(container corev1.Container) (map[string]int64, error) {
 	for _, v := range container.Env {
 		if strings.HasPrefix(v.Name, "FPGA_REGION") || strings.HasPrefix(v.Name, "FPGA_AFU") {
-			return nil, errors.Errorf("environment variable '%s' is not allowed", v.Name)
+			return errors.Errorf("environment variable '%s' is not allowed", v.Name)
 		}
 	}
-
+	return nil
-	// Container may happen to have Requests, but not Limits. Check Requests first,
-	// then in the next loop iterate over Limits.
-	for resourceName, resourceQuantity := range container.Resources.Requests {
-		rname := strings.ToLower(string(resourceName))
-		if !strings.HasPrefix(rname, namespace) {
-			// Skip non-FPGA resources in Requests.
-			continue
-		}
-
-		if container.Resources.Limits[resourceName] != resourceQuantity {
-			return nil, errors.Errorf(
-				"'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted",
-				rname)
-		}
-	}
-
-	resources := make(map[string]int64)
-	for resourceName, resourceQuantity := range container.Resources.Limits {
-		rname := strings.ToLower(string(resourceName))
-		if !strings.HasPrefix(rname, namespace) {
-			// Skip non-FPGA resources in Limits.
-			continue
-		}
-
-		if container.Resources.Requests[resourceName] != resourceQuantity {
-			return nil, errors.Errorf(
-				"'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted",
-				rname)
-		}
-
-		quantity, ok := resourceQuantity.AsInt64()
-		if !ok {
-			return nil, errors.Errorf("resource quantity isn't of integral type for %q", rname)
-		}
-
-		resources[rname] = quantity
-	}
-
-	return resources, nil
 }
 
 func (p *patcher) getPatchOps(containerIdx int, container corev1.Container) ([]string, error) {
-	requestedResources, err := getRequestedResources(container)
+	if err := validateContainer(container); err != nil {
+		return nil, err
+	}
+
+	requestedResources, err := containers.GetRequestedResources(container, namespace)
 	if err != nil {
 		return nil, err
 	}
@@ -203,14 +168,11 @@ func (p *patcher) getPatchOps(containerIdx int, container corev1.Container) ([]s
 		}
 
 		switch mode {
-		case regiondevel:
+		case regiondevel, af:
 			// Do nothing.
-			// The requested resources are exposed by FPGA plugins working in "regiondevel" mode.
+			// The requested resources are exposed by FPGA plugins working in "regiondevel/af" mode.
-			// In this mode the workload is supposed to program FPGA regions.
+			// In "regiondevel" mode the workload is supposed to program FPGA regions.
 			// A cluster admin has to add FpgaRegion CRDs to allow this.
-		case af:
-			// Do nothing.
-			// The requested resources are exposed by FPGA plugins working in "af" mode.
 		case region:
 			// Let fpga_crihook know how to program the regions by setting ENV variables.
 			// The requested resources are exposed by FPGA plugins working in "region" mode.

pkg/fpgacontroller/patcher/patcher_test.go

@@ -86,6 +86,71 @@ func TestPatcherStorageFunctions(t *testing.T) {
 	}
 }
 
+func TestValidateContainerEnv(t *testing.T) {
+	tcases := []struct {
+		name        string
+		container   corev1.Container
+		expectedErr bool
+	}{
+		{
+			name: "Container OK",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"cpu": resource.MustParse("1"),
+					},
+				},
+			},
+			expectedErr: false,
+		},
+		{
+			name: "Wrong ENV FPGA_AFU",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"cpu": resource.MustParse("1"),
+					},
+				},
+				Env: []corev1.EnvVar{
+					{
+						Name:  "FPGA_AFU",
+						Value: "fake value",
+					},
+				},
+			},
+			expectedErr: true,
+		},
+		{
+			name: "Wrong ENV FPGA_REGION",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"cpu": resource.MustParse("1"),
+					},
+				},
+				Env: []corev1.EnvVar{
+					{
+						Name:  "FPGA_REGION",
+						Value: "fake value",
+					},
+				},
+			},
+			expectedErr: true,
+		},
+	}
+	for _, tt := range tcases {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateContainer(tt.container)
+			if tt.expectedErr && err == nil {
+				t.Errorf("Test case '%s': no error returned", tt.name)
+			}
+			if !tt.expectedErr && err != nil {
+				t.Errorf("Test case '%s': unexpected error: %+v", tt.name, err)
+			}
+		})
+	}
+}
+
 func TestGetPatchOps(t *testing.T) {
 	tcases := []struct {
 		name        string
@@ -229,23 +294,6 @@ func TestGetPatchOps(t *testing.T) {
 			},
 			expectedErr: true,
 		},
-		{
-			name: "Wrong ENV",
-			container: corev1.Container{
-				Resources: corev1.ResourceRequirements{
-					Limits: corev1.ResourceList{
-						"cpu": resource.MustParse("1"),
-					},
-				},
-				Env: []corev1.EnvVar{
-					{
-						Name:  "FPGA_REGION",
-						Value: "fake value",
-					},
-				},
-			},
-			expectedErr: true,
-		},
 		{
 			name: "Wrong type of quantity",
 			container: corev1.Container{

pkg/internal/containers/containers.go

@@ -0,0 +1,64 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package containers
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// GetRequestedResources validates the container's requirements first, then returns them as a map.
+func GetRequestedResources(container corev1.Container, ns string) (map[string]int64, error) {
+	// Container may happen to have Requests, but not Limits. Check Requests first,
+	// then in the next loop iterate over Limits.
+	for resourceName, resourceQuantity := range container.Resources.Requests {
+		rname := strings.ToLower(string(resourceName))
+		if !strings.HasPrefix(rname, ns) {
+			continue
+		}
+
+		if container.Resources.Limits[resourceName] != resourceQuantity {
+			return nil, errors.Errorf(
+				"'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted",
+				rname)
+		}
+	}
+
+	resources := make(map[string]int64)
+	for resourceName, resourceQuantity := range container.Resources.Limits {
+		rname := strings.ToLower(string(resourceName))
+		if !strings.HasPrefix(rname, ns) {
+			continue
+		}
+
+		if container.Resources.Requests[resourceName] != resourceQuantity {
+			return nil, errors.Errorf(
+				"'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted",
+				rname)
+		}
+
+		quantity, ok := resourceQuantity.AsInt64()
+		if !ok {
+			return nil, errors.Errorf("resource quantity isn't of integral type for %q", rname)
+		}
+
+		resources[rname] = quantity
+	}
+
+	return resources, nil
+}

pkg/internal/containers/containers_test.go

@@ -0,0 +1,134 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package containers
+
+import (
+	"flag"
+	"reflect"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+func init() {
+	_ = flag.Set("v", "4")
+}
+
+func TestGetRequestedResources(t *testing.T) {
+	tcases := []struct {
+		name           string
+		namespace      string
+		container      corev1.Container
+		expectedErr    bool
+		expectedResult map[string]int64
+	}{
+		{
+			name:      "Normal case",
+			namespace: "device.intel.com",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"device.intel.com/type":  resource.MustParse("1"),
+						"device.intel.com/type2": resource.MustParse("2"),
+						"cpu":                    resource.MustParse("1"),
+					},
+					Requests: corev1.ResourceList{
+						"device.intel.com/type":  resource.MustParse("1"),
+						"device.intel.com/type2": resource.MustParse("2"),
+						"cpu":                    resource.MustParse("3"),
+					},
+				},
+			},
+			expectedResult: map[string]int64{
+				"device.intel.com/type":  1,
+				"device.intel.com/type2": 2,
+			},
+		},
+		{
+			name:      "Unmatched device",
+			namespace: "device2.intel.com",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"device.intel.com/type":  resource.MustParse("1"),
+						"device.intel.com/type2": resource.MustParse("2"),
+						"cpu":                    resource.MustParse("1"),
+					},
+					Requests: corev1.ResourceList{
+						"device.intel.com/type": resource.MustParse("1"),
+						"device.intel.com/typ2": resource.MustParse("2"),
+						"cpu":                   resource.MustParse("3"),
+					},
+				},
+			},
+			expectedResult: map[string]int64{},
+		},
+		{
+			name:      "Unequal device resources in Limits and Requests 1",
+			namespace: "device.intel.com",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"device.intel.com/type": resource.MustParse("1"),
+					},
+				},
+			},
+			expectedErr: true,
+		},
+		{
+			name:      "Unequal device resources in Limits and Requests 2",
+			namespace: "device.intel.com",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						"device.intel.com/type": resource.MustParse("1"),
+					},
+				},
+			},
+			expectedErr: true,
+		},
+		{
+			name:      "Wrong type of quantity",
+			namespace: "device.intel.com",
+			container: corev1.Container{
+				Resources: corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{
+						"device.intel.com/type": resource.MustParse("1.1"),
+					},
+					Requests: corev1.ResourceList{
+						"device.intel.com/type": resource.MustParse("1.1"),
+					},
+				},
+			},
+			expectedErr: true,
+		},
+	}
+
+	for _, tt := range tcases {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := GetRequestedResources(tt.container, tt.namespace)
+			if tt.expectedErr && err == nil {
+				t.Errorf("Test case '%s': no error returned", tt.name)
+			}
+			if !tt.expectedErr && err != nil {
+				t.Errorf("Test case '%s': unexpected error: %+v", tt.name, err)
+			}
+			if !reflect.DeepEqual(result, tt.expectedResult) {
+				t.Errorf("test case '%s': result %+v does not match expected %+v\n", tt.name, result, tt.expectedResult)
+			}
+		})
+	}
+}

pkg/webhooks/sgx/sgx.go

@@ -0,0 +1,94 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sgx
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/internal/containers"
+)
+
+// +kubebuilder:webhook:path=/pods-sgx,mutating=true,failurePolicy=ignore,groups="",resources=pods,verbs=create;update,versions=v1,name=sgx.mutator.webhooks.intel.com,sideEffects=None
+
+// SgxMutator annotates Pods.
+type SgxMutator struct {
+	Client  client.Client
+	decoder *admission.Decoder
+}
+
+const (
+	namespace           = "sgx.intel.com"
+	encl                = namespace + "/enclave"
+	epc                 = namespace + "/epc"
+	provision           = namespace + "/provision"
+	provisionAnnotation = namespace + "/needs-provision"
+)
+
+func (s *SgxMutator) Handle(ctx context.Context, req admission.Request) admission.Response {
+	pod := &corev1.Pod{}
+
+	if err := s.decoder.Decode(req, pod); err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	totalEpc := int64(0)
+
+	if pod.Annotations == nil {
+		pod.Annotations = map[string]string{}
+	}
+	for idx, container := range pod.Spec.Containers {
+		requestedResources, err := containers.GetRequestedResources(container, namespace)
+		if err != nil {
+			return admission.Errored(http.StatusInternalServerError, err)
+		}
+		if epcSize, ok := requestedResources[epc]; ok {
+			totalEpc += epcSize
+
+			attestation, found := pod.Annotations[provisionAnnotation]
+			if found && attestation == "yes" {
+				pod.Spec.Containers[idx].Resources.Limits[corev1.ResourceName(provision)] = resource.MustParse("1")
+				pod.Spec.Containers[idx].Resources.Requests[corev1.ResourceName(provision)] = resource.MustParse("1")
+			}
+			pod.Spec.Containers[idx].Resources.Limits[corev1.ResourceName(encl)] = resource.MustParse("1")
+			pod.Spec.Containers[idx].Resources.Requests[corev1.ResourceName(encl)] = resource.MustParse("1")
+		}
+	}
+
+	if totalEpc != 0 {
+		quantity := resource.NewQuantity(totalEpc, resource.BinarySI)
+		pod.Annotations["sgx.intel.com/epc"] = quantity.String()
+	}
+
+	marshaledPod, err := json.Marshal(pod)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
+}
+
+// SgxMutator implements admission.DecoderInjector.
+// A decoder will be automatically injected.
+func (s *SgxMutator) InjectDecoder(d *admission.Decoder) error {
+	s.decoder = d
+	return nil
+}

test/envtest/sgxdeviceplugin_controller_test.go

@@ -0,0 +1,88 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envtest
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1"
+)
+
+var _ = Describe("SgxDevicePlugin Controller", func() {
+
+	const timeout = time.Second * 30
+	const interval = time.Second * 1
+
+	Context("Basic CRUD operations", func() {
+		It("should handle SgxDevicePlugin objects correctly", func() {
+			spec := devicepluginv1.SgxDevicePluginSpec{
+				Image:     "sgx-testimage",
+				InitImage: "sgx-testinitimage",
+			}
+
+			key := types.NamespacedName{
+				Name:      "sgxdeviceplugin-test",
+				Namespace: "default",
+			}
+
+			toCreate := &devicepluginv1.SgxDevicePlugin{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      key.Name,
+					Namespace: key.Namespace,
+				},
+				Spec: spec,
+			}
+
+			By("creating SgxDevicePlugin successfully")
+			Expect(k8sClient.Create(context.Background(), toCreate)).Should(Succeed())
+			time.Sleep(time.Second * 5)
+
+			fetched := &devicepluginv1.SgxDevicePlugin{}
+			Eventually(func() bool {
+				_ = k8sClient.Get(context.Background(), key, fetched)
+				return len(fetched.Status.ControlledDaemonSet.UID) > 0
+			}, timeout, interval).Should(BeTrue())
+
+			By("updating image name successfully")
+			updatedImage := "updated-sgx-testimage"
+			fetched.Spec.Image = updatedImage
+
+			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
+			fetchedUpdated := &devicepluginv1.SgxDevicePlugin{}
+			Eventually(func() string {
+				_ = k8sClient.Get(context.Background(), key, fetchedUpdated)
+				return fetchedUpdated.Spec.Image
+			}, timeout, interval).Should(Equal(updatedImage))
+
+			By("deleting SgxDevicePlugin successfully")
+			Eventually(func() error {
+				f := &devicepluginv1.SgxDevicePlugin{}
+				_ = k8sClient.Get(context.Background(), key, f)
+				return k8sClient.Delete(context.Background(), f)
+			}, timeout, interval).Should(Succeed())
+
+			Eventually(func() error {
+				f := &devicepluginv1.SgxDevicePlugin{}
+				return k8sClient.Get(context.Background(), key, f)
+			}, timeout, interval).ShouldNot(Succeed())
+		})
+	})
+})

test/envtest/suite_test.go

@@ -33,6 +33,7 @@ import (
 	fpgactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga"
 	gpuctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu"
 	qatctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat"
+	sgxctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/sgx"
 )
 
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -76,6 +77,8 @@ var _ = BeforeSuite(func(done Done) {
 
 	err = gpuctr.SetupReconciler(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
+	err = sgxctr.SetupReconciler(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
 	err = qatctr.SetupReconciler(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 	err = fpgactr.SetupReconciler(k8sManager)