diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 109c9244..357d2525 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -30,8 +30,10 @@ import ( "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/sgx" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpgacontroller" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpgacontroller/patcher" + sgxwebhook "github.com/intel/intel-device-plugins-for-kubernetes/pkg/webhooks/sgx" ) var ( @@ -90,12 +92,25 @@ func main() { os.Exit(1) } + if err = sgx.SetupReconciler(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SgxDevicePlugin") + os.Exit(1) + } + if err = (&devicepluginv1.SgxDevicePlugin{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "SgxDevicePlugin") + os.Exit(1) + } + pm := patcher.NewPatcherManager(mgr.GetLogger().WithName("webhooks").WithName("Fpga")) mgr.GetWebhookServer().Register("/pods", &webhook.Admission{ Handler: admission.HandlerFunc(pm.GetPodMutator()), }) + mgr.GetWebhookServer().Register("/pods-sgx", &webhook.Admission{ + Handler: &sgxwebhook.SgxMutator{Client: mgr.GetClient()}, + }) + if err = fpga.SetupReconciler(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "FpgaDevicePlugin") os.Exit(1) diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml new file mode 100644 index 00000000..622a2fa5 --- /dev/null +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml @@ -0,0 +1,151 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: sgxdeviceplugins.deviceplugin.intel.com +spec: + additionalPrinterColumns: + - JSONPath: .status.desiredNumberScheduled + name: Desired + type: integer + - JSONPath: .status.numberReady + name: Ready + type: integer + - JSONPath: .spec.nodeSelector + name: Node Selector + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: deviceplugin.intel.com + names: + kind: SgxDevicePlugin + listKind: SgxDevicePluginList + plural: sgxdeviceplugins + singular: sgxdeviceplugin + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: SgxDevicePlugin is the Schema for the sgxdeviceplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SgxDevicePluginSpec defines the desired state of SgxDevicePlugin. + properties: + enclaveLimit: + description: EnclaveLimit is a number of containers that can share the + same SGX enclave device. + minimum: 1 + type: integer + image: + description: Image is a container image with SGX device plugin executable. + type: string + initImage: + description: InitImage is a container image with tools (e.g., SGX NFD + source hook) installed on each node. + type: string + logLevel: + description: LogLevel sets the plugin's log level. + minimum: 0 + type: integer + nodeSelector: + additionalProperties: + type: string + description: NodeSelector provides a simple way to constrain device + plugin pods to nodes with particular labels. + type: object + provisionLimit: + description: ProvisionLimit is a number of containers that can share + the same SGX provision device. + minimum: 1 + type: integer + type: object + status: + description: SgxDevicePluginStatus defines the observed state of SgxDevicePlugin. + properties: + controlledDaemonSet: + description: ControlledDaemoSet references the DaemonSet controlled + by the operator. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + desiredNumberScheduled: + description: The total number of nodes that should be running the device + plugin pod (including nodes correctly running the device plugin pod). + format: int32 + type: integer + nodeNames: + description: The list of Node names where the device plugin pods are + running. + items: + type: string + type: array + numberReady: + description: The number of nodes that should be running the device plugin + pod and have one or more of the device plugin pod running and ready. + format: int32 + type: integer + required: + - desiredNumberScheduled + - numberReady + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deployments/operator/crd/kustomization.yaml b/deployments/operator/crd/kustomization.yaml index 46b3161c..3181a299 100644 --- a/deployments/operator/crd/kustomization.yaml +++ b/deployments/operator/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bases/deviceplugin.intel.com_gpudeviceplugins.yaml - bases/deviceplugin.intel.com_qatdeviceplugins.yaml - bases/deviceplugin.intel.com_fpgadeviceplugins.yaml +- bases/deviceplugin.intel.com_sgxdeviceplugins.yaml - bases/fpga.intel.com_acceleratorfunctions.yaml - bases/fpga.intel.com_fpgaregions.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/deployments/operator/rbac/role.yaml b/deployments/operator/rbac/role.yaml index 72db0dd2..a2b2aa71 100644 --- a/deployments/operator/rbac/role.yaml +++ b/deployments/operator/rbac/role.yaml @@ -86,6 +86,26 @@ rules: - get - patch - update +- apiGroups: + - deviceplugin.intel.com + resources: + - sgxdeviceplugins + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - deviceplugin.intel.com + resources: + - sgxdeviceplugins/status + verbs: + - get + - patch + - update - apiGroups: - fpga.intel.com resources: diff --git a/deployments/operator/webhook/manifests.yaml b/deployments/operator/webhook/manifests.yaml index 1e5475b1..f7fa2794 100644 --- a/deployments/operator/webhook/manifests.yaml +++ b/deployments/operator/webhook/manifests.yaml @@ -60,6 +60,25 @@ webhooks: - UPDATE resources: - qatdeviceplugins +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /mutate-deviceplugin-intel-com-v1-sgxdeviceplugin + failurePolicy: Fail + name: msgxdeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - sgxdeviceplugins + sideEffects: None - clientConfig: caBundle: Cg== service: @@ -78,6 +97,25 @@ webhooks: - UPDATE resources: - pods +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /pods-sgx + failurePolicy: Ignore + name: sgx.mutator.webhooks.intel.com + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: NoneOnDryRun --- apiVersion: admissionregistration.k8s.io/v1beta1 @@ -140,3 +178,22 @@ webhooks: - UPDATE resources: - qatdeviceplugins +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /validate-deviceplugin-intel-com-v1-sgxdeviceplugin + failurePolicy: Fail + name: vsgxdeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - sgxdeviceplugins + sideEffects: NoneOnDryRun diff --git a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go new file mode 100644 index 00000000..a68ced5d --- /dev/null +++ b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go @@ -0,0 +1,99 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// SgxDevicePluginSpec defines the desired state of SgxDevicePlugin. +type SgxDevicePluginSpec struct { + // Important: Run "make generate" to regenerate code after modifying this file + + // Image is a container image with SGX device plugin executable. + Image string `json:"image,omitempty"` + + // InitImage is a container image with tools (e.g., SGX NFD source hook) installed on each node. + InitImage string `json:"initImage,omitempty"` + + // EnclaveLimit is a number of containers that can share the same SGX enclave device. + // +kubebuilder:validation:Minimum=1 + EnclaveLimit int `json:"enclaveLimit,omitempty"` + + // ProvisionLimit is a number of containers that can share the same SGX provision device. + // +kubebuilder:validation:Minimum=1 + ProvisionLimit int `json:"provisionLimit,omitempty"` + + // LogLevel sets the plugin's log level. + // +kubebuilder:validation:Minimum=0 + LogLevel int `json:"logLevel,omitempty"` + + // NodeSelector provides a simple way to constrain device plugin pods to nodes with particular labels. + NodeSelector map[string]string `json:"nodeSelector,omitempty"` +} + +// SgxDevicePluginStatus defines the observed state of SgxDevicePlugin. +type SgxDevicePluginStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make generate" to regenerate code after modifying this file + + // ControlledDaemoSet references the DaemonSet controlled by the operator. + // +optional + ControlledDaemonSet v1.ObjectReference `json:"controlledDaemonSet,omitempty"` + + // The total number of nodes that should be running the device plugin + // pod (including nodes correctly running the device plugin pod). + DesiredNumberScheduled int32 `json:"desiredNumberScheduled"` + + // The number of nodes that should be running the device plugin pod and have one + // or more of the device plugin pod running and ready. + NumberReady int32 `json:"numberReady"` + + // The list of Node names where the device plugin pods are running. + // +optional + NodeNames []string `json:"nodeNames,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.status.desiredNumberScheduled` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.numberReady` +// +kubebuilder:printcolumn:name="Node Selector",type=string,JSONPath=`.spec.nodeSelector` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// SgxDevicePlugin is the Schema for the sgxdeviceplugins API. +type SgxDevicePlugin struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SgxDevicePluginSpec `json:"spec,omitempty"` + Status SgxDevicePluginStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SgxDevicePluginList contains a list of SgxDevicePlugin. +type SgxDevicePluginList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SgxDevicePlugin `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SgxDevicePlugin{}, &SgxDevicePluginList{}) +} diff --git a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go new file mode 100644 index 00000000..97202749 --- /dev/null +++ b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_webhook.go @@ -0,0 +1,98 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/version" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" +) + +const ( + sgxPluginKind = "SgxDevicePlugin" +) + +var ( + // sgxdevicepluginlog is for logging in this package. + sgxdevicepluginlog = logf.Log.WithName("sgxdeviceplugin-resource") + + sgxMinVersion = version.MustParseSemantic("0.19.0") +) + +// SetupWebhookWithManager sets up a webhook for SgxDevicePlugin custom resources. +func (r *SgxDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=true,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=create;update,versions=v1,name=msgxdeviceplugin.kb.io,sideEffects=None + +var _ webhook.Defaulter = &SgxDevicePlugin{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (r *SgxDevicePlugin) Default() { + sgxdevicepluginlog.Info("default", "name", r.Name) + + if len(r.Spec.Image) == 0 { + r.Spec.Image = "intel/intel-sgx-plugin:0.19.0" + } + + if len(r.Spec.InitImage) == 0 { + r.Spec.Image = "intel/intel-sgx-initcontainer:0.19.0" + } +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-sgxdeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=sgxdeviceplugins,versions=v1,name=vsgxdeviceplugin.kb.io,sideEffects=NoneOnDryRun + +var _ webhook.Validator = &SgxDevicePlugin{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *SgxDevicePlugin) ValidateCreate() error { + sgxdevicepluginlog.Info("validate create", "name", r.Name) + + if controllers.GetDevicePluginCount(sgxPluginKind) > 0 { + return errors.Errorf("an instance of %q already exists in the cluster", sgxPluginKind) + } + + return r.validatePlugin() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *SgxDevicePlugin) ValidateUpdate(old runtime.Object) error { + sgxdevicepluginlog.Info("validate update", "name", r.Name) + + return r.validatePlugin() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *SgxDevicePlugin) ValidateDelete() error { + sgxdevicepluginlog.Info("validate delete", "name", r.Name) + + return nil +} + +func (r *SgxDevicePlugin) validatePlugin() error { + if err := validatePluginImage(r.Spec.Image, "intel-sgx-plugin", sgxMinVersion); err != nil { + return err + } + + return validatePluginImage(r.Spec.InitImage, "intel-sgx-initcontainer", sgxMinVersion) +} diff --git a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go index ac22f4b8..e225c7f2 100644 --- a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go @@ -332,3 +332,105 @@ func (in *QatDevicePluginStatus) DeepCopy() *QatDevicePluginStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SgxDevicePlugin) DeepCopyInto(out *SgxDevicePlugin) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePlugin. +func (in *SgxDevicePlugin) DeepCopy() *SgxDevicePlugin { + if in == nil { + return nil + } + out := new(SgxDevicePlugin) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SgxDevicePlugin) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SgxDevicePluginList) DeepCopyInto(out *SgxDevicePluginList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SgxDevicePlugin, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginList. +func (in *SgxDevicePluginList) DeepCopy() *SgxDevicePluginList { + if in == nil { + return nil + } + out := new(SgxDevicePluginList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SgxDevicePluginList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SgxDevicePluginSpec) DeepCopyInto(out *SgxDevicePluginSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginSpec. +func (in *SgxDevicePluginSpec) DeepCopy() *SgxDevicePluginSpec { + if in == nil { + return nil + } + out := new(SgxDevicePluginSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SgxDevicePluginStatus) DeepCopyInto(out *SgxDevicePluginStatus) { + *out = *in + out.ControlledDaemonSet = in.ControlledDaemonSet + if in.NodeNames != nil { + in, out := &in.NodeNames, &out.NodeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SgxDevicePluginStatus. +func (in *SgxDevicePluginStatus) DeepCopy() *SgxDevicePluginStatus { + if in == nil { + return nil + } + out := new(SgxDevicePluginStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controllers/sgx/controller.go b/pkg/controllers/sgx/controller.go new file mode 100644 index 00000000..fd628816 --- /dev/null +++ b/pkg/controllers/sgx/controller.go @@ -0,0 +1,253 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sgx contains SGX specific reconciliation logic. +package sgx + +import ( + "context" + "reflect" + "strconv" + "strings" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/reference" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" + "github.com/pkg/errors" +) + +const ( + ownerKey = ".metadata.controller.sgx" + appLabel = "intel-sgx-plugin" +) + +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins/status,verbs=get;update;patch + +// SetupReconciler creates a new reconciler for SgxDevicePlugin objects. +func SetupReconciler(mgr ctrl.Manager) error { + c := &controller{scheme: mgr.GetScheme()} + return controllers.SetupWithManager(mgr, c, devicepluginv1.GroupVersion.String(), "SgxDevicePlugin", ownerKey) +} + +type controller struct { + scheme *runtime.Scheme +} + +func (c *controller) CreateEmptyObject() runtime.Object { + return &devicepluginv1.SgxDevicePlugin{} +} + +func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { + var list devicepluginv1.SgxDevicePluginList + if err := clnt.List(ctx, &list); err != nil { + return 0, err + } + + return len(list.Items), nil +} + +func (c *controller) NewDaemonSet(rawObj runtime.Object) *apps.DaemonSet { + devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin) + + var nodeSelector map[string]string + dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector) + if dpNodeSelectorSize > 0 { + nodeSelector = make(map[string]string, dpNodeSelectorSize+1) + for k, v := range devicePlugin.Spec.NodeSelector { + nodeSelector[k] = v + } + nodeSelector["kubernetes.io/arch"] = "amd64" + } else { + nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + } + + yes := true + directoryOrCreate := v1.HostPathDirectoryOrCreate + return &apps.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: devicePlugin.Namespace, + GenerateName: devicePlugin.Name + "-", + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": appLabel, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Image: devicePlugin.Spec.InitImage, + ImagePullPolicy: "IfNotPresent", + Name: "intel-sgx-initcontainer", + SecurityContext: &v1.SecurityContext{ + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/etc/kubernetes/node-feature-discovery/source.d/", + Name: "nfd-source-hooks", + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: appLabel, + Args: getPodArgs(devicePlugin), + Image: devicePlugin.Spec.Image, + ImagePullPolicy: "IfNotPresent", + SecurityContext: &v1.SecurityContext{ + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "sgxdevices", + MountPath: "/dev/sgx", + ReadOnly: true, + }, + { + Name: "kubeletsockets", + MountPath: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + NodeSelector: nodeSelector, + Volumes: []v1.Volume{ + { + Name: "sgxdevices", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/sgx", + }, + }, + }, + { + Name: "kubeletsockets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/device-plugins", + }, + }, + }, + { + Name: "nfd-source-hooks", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/node-feature-discovery/source.d/", + Type: &directoryOrCreate, + }, + }, + }, + }, + }, + }, + }, + } +} + +func (c *controller) UpdateDaemonSet(rawObj runtime.Object, ds *apps.DaemonSet) (updated bool) { + dp := rawObj.(*devicepluginv1.SgxDevicePlugin) + + if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image { + ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image + updated = true + } + + if dp.Spec.NodeSelector == nil { + dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + } else { + dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64" + } + if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { + ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector + updated = true + } + + newargs := getPodArgs(dp) + if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") { + ds.Spec.Template.Spec.Containers[0].Args = newargs + updated = true + } + + return updated +} + +func (c *controller) UpdateStatus(rawObj runtime.Object, ds *apps.DaemonSet, nodeNames []string) (updated bool, err error) { + dp := rawObj.(*devicepluginv1.SgxDevicePlugin) + + dsRef, err := reference.GetReference(c.scheme, ds) + if err != nil { + return false, errors.Wrap(err, "unable to make reference to controlled daemon set") + } + + if dp.Status.ControlledDaemonSet.UID != dsRef.UID { + dp.Status.ControlledDaemonSet = *dsRef + updated = true + } + + if dp.Status.DesiredNumberScheduled != ds.Status.DesiredNumberScheduled { + dp.Status.DesiredNumberScheduled = ds.Status.DesiredNumberScheduled + updated = true + } + + if dp.Status.NumberReady != ds.Status.NumberReady { + dp.Status.NumberReady = ds.Status.NumberReady + updated = true + } + + if strings.Join(dp.Status.NodeNames, ",") != strings.Join(nodeNames, ",") { + dp.Status.NodeNames = nodeNames + updated = true + } + + return updated, nil +} + +func getPodArgs(sdp *devicepluginv1.SgxDevicePlugin) []string { + args := make([]string, 0, 4) + args = append(args, "-v", strconv.Itoa(sdp.Spec.LogLevel)) + + if sdp.Spec.EnclaveLimit > 0 { + args = append(args, "-enclave-limit", strconv.Itoa(sdp.Spec.EnclaveLimit)) + } else { + args = append(args, "-enclave-limit", "1") + } + + if sdp.Spec.ProvisionLimit > 0 { + args = append(args, "-provision-limit", strconv.Itoa(sdp.Spec.ProvisionLimit)) + } else { + args = append(args, "-provision-limit", "1") + } + + return args +} diff --git a/pkg/fpgacontroller/patcher/patcher.go b/pkg/fpgacontroller/patcher/patcher.go index ba1d0f51..baaaee6f 100644 --- a/pkg/fpgacontroller/patcher/patcher.go +++ b/pkg/fpgacontroller/patcher/patcher.go @@ -29,6 +29,7 @@ import ( fpgav2 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/fpga.intel.com/v2" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/internal/containers" ) const ( @@ -133,57 +134,21 @@ func (p *patcher) RemoveRegion(name string) { delete(p.resourceModeMap, namespace+"/"+name) } -// getRequestedResources validates the container's requirements first, then returns them as a map. -func getRequestedResources(container corev1.Container) (map[string]int64, error) { +func validateContainer(container corev1.Container) error { for _, v := range container.Env { if strings.HasPrefix(v.Name, "FPGA_REGION") || strings.HasPrefix(v.Name, "FPGA_AFU") { - return nil, errors.Errorf("environment variable '%s' is not allowed", v.Name) + return errors.Errorf("environment variable '%s' is not allowed", v.Name) } } - - // Container may happen to have Requests, but not Limits. Check Requests first, - // then in the next loop iterate over Limits. - for resourceName, resourceQuantity := range container.Resources.Requests { - rname := strings.ToLower(string(resourceName)) - if !strings.HasPrefix(rname, namespace) { - // Skip non-FPGA resources in Requests. - continue - } - - if container.Resources.Limits[resourceName] != resourceQuantity { - return nil, errors.Errorf( - "'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted", - rname) - } - } - - resources := make(map[string]int64) - for resourceName, resourceQuantity := range container.Resources.Limits { - rname := strings.ToLower(string(resourceName)) - if !strings.HasPrefix(rname, namespace) { - // Skip non-FPGA resources in Limits. - continue - } - - if container.Resources.Requests[resourceName] != resourceQuantity { - return nil, errors.Errorf( - "'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted", - rname) - } - - quantity, ok := resourceQuantity.AsInt64() - if !ok { - return nil, errors.Errorf("resource quantity isn't of integral type for %q", rname) - } - - resources[rname] = quantity - } - - return resources, nil + return nil } func (p *patcher) getPatchOps(containerIdx int, container corev1.Container) ([]string, error) { - requestedResources, err := getRequestedResources(container) + if err := validateContainer(container); err != nil { + return nil, err + } + + requestedResources, err := containers.GetRequestedResources(container, namespace) if err != nil { return nil, err } @@ -203,14 +168,11 @@ func (p *patcher) getPatchOps(containerIdx int, container corev1.Container) ([]s } switch mode { - case regiondevel: + case regiondevel, af: // Do nothing. - // The requested resources are exposed by FPGA plugins working in "regiondevel" mode. - // In this mode the workload is supposed to program FPGA regions. + // The requested resources are exposed by FPGA plugins working in "regiondevel/af" mode. + // In "regiondevel" mode the workload is supposed to program FPGA regions. // A cluster admin has to add FpgaRegion CRDs to allow this. - case af: - // Do nothing. - // The requested resources are exposed by FPGA plugins working in "af" mode. case region: // Let fpga_crihook know how to program the regions by setting ENV variables. // The requested resources are exposed by FPGA plugins working in "region" mode. diff --git a/pkg/fpgacontroller/patcher/patcher_test.go b/pkg/fpgacontroller/patcher/patcher_test.go index 41b96c5a..bbc1c725 100644 --- a/pkg/fpgacontroller/patcher/patcher_test.go +++ b/pkg/fpgacontroller/patcher/patcher_test.go @@ -86,6 +86,71 @@ func TestPatcherStorageFunctions(t *testing.T) { } } +func TestValidateContainerEnv(t *testing.T) { + tcases := []struct { + name string + container corev1.Container + expectedErr bool + }{ + { + name: "Container OK", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + }, + expectedErr: false, + }, + { + name: "Wrong ENV FPGA_AFU", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + Env: []corev1.EnvVar{ + { + Name: "FPGA_AFU", + Value: "fake value", + }, + }, + }, + expectedErr: true, + }, + { + name: "Wrong ENV FPGA_REGION", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + Env: []corev1.EnvVar{ + { + Name: "FPGA_REGION", + Value: "fake value", + }, + }, + }, + expectedErr: true, + }, + } + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + err := validateContainer(tt.container) + if tt.expectedErr && err == nil { + t.Errorf("Test case '%s': no error returned", tt.name) + } + if !tt.expectedErr && err != nil { + t.Errorf("Test case '%s': unexpected error: %+v", tt.name, err) + } + }) + } +} + func TestGetPatchOps(t *testing.T) { tcases := []struct { name string @@ -229,23 +294,6 @@ func TestGetPatchOps(t *testing.T) { }, expectedErr: true, }, - { - name: "Wrong ENV", - container: corev1.Container{ - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("1"), - }, - }, - Env: []corev1.EnvVar{ - { - Name: "FPGA_REGION", - Value: "fake value", - }, - }, - }, - expectedErr: true, - }, { name: "Wrong type of quantity", container: corev1.Container{ diff --git a/pkg/internal/containers/containers.go b/pkg/internal/containers/containers.go new file mode 100644 index 00000000..21710c50 --- /dev/null +++ b/pkg/internal/containers/containers.go @@ -0,0 +1,64 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package containers + +import ( + "strings" + + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" +) + +// GetRequestedResources validates the container's requirements first, then returns them as a map. +func GetRequestedResources(container corev1.Container, ns string) (map[string]int64, error) { + // Container may happen to have Requests, but not Limits. Check Requests first, + // then in the next loop iterate over Limits. + for resourceName, resourceQuantity := range container.Resources.Requests { + rname := strings.ToLower(string(resourceName)) + if !strings.HasPrefix(rname, ns) { + continue + } + + if container.Resources.Limits[resourceName] != resourceQuantity { + return nil, errors.Errorf( + "'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted", + rname) + } + } + + resources := make(map[string]int64) + for resourceName, resourceQuantity := range container.Resources.Limits { + rname := strings.ToLower(string(resourceName)) + if !strings.HasPrefix(rname, ns) { + continue + } + + if container.Resources.Requests[resourceName] != resourceQuantity { + return nil, errors.Errorf( + "'limits' and 'requests' for %q must be equal as extended resources cannot be overcommitted", + rname) + } + + quantity, ok := resourceQuantity.AsInt64() + if !ok { + return nil, errors.Errorf("resource quantity isn't of integral type for %q", rname) + } + + resources[rname] = quantity + } + + return resources, nil +} diff --git a/pkg/internal/containers/containers_test.go b/pkg/internal/containers/containers_test.go new file mode 100644 index 00000000..b5f18601 --- /dev/null +++ b/pkg/internal/containers/containers_test.go @@ -0,0 +1,134 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package containers + +import ( + "flag" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func init() { + _ = flag.Set("v", "4") +} + +func TestGetRequestedResources(t *testing.T) { + tcases := []struct { + name string + namespace string + container corev1.Container + expectedErr bool + expectedResult map[string]int64 + }{ + { + name: "Normal case", + namespace: "device.intel.com", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + "device.intel.com/type2": resource.MustParse("2"), + "cpu": resource.MustParse("1"), + }, + Requests: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + "device.intel.com/type2": resource.MustParse("2"), + "cpu": resource.MustParse("3"), + }, + }, + }, + expectedResult: map[string]int64{ + "device.intel.com/type": 1, + "device.intel.com/type2": 2, + }, + }, + { + name: "Unmatched device", + namespace: "device2.intel.com", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + "device.intel.com/type2": resource.MustParse("2"), + "cpu": resource.MustParse("1"), + }, + Requests: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + "device.intel.com/typ2": resource.MustParse("2"), + "cpu": resource.MustParse("3"), + }, + }, + }, + expectedResult: map[string]int64{}, + }, + { + name: "Unequal device resources in Limits and Requests 1", + namespace: "device.intel.com", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + }, + }, + }, + expectedErr: true, + }, + { + name: "Unequal device resources in Limits and Requests 2", + namespace: "device.intel.com", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1"), + }, + }, + }, + expectedErr: true, + }, + { + name: "Wrong type of quantity", + namespace: "device.intel.com", + container: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1.1"), + }, + Requests: corev1.ResourceList{ + "device.intel.com/type": resource.MustParse("1.1"), + }, + }, + }, + expectedErr: true, + }, + } + + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + result, err := GetRequestedResources(tt.container, tt.namespace) + if tt.expectedErr && err == nil { + t.Errorf("Test case '%s': no error returned", tt.name) + } + if !tt.expectedErr && err != nil { + t.Errorf("Test case '%s': unexpected error: %+v", tt.name, err) + } + if !reflect.DeepEqual(result, tt.expectedResult) { + t.Errorf("test case '%s': result %+v does not match expected %+v\n", tt.name, result, tt.expectedResult) + } + }) + } +} diff --git a/pkg/webhooks/sgx/sgx.go b/pkg/webhooks/sgx/sgx.go new file mode 100644 index 00000000..d0cc2d2a --- /dev/null +++ b/pkg/webhooks/sgx/sgx.go @@ -0,0 +1,94 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sgx + +import ( + "context" + "encoding/json" + "net/http" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/internal/containers" +) + +// +kubebuilder:webhook:path=/pods-sgx,mutating=true,failurePolicy=ignore,groups="",resources=pods,verbs=create;update,versions=v1,name=sgx.mutator.webhooks.intel.com,sideEffects=None + +// SgxMutator annotates Pods. +type SgxMutator struct { + Client client.Client + decoder *admission.Decoder +} + +const ( + namespace = "sgx.intel.com" + encl = namespace + "/enclave" + epc = namespace + "/epc" + provision = namespace + "/provision" + provisionAnnotation = namespace + "/needs-provision" +) + +func (s *SgxMutator) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + + if err := s.decoder.Decode(req, pod); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + totalEpc := int64(0) + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + for idx, container := range pod.Spec.Containers { + requestedResources, err := containers.GetRequestedResources(container, namespace) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + if epcSize, ok := requestedResources[epc]; ok { + totalEpc += epcSize + + attestation, found := pod.Annotations[provisionAnnotation] + if found && attestation == "yes" { + pod.Spec.Containers[idx].Resources.Limits[corev1.ResourceName(provision)] = resource.MustParse("1") + pod.Spec.Containers[idx].Resources.Requests[corev1.ResourceName(provision)] = resource.MustParse("1") + } + pod.Spec.Containers[idx].Resources.Limits[corev1.ResourceName(encl)] = resource.MustParse("1") + pod.Spec.Containers[idx].Resources.Requests[corev1.ResourceName(encl)] = resource.MustParse("1") + } + } + + if totalEpc != 0 { + quantity := resource.NewQuantity(totalEpc, resource.BinarySI) + pod.Annotations["sgx.intel.com/epc"] = quantity.String() + } + + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +// SgxMutator implements admission.DecoderInjector. +// A decoder will be automatically injected. +func (s *SgxMutator) InjectDecoder(d *admission.Decoder) error { + s.decoder = d + return nil +} diff --git a/test/envtest/sgxdeviceplugin_controller_test.go b/test/envtest/sgxdeviceplugin_controller_test.go new file mode 100644 index 00000000..7e31d663 --- /dev/null +++ b/test/envtest/sgxdeviceplugin_controller_test.go @@ -0,0 +1,88 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envtest + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" +) + +var _ = Describe("SgxDevicePlugin Controller", func() { + + const timeout = time.Second * 30 + const interval = time.Second * 1 + + Context("Basic CRUD operations", func() { + It("should handle SgxDevicePlugin objects correctly", func() { + spec := devicepluginv1.SgxDevicePluginSpec{ + Image: "sgx-testimage", + InitImage: "sgx-testinitimage", + } + + key := types.NamespacedName{ + Name: "sgxdeviceplugin-test", + Namespace: "default", + } + + toCreate := &devicepluginv1.SgxDevicePlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Spec: spec, + } + + By("creating SgxDevicePlugin successfully") + Expect(k8sClient.Create(context.Background(), toCreate)).Should(Succeed()) + time.Sleep(time.Second * 5) + + fetched := &devicepluginv1.SgxDevicePlugin{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), key, fetched) + return len(fetched.Status.ControlledDaemonSet.UID) > 0 + }, timeout, interval).Should(BeTrue()) + + By("updating image name successfully") + updatedImage := "updated-sgx-testimage" + fetched.Spec.Image = updatedImage + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + fetchedUpdated := &devicepluginv1.SgxDevicePlugin{} + Eventually(func() string { + _ = k8sClient.Get(context.Background(), key, fetchedUpdated) + return fetchedUpdated.Spec.Image + }, timeout, interval).Should(Equal(updatedImage)) + + By("deleting SgxDevicePlugin successfully") + Eventually(func() error { + f := &devicepluginv1.SgxDevicePlugin{} + _ = k8sClient.Get(context.Background(), key, f) + return k8sClient.Delete(context.Background(), f) + }, timeout, interval).Should(Succeed()) + + Eventually(func() error { + f := &devicepluginv1.SgxDevicePlugin{} + return k8sClient.Get(context.Background(), key, f) + }, timeout, interval).ShouldNot(Succeed()) + }) + }) +}) diff --git a/test/envtest/suite_test.go b/test/envtest/suite_test.go index 49f5cfa3..e8ac4a05 100644 --- a/test/envtest/suite_test.go +++ b/test/envtest/suite_test.go @@ -33,6 +33,7 @@ import ( fpgactr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga" gpuctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu" qatctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat" + sgxctr "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/sgx" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -76,6 +77,8 @@ var _ = BeforeSuite(func(done Done) { err = gpuctr.SetupReconciler(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = sgxctr.SetupReconciler(k8sManager) + Expect(err).ToNot(HaveOccurred()) err = qatctr.SetupReconciler(k8sManager) Expect(err).ToNot(HaveOccurred()) err = fpgactr.SetupReconciler(k8sManager)