mirror of
https://github.com/intel/intel-device-plugins-for-kubernetes.git
synced 2025-06-03 03:59:37 +00:00
37618d4f85
The device plugins daemonsets are cluster wide and currently only one device plugin instance per device is possible so making the corresponding deviceplugin/v1 CRDs non-namespaced (i.e., scope: cluster) fits better. Previously, the device plugin daemonset was deployed in the same namespace as the CR for that device but with the cluster scoped CRDs we default to use the same namespace as the operator, unless overridden via DEVICEPLUGIN_NAMESPACE env variable or a command line parameter to operator manager deployment. Three additional changes in this commit: - enable DSA envtest tests - update controller-runtime to v0.8.1 - change device plugin envtest suite to use klog/v2 Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>
293 lines
8.1 KiB
Go
293 lines
8.1 KiB
Go
// Copyright 2020 Intel Corporation. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package sgx contains SGX specific reconciliation logic.
|
|
package sgx
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/client-go/tools/reference"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1"
|
|
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
const (
|
|
ownerKey = ".metadata.controller.sgx"
|
|
appLabel = "intel-sgx-plugin"
|
|
)
|
|
|
|
// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins,verbs=get;list;watch;create;update;patch;delete
|
|
// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=sgxdeviceplugins/status,verbs=get;update;patch
|
|
|
|
// SetupReconciler creates a new reconciler for SgxDevicePlugin objects.
|
|
func SetupReconciler(mgr ctrl.Manager, namespace string, withWebhook bool) error {
|
|
c := &controller{scheme: mgr.GetScheme(), ns: namespace}
|
|
if err := controllers.SetupWithManager(mgr, c, devicepluginv1.GroupVersion.String(), "SgxDevicePlugin", ownerKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if withWebhook {
|
|
return (&devicepluginv1.SgxDevicePlugin{}).SetupWebhookWithManager(mgr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type controller struct {
|
|
scheme *runtime.Scheme
|
|
ns string
|
|
}
|
|
|
|
func (c *controller) CreateEmptyObject() client.Object {
|
|
return &devicepluginv1.SgxDevicePlugin{}
|
|
}
|
|
|
|
func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) {
|
|
var list devicepluginv1.SgxDevicePluginList
|
|
if err := clnt.List(ctx, &list); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return len(list.Items), nil
|
|
}
|
|
|
|
func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
|
|
devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
|
|
|
|
var nodeSelector map[string]string
|
|
dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector)
|
|
if dpNodeSelectorSize > 0 {
|
|
nodeSelector = make(map[string]string, dpNodeSelectorSize+1)
|
|
for k, v := range devicePlugin.Spec.NodeSelector {
|
|
nodeSelector[k] = v
|
|
}
|
|
nodeSelector["kubernetes.io/arch"] = "amd64"
|
|
} else {
|
|
nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
|
|
}
|
|
|
|
yes := true
|
|
charDevice := v1.HostPathCharDev
|
|
directoryOrCreate := v1.HostPathDirectoryOrCreate
|
|
return &apps.DaemonSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: c.ns,
|
|
GenerateName: devicePlugin.Name + "-",
|
|
Labels: map[string]string{
|
|
"app": appLabel,
|
|
},
|
|
},
|
|
Spec: apps.DaemonSetSpec{
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{
|
|
"app": appLabel,
|
|
},
|
|
},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"app": appLabel,
|
|
},
|
|
},
|
|
Spec: v1.PodSpec{
|
|
InitContainers: []v1.Container{
|
|
{
|
|
Image: devicePlugin.Spec.InitImage,
|
|
ImagePullPolicy: "IfNotPresent",
|
|
Name: "intel-sgx-initcontainer",
|
|
SecurityContext: &v1.SecurityContext{
|
|
ReadOnlyRootFilesystem: &yes,
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
MountPath: "/etc/kubernetes/node-feature-discovery/source.d/",
|
|
Name: "nfd-source-hooks",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: appLabel,
|
|
Args: getPodArgs(devicePlugin),
|
|
Image: devicePlugin.Spec.Image,
|
|
ImagePullPolicy: "IfNotPresent",
|
|
SecurityContext: &v1.SecurityContext{
|
|
ReadOnlyRootFilesystem: &yes,
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "sgxdevices",
|
|
MountPath: "/dev/sgx",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
Name: "sgx-enclave",
|
|
MountPath: "/dev/sgx_enclave",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
Name: "sgx-provision",
|
|
MountPath: "/dev/sgx_provision",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
Name: "kubeletsockets",
|
|
MountPath: "/var/lib/kubelet/device-plugins",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
NodeSelector: nodeSelector,
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "sgxdevices",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/dev/sgx",
|
|
Type: &directoryOrCreate,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "sgx-enclave",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/dev/sgx_enclave",
|
|
Type: &charDevice,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "sgx-provision",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/dev/sgx_provision",
|
|
Type: &charDevice,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "kubeletsockets",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/var/lib/kubelet/device-plugins",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "nfd-source-hooks",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/etc/kubernetes/node-feature-discovery/source.d/",
|
|
Type: &directoryOrCreate,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) {
|
|
dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
|
|
|
|
if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image {
|
|
ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image
|
|
updated = true
|
|
}
|
|
|
|
if dp.Spec.NodeSelector == nil {
|
|
dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"}
|
|
} else {
|
|
dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64"
|
|
}
|
|
if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
|
|
ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector
|
|
updated = true
|
|
}
|
|
|
|
newargs := getPodArgs(dp)
|
|
if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") {
|
|
ds.Spec.Template.Spec.Containers[0].Args = newargs
|
|
updated = true
|
|
}
|
|
|
|
return updated
|
|
}
|
|
|
|
func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, nodeNames []string) (updated bool, err error) {
|
|
dp := rawObj.(*devicepluginv1.SgxDevicePlugin)
|
|
|
|
dsRef, err := reference.GetReference(c.scheme, ds)
|
|
if err != nil {
|
|
return false, errors.Wrap(err, "unable to make reference to controlled daemon set")
|
|
}
|
|
|
|
if dp.Status.ControlledDaemonSet.UID != dsRef.UID {
|
|
dp.Status.ControlledDaemonSet = *dsRef
|
|
updated = true
|
|
}
|
|
|
|
if dp.Status.DesiredNumberScheduled != ds.Status.DesiredNumberScheduled {
|
|
dp.Status.DesiredNumberScheduled = ds.Status.DesiredNumberScheduled
|
|
updated = true
|
|
}
|
|
|
|
if dp.Status.NumberReady != ds.Status.NumberReady {
|
|
dp.Status.NumberReady = ds.Status.NumberReady
|
|
updated = true
|
|
}
|
|
|
|
if strings.Join(dp.Status.NodeNames, ",") != strings.Join(nodeNames, ",") {
|
|
dp.Status.NodeNames = nodeNames
|
|
updated = true
|
|
}
|
|
|
|
return updated, nil
|
|
}
|
|
|
|
func getPodArgs(sdp *devicepluginv1.SgxDevicePlugin) []string {
|
|
args := make([]string, 0, 4)
|
|
args = append(args, "-v", strconv.Itoa(sdp.Spec.LogLevel))
|
|
|
|
if sdp.Spec.EnclaveLimit > 0 {
|
|
args = append(args, "-enclave-limit", strconv.Itoa(sdp.Spec.EnclaveLimit))
|
|
} else {
|
|
args = append(args, "-enclave-limit", "1")
|
|
}
|
|
|
|
if sdp.Spec.ProvisionLimit > 0 {
|
|
args = append(args, "-provision-limit", strconv.Itoa(sdp.Spec.ProvisionLimit))
|
|
} else {
|
|
args = append(args, "-provision-limit", "1")
|
|
}
|
|
|
|
return args
|
|
}
|