From 861b23308de1bf618e2f141ac58b30e4e34cdc5b Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 19 Jun 2018 17:33:47 +0300 Subject: [PATCH] Check node's annotations to set mode of FPGA plugin --- cmd/fpga_plugin/README.md | 31 ++++++++++++- cmd/fpga_plugin/fpga_plugin.go | 38 ++++++++++++++++ deployments/fpga_plugin/fpga_plugin.yaml | 43 +++++++++++++++++++ .../fpga_plugin_service_account.yaml | 28 ++++++++++++ 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 deployments/fpga_plugin/fpga_plugin.yaml create mode 100644 deployments/fpga_plugin/fpga_plugin_service_account.yaml diff --git a/cmd/fpga_plugin/README.md b/cmd/fpga_plugin/README.md index 9f9bd296..94d76102 100644 --- a/cmd/fpga_plugin/README.md +++ b/cmd/fpga_plugin/README.md @@ -23,7 +23,8 @@ $ ls /var/lib/kubelet/device-plugins/kubelet.sock ##### Run FPGA device plugin as administrator ``` -$ sudo $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/cmd/fpga_plugin/fpga_plugin -mode af +$ export NODE_NAME="" # if the node's name was overridden and differs from hostname +$ sudo -E $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/cmd/fpga_plugin/fpga_plugin -mode af -kubeconfig /var/run/kubernetes/admin.kubeconfig FPGA device plugin started in af mode device-plugin start server at: /var/lib/kubelet/device-plugins/intel-fpga-af-f7df405cbd7acf7222f144b0b93acd18.sock device-plugin registered @@ -40,7 +41,8 @@ $ kubectl describe node | grep intel.com/fpga ##### Run FPGA device plugin as administrator ``` -$ sudo $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/cmd/fpga_plugin/fpga_plugin -mode region +$ export NODE_NAME="" # if the node's name was overridden and differs from hostname +$ sudo -E $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/cmd/fpga_plugin/fpga_plugin -mode region -kubeconfig /var/run/kubernetes/admin.kubeconfig FPGA device plugin started in region mode device-plugin start server at: /var/lib/kubelet/device-plugins/intel-fpga-region-ce48969398f05f33946d560708be108a.sock device-plugin registered @@ -53,3 +55,28 @@ $ kubectl describe node | grep intel.com/fpga intel.com/fpga-region-ce48969398f05f33946d560708be108a: 1 ``` +### Deploy FPGA device plugin as DaemonSet + +To deploy the plugin in a production cluster create a service account +for the plugin: + + $ kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml + serviceaccount/intel-fpga-plugin-controller created + clusterrole.rbac.authorization.k8s.io/node-getter created + clusterrolebinding.rbac.authorization.k8s.io/get-nodes created + +Then create the DaemonSet itself + + $ kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml + daemonset.apps/intel-fpga-plugin created + +You may want to modify the file `deployments/fpga_plugin/fpga_plugin.yaml` to +use your own container image. But the command + + $ make intel-fpga-plugin + +can provide an image built from the sources. This image launches `fpga_plugin` +in the `af` mode by default. The mode can be overriden on per node basis with +this node annotation: + + $ kubectl annotate node mynode "fpga.intel.com/device-plugin-mode=region" diff --git a/cmd/fpga_plugin/fpga_plugin.go b/cmd/fpga_plugin/fpga_plugin.go index 656010e5..523ba9d9 100644 --- a/cmd/fpga_plugin/fpga_plugin.go +++ b/cmd/fpga_plugin/fpga_plugin.go @@ -22,7 +22,12 @@ import ( "github.com/golang/glog" "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + utilnode "k8s.io/kubernetes/pkg/util/node" "github.com/intel/intel-device-plugins-for-kubernetes/cmd/fpga_plugin/devicecache" "github.com/intel/intel-device-plugins-for-kubernetes/internal/deviceplugin" @@ -118,11 +123,44 @@ func handleUpdate(dms map[string]*deviceManager, updateInfo devicecache.UpdateIn func main() { var mode string + var kubeconfig string + var master string + var config *rest.Config + var err error + flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file") + flag.StringVar(&master, "master", "", "master url") flag.StringVar(&mode, "mode", string(devicecache.AfMode), fmt.Sprintf("device plugin mode: '%s' (default), '%s' or '%s'", devicecache.AfMode, devicecache.RegionMode, devicecache.RegionDevelMode)) flag.Parse() + if kubeconfig == "" { + config, err = rest.InClusterConfig() + } else { + config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig) + } + if err != nil { + glog.Fatal(err) + } + + // if NODE_NAME is not set then try to use hostname + nodeName := utilnode.GetHostname(os.Getenv("NODE_NAME")) + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + glog.Fatal(err) + } + + node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + glog.Fatal(err) + } + + if nodeMode, ok := node.ObjectMeta.Annotations["fpga.intel.com/device-plugin-mode"]; ok { + glog.Info("Overriding mode to ", nodeMode) + mode = nodeMode + } + updatesCh := make(chan devicecache.UpdateInfo) cache, err := devicecache.NewCache(sysfsDirectory, devfsDirectory, mode, updatesCh) diff --git a/deployments/fpga_plugin/fpga_plugin.yaml b/deployments/fpga_plugin/fpga_plugin.yaml new file mode 100644 index 00000000..c4378cd0 --- /dev/null +++ b/deployments/fpga_plugin/fpga_plugin.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-fpga-plugin + namespace: kube-system + labels: + app: intel-fpga-plugin +spec: + selector: + matchLabels: + app: intel-fpga-plugin + template: + metadata: + labels: + app: intel-fpga-plugin + spec: + serviceAccountName: intel-fpga-plugin-controller + containers: + - name: intel-fpga-plugin + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: intel-fpga-plugin:devel + imagePullPolicy: IfNotPresent + volumeMounts: + - name: devfs + mountPath: /dev + - name: sysfs + mountPath: /sys + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: devfs + hostPath: + path: /dev + - name: sysfs + hostPath: + path: /sys + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/deployments/fpga_plugin/fpga_plugin_service_account.yaml b/deployments/fpga_plugin/fpga_plugin_service_account.yaml new file mode 100644 index 00000000..261d8bec --- /dev/null +++ b/deployments/fpga_plugin/fpga_plugin_service_account.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-fpga-plugin-controller + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-getter +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: get-nodes + namespace: kube-system +subjects: +- kind: ServiceAccount + name: intel-fpga-plugin-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: node-getter + apiGroup: rbac.authorization.k8s.io