Merge pull request #300 from askervin/55B_fpga_kustomization

fpga plugin kustomization, stage 2

.gitignore

@@ -12,6 +12,8 @@ cmd/fpga_tool/fpga_tool
 cmd/gpu_plugin/gpu_plugin
 cmd/qat_plugin/qat_plugin
 
+deployments/fpga_admissionwebhook/base/intel-fpga-webhook-certs-secret
+
 *.h
 *.gbs
 *.gbs.*

build/docker/intel-fpga-plugin.Dockerfile

@@ -36,4 +36,4 @@ RUN chmod a+x /go/bin/fpga_plugin \
 
 FROM scratch as final
 COPY --from=builder /install_root /
-CMD ["/usr/local/bin/intel_fpga_device_plugin"]
+ENTRYPOINT ["/usr/local/bin/intel_fpga_device_plugin"]

cmd/fpga_plugin/README.md

@@ -189,6 +189,31 @@ If you intend to deploy your own image, you will need to reference the
 If you do not want to deploy the `devel` tagged image, you will need to edit the
 YAML deployment files to reference your required image.
 
+### For beta testing: new deployment model
+
+The FPGA plugin deployment is currently being rewritten to enable
+straight-forward deployment of both `af/preprogrammed` and
+`region/orchestrated` modes. The deployment has two steps:
+
+1. Run `scripts/fpga-plugin-prepare-for-kustomization.sh`. This will
+   create the necessary secrets: a key and a signed certificate for
+   the FPGA admission controller.
+
+2. Depending on the FPGA mode, run either
+   ```bash
+   $ kubectl create -k deployments/fpga_plugin/overlays/af
+   ```
+   or
+   ```bash
+   $ kubectl create -k deployments/fpga_plugin/overlays/region
+   ```
+   This will create the service account and deploy
+   both the FPGA plugin and the admission controller in the chosen mode.
+
+This deployment model is under development. The remaining part of this
+document goes through the current deployment model: here for the
+FPGA plugin and in the next document for the FPGA admission controller.
+
 ### Create a service account
 
 To deploy the plugin in a production cluster, create a service account

deployments/fpga_admissionwebhook/base/acceleratorfunction-crd.yaml

@@ -0,0 +1,22 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: acceleratorfunctions.fpga.intel.com
+spec:
+  group: fpga.intel.com
+  version: v1
+  scope: Namespaced
+  names:
+    plural: acceleratorfunctions
+    singular: acceleratorfunction
+    kind: AcceleratorFunction
+    shortNames:
+    - af
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            afuId:
+              type: string
+              pattern: '^[0-9a-f]{8,128}$'

deployments/fpga_admissionwebhook/base/acceleratorfunctions.yaml

@@ -0,0 +1,66 @@
+# DCP 1.0
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.0-compress
+spec:
+  afuId: 946c21d1e49704a5e5daa0805bc6b0785e1765bf
+---
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.0-nlb0
+spec:
+  afuId: d8424dc4a4a3c413f89e433683f9040b
+---
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.0-nlb3
+spec:
+  afuId: f7df405cbd7acf7222f144b0b93acd18
+---
+# DCP 1.1
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.1-nlb0
+spec:
+  afuId: d8424dc4a4a3c413f89e433683f9040b
+---
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.1-nlb3
+spec:
+  afuId: f7df405cbd7acf7222f144b0b93acd18
+---
+# DCP 1.2
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.2-nlb0
+spec:
+  afuId: d8424dc4a4a3c413f89e433683f9040b
+---
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: arria10.dcp1.2-nlb3
+spec:
+  afuId: f7df405cbd7acf7222f144b0b93acd18
+---
+# D5005
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: d5005-nlb0
+spec:
+  afuId: d8424dc4a4a3c413f89e433683f9040b
+---
+apiVersion: fpga.intel.com/v1
+kind: AcceleratorFunction
+metadata:
+  name: d5005-nlb3
+spec:
+  afuId: f7df405cbd7acf7222f144b0b93acd18

deployments/fpga_admissionwebhook/base/fpga-mutator-webhook-cfg.yaml

@@ -0,0 +1,23 @@
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: fpga-mutator-webhook-cfg
+  labels:
+    app: intel-fpga-webhook
+webhooks:
+- name: fpga.mutator.webhooks.intel.com
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  clientConfig:
+    service:
+      path: "/pods"
+      namespace: default
+      name: intel-fpga-webhook-svc
+    caBundle: $(CA_BUNDLE)

deployments/fpga_admissionwebhook/base/fpga-reader-clusterrole.yaml

@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fpga-reader
+rules:
+- apiGroups: ["fpga.intel.com"]
+  resources: ["acceleratorfunctions", "fpgaregions"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: default-fpga-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fpga-reader
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: default

deployments/fpga_admissionwebhook/base/intel-fpga-webhook-deployment.yaml

@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+      name: intel-fpga-webhook-deployment
+      labels:
+        app: intel-fpga-webhook
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: intel-fpga-webhook
+  template:
+    metadata:
+      labels:
+        app: intel-fpga-webhook
+    spec:
+      containers:
+        - name: fpga-mutator
+          image: intel/intel-fpga-admissionwebhook:devel
+          imagePullPolicy: IfNotPresent
+          ports:
+          - containerPort: 8443
+            name: webhook-api
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 3210
+            runAsGroup: 3210
+            readOnlyRootFilesystem: true
+          args:
+            - -tls-cert-file=/etc/webhook/certs/cert.pem
+            - -tls-private-key-file=/etc/webhook/certs/key.pem
+            - -mode=preprogrammed
+            - -debug
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: intel-fpga-webhook-certs

deployments/fpga_admissionwebhook/base/intel-fpga-webhook-svc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: intel-fpga-webhook-svc
+  labels:
+    app: intel-fpga-webhook
+spec:
+  ports:
+  - port: 443
+    targetPort: 8443
+  selector:
+    app: intel-fpga-webhook

deployments/fpga_admissionwebhook/base/kustomization.yaml

@@ -0,0 +1,29 @@
+resources:
+  - fpga-reader-clusterrole.yaml
+  - intel-fpga-webhook-deployment.yaml
+  - intel-fpga-webhook-svc.yaml
+  - fpga-mutator-webhook-cfg.yaml
+  - acceleratorfunction-crd.yaml
+  - acceleratorfunctions.yaml
+
+configurations:
+  - kustomize-varreference.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+  - name: intel-fpga-webhook-certs
+    files:
+      - key.pem=intel-fpga-webhook-certs-secret/key.pem
+      - cert.pem=intel-fpga-webhook-certs-secret/cert.pem
+      - client-ca-file=intel-fpga-webhook-certs-secret/client-ca-file
+
+vars:
+  - name: CA_BUNDLE
+    objref:
+      kind: Secret
+      version: v1
+      name: intel-fpga-webhook-certs
+    fieldref:
+      fieldpath: data[client-ca-file]

deployments/fpga_admissionwebhook/base/kustomize-varreference.yaml

@@ -0,0 +1,3 @@
+varReference:
+  - path: webhooks/clientConfig/caBundle
+    kind: MutatingWebhookConfiguration

deployments/fpga_admissionwebhook/overlays/orchestrated/change-mode-orchestrated.yaml

@@ -0,0 +1,8 @@
+# make sure to change only the -mode=preprogrammed argument
+- op: test
+  path: /spec/template/spec/containers/0/args/2
+  value: -mode=preprogrammed
+
+- op: replace
+  path: /spec/template/spec/containers/0/args/2
+  value: -mode=orchestrated

deployments/fpga_admissionwebhook/overlays/orchestrated/fpgaregion-crd.yaml

@@ -0,0 +1,22 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: fpgaregions.fpga.intel.com
+spec:
+  group: fpga.intel.com
+  version: v1
+  scope: Namespaced
+  names:
+    plural: fpgaregions
+    singular: fpgaregion
+    kind: FpgaRegion
+    shortNames:
+    - fpga
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            interfaceId:
+              type: string
+              pattern: '^[0-9a-f]{8,128}$'

deployments/fpga_admissionwebhook/overlays/orchestrated/fpgaregions.yaml

@@ -0,0 +1,31 @@
+# DCP 1.0
+apiVersion: fpga.intel.com/v1
+kind: FpgaRegion
+metadata:
+  name: arria10.dcp1.0
+spec:
+  interfaceId: ce48969398f05f33946d560708be108a
+---
+# DCP 1.1
+apiVersion: fpga.intel.com/v1
+kind: FpgaRegion
+metadata:
+  name: arria10.dcp1.1
+spec:
+  interfaceId: 9926ab6d6c925a68aabca7d84c545738
+---
+# DCP 1.2
+apiVersion: fpga.intel.com/v1
+kind: FpgaRegion
+metadata:
+  name: arria10.dcp1.2
+spec:
+  interfaceId: 69528db6eb31577a8c3668f9faa081f6
+---
+# D5005
+apiVersion: fpga.intel.com/v1
+kind: FpgaRegion
+metadata:
+  name: d5005
+spec:
+  interfaceId: bfac4d851ee856fe8c95865ce1bbaa2d

deployments/fpga_admissionwebhook/overlays/orchestrated/kustomization.yaml

@@ -0,0 +1,15 @@
+bases:
+  - ../../base
+
+resources:
+  - fpgaregion-crd.yaml
+  - fpgaregions.yaml
+
+patchesJson6902:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: intel-fpga-webhook-deployment
+    path:
+      change-mode-orchestrated.yaml

deployments/fpga_admissionwebhook/overlays/preprogrammed/kustomization.yaml

@@ -0,0 +1,2 @@
+bases:
+  - ../../base

deployments/fpga_plugin/base/intel-fpga-plugin-controller-serviceaccount.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: intel-fpga-plugin-controller
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: node-getter
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: get-nodes
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: intel-fpga-plugin-controller
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: node-getter
+  apiGroup: rbac.authorization.k8s.io

deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-fpga-plugin
+  namespace: kube-system
+  labels:
+    app: intel-fpga-plugin
+spec:
+  selector:
+    matchLabels:
+      app: intel-fpga-plugin
+  template:
+    metadata:
+      labels:
+        app: intel-fpga-plugin
+    spec:
+      serviceAccountName: intel-fpga-plugin-controller
+      initContainers:
+      - name: intel-fpga-initcontainer
+        image: intel/intel-fpga-initcontainer:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /opt/intel/fpga-sw
+          name: intel-fpga-sw
+        - mountPath: /etc/containers/oci/hooks.d
+          name: oci-hooks-config
+      containers:
+      - name: intel-fpga-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        image: intel/intel-fpga-plugin:devel
+        imagePullPolicy: IfNotPresent
+        args:
+          - -mode=af
+        terminationMessagePath: /tmp/termination-log
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - name: devfs
+          mountPath: /dev
+          readOnly: true
+        - name: sysfs
+          mountPath: /sys/class
+          readOnly: true
+        - name: kubeletsockets
+          mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+      - name: devfs
+        hostPath:
+          path: /dev
+      - name: sysfs
+        hostPath:
+          path: /sys/class
+      - name: kubeletsockets
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: intel-fpga-sw
+        hostPath:
+          path: /opt/intel/fpga-sw
+          type: DirectoryOrCreate
+      - name: oci-hooks-config
+        hostPath:
+          path: /etc/containers/oci/hooks.d
+          type: DirectoryOrCreate

deployments/fpga_plugin/base/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - intel-fpga-plugin-controller-serviceaccount.yaml
+  - intel-fpga-plugin-daemonset.yaml

deployments/fpga_plugin/overlays/af/kustomization.yaml

@@ -0,0 +1,3 @@
+bases:
+  - ../../base
+  - ../../../fpga_admissionwebhook/overlays/preprogrammed

deployments/fpga_plugin/overlays/region/change-mode-region.yaml

@@ -0,0 +1,8 @@
+# make sure to change only the -mode=af argument
+- op: test
+  path: /spec/template/spec/containers/0/args/0
+  value: -mode=af
+
+- op: replace
+  path: /spec/template/spec/containers/0/args/0
+  value: -mode=region

deployments/fpga_plugin/overlays/region/kustomization.yaml

@@ -0,0 +1,12 @@
+bases:
+  - ../../base
+  - ../../../fpga_admissionwebhook/overlays/orchestrated
+
+patchesJson6902:
+  - target:
+      group: apps
+      version: v1
+      kind: DaemonSet
+      name: intel-fpga-plugin
+    path:
+      change-mode-region.yaml

scripts/fpga-plugin-prepare-for-kustomization.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script prepares fpga_plugin ready for full deployment by
+#
+# kubectl apply -k deployments/fpga_plugin/overlays/af
+#
+# or
+#
+# kubectl apply -k deployments/fpga_plugin/overlays/region
+
+service="intel-fpga-webhook-svc"
+secret="intel-fpga-webhook-certs"
+
+script_dir="$(realpath $(dirname $0))"
+srcroot="$(realpath ${script_dir}/..)"
+
+kustomize_secret_dir="${srcroot}/deployments/fpga_admissionwebhook/base/${secret}-secret"
+
+mkdir -p "${kustomize_secret_dir}"
+
+# Create signed cert files to kustomize_secret_dir
+${script_dir}/webhook-create-signed-cert.sh --output-dir ${kustomize_secret_dir} --service=$service && {
+    echo ""
+    echo created for kustomization:
+    echo - "${kustomize_secret_dir}"
+}