github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #414 from rojkov/fpga-plugin-kustomize

Browse Source 
fpga: finalize plugin kustomization
This commit is contained in:
Ed Bartosh 2020-07-01 12:33:11 +03:00 committed by GitHub
commit c923b712d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 240 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/fpga_admissionwebhook/README.md
View File

@ -98,7 +98,7 @@ mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgawebhook-mutat
clusterrole.rbac.authorization.k8s.io/intelfpgawebhook-manager-role created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgawebhook-manager-rolebinding created
service/intelfpgawebhook-webhook-service created
deployment.apps/intelfpgawebhook-controller-manager created
deployment.apps/intelfpgawebhook-webhook created
certificate.cert-manager.io/intelfpgawebhook-serving-cert created
issuer.cert-manager.io/intelfpgawebhook-selfsigned-issuer created
```

137
cmd/fpga_plugin/README.md
View File

@ -11,17 +11,12 @@
* [Getting the source code](#getting-the-source-code)
* [Verify node kubelet config](#verify-node-kubelet-config)
* [Deploying as a DaemonSet](#deploying-as-a-daemonset)
* [Create a service account](#create-a-service-account)
* [Deploying `region` mode](#deploying-region-mode)
* [Deploying `af` mode](#deploying-af-mode)
* [Deploy the DaemonSet](#deploy-the-daemonset)
* [Verify plugin registration](#verify-plugin-registration)
* [Building the plugin image](#building-the-plugin-image)
* [Deploy by hand](#deploy-by-hand)
* [Build FPGA device plugin](#build-fpga-device-plugin)
* [Run FPGA device plugin in af mode](#run-fpga-device-plugin-in-af-mode)
* [Run FPGA device plugin in region mode](#run-fpga-device-plugin-in-region-mode)
* [Next steps](#next-steps)
# Introduction
@ -177,82 +172,83 @@ $ ls /var/lib/kubelet/device-plugins/kubelet.sock
## Deploying as a DaemonSet
You can deploy the plugin in either of the modes, using the
[DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml)
supplied. Details are in the following sections. Actions common to both deployment modes are detailed
first. Mode specific actions are then detailed.
As a pre-requisite you need to have [cert-manager](https://cert-manager.io)
up and running:
```bash
$ kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.yaml
$ kubectl get pods -n cert-manager
NAME READY STATUS RESTARTS AGE
cert-manager-7747db9d88-bd2nl 1/1 Running 0 1m
cert-manager-cainjector-87c85c6ff-59sb5 1/1 Running 0 1m
cert-manager-webhook-64dc9fff44-29cfc 1/1 Running 0 1m
```
Depending on the FPGA mode, run either
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/af
namespace/intelfpgaplugin-system created
customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
service/intelfpgaplugin-webhook-service created
deployment.apps/intelfpgaplugin-webhook created
daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
certificate.cert-manager.io/intelfpgaplugin-serving-cert created
issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
```
or
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/region
namespace/intelfpgaplugin-system created
customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
service/intelfpgaplugin-webhook-service created
deployment.apps/intelfpgaplugin-webhook created
daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
certificate.cert-manager.io/intelfpgaplugin-serving-cert created
issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
```
The command should result in two pods running:
```bash
$ kubectl get pods -n intelfpgaplugin-system
NAME READY STATUS RESTARTS AGE
intelfpgaplugin-fpgadeviceplugin-skcw5 1/1 Running 0 57s
intelfpgaplugin-webhook-7d6bcb8b57-k52b9 1/1 Running 0 57s
```
If you intend to deploy your own image, you will need to reference the
[image build section](#build-the-plugin-image) first.
If you do not want to deploy the `devel` tagged image, you will need to edit the
YAML deployment files to reference your required image.
If you do not want to deploy the `devel` or release tagged image, you will need to create your
own kustomization overlay referencing your required image.
### For beta testing: new deployment model
The FPGA plugin deployment is currently being rewritten to enable
straight-forward deployment of both `af` and
`region` modes. The deployment has two steps:
1. Run `scripts/fpga-plugin-prepare-for-kustomization.sh`. This will
create the necessary secrets: a key and a signed certificate for
the FPGA admission controller.
2. Depending on the FPGA mode, run either
```bash
$ kubectl create -k deployments/fpga_plugin/overlays/af
```
or
```bash
$ kubectl create -k deployments/fpga_plugin/overlays/region
```
This will create the service account and deploy
both the FPGA plugin and the admission controller in the chosen mode.
This deployment model is under development. The remaining part of this
document goes through the current deployment model: here for the
FPGA plugin and in the next document for the FPGA admission controller.
### Create a service account
To deploy the plugin in a production cluster, create a service account
for the plugin:
If you need the FPGA plugin on some nodes to operate in a different mode then add this
annotation to the nodes:
```bash
$ kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml
serviceaccount/intel-fpga-plugin-controller created
clusterrole.rbac.authorization.k8s.io/node-getter created
clusterrolebinding.rbac.authorization.k8s.io/get-nodes created
$ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=region'
```
### Deploying `region` mode
To deploy the FPGA plugin DaemonSet in `region` mode, you need to set the plugin
mode annotation on all of your nodes, otherwise the FPGA plugin will run in its default
`af` mode.
or
```bash
$ kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region'
$ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=af'
```
And restart the pods on the nodes.
### Deploying `af` mode
To deploy the FPGA plugin DaemonSet in `af` mode, you do not need to set the mode annotation on
your nodes, as the FPGA plugin runs in `af` mode by default.
> **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml)
> **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
> also deploys the [FPGA CRI-O hook](../fpga_criohook) `initcontainer` image, but it will be
> benign (un-used) when running the FPGA plugin in `af` mode.
### Deploy the DaemonSet
You can then use the example DaemonSet YAML file provided to deploy the plugin.
```bash
$ kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml
daemonset.apps/intel-fpga-plugin created
```
### Verify plugin registration
Verify the FPGA plugin has been deployed on the nodes. The below shows the output
@ -288,8 +284,8 @@ Successfully tagged intel/intel-fpga-plugin:devel
This image launches `fpga_plugin` in `af` mode by default.
To use your own container image, modify the
[`deployments/fpga_plugin/fpga_plugin.yaml`](../../deployments/fpga_plugin/fpga_plugin.yaml)
To use your own container image, create you own kustomization overlay patching
[`deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml`](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
file.
## Deploy by hand
@ -337,6 +333,3 @@ FPGA device plugin started in region mode
device-plugin start server at: /var/lib/kubelet/device-plugins/fpga.intel.com-region-ce48969398f05f33946d560708be108a.sock
device-plugin registered
```
# Next steps
Continue installation with the [FPGA admission controller webhook](../fpga_admissionwebhook/README.md).

63
demo/screencast-fpga-orchestrated.sh
View File

@ -39,18 +39,12 @@ cleanup()
out 'Cleanup demo artifacts' 20
out 'delete test pod:' 20
command 'kubectl delete pod test-fpga-region || true' 20
out 'delete ServiceAccount:' 20
command 'kubectl delete ServiceAccount intel-fpga-plugin-controller --namespace kube-system || true' 20
out 'delete ClusterRole:' 20
command 'kubectl delete ClusterRole node-getter --namespace kube-system || true' 20
out 'delete ClusterRoleBinding:' 20
command 'kubectl delete ClusterRoleBinding get-nodes --namespace kube-system || true' 20
out 'delete mappings' 20
command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
out 'delete namespace and all the objects in the intelfpgaplugin-system namespace:' 20
command 'kubectl delete namespace intelfpgaplugin-system || true' 20
out 'delete node annotation:' 20
command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode- || true' 20
out 'delete plugin daemonset:' 20
command 'kubectl delete daemonset intel-fpga-plugin --namespace kube-system || true' 20
out 'delete webhook deployment:' 20
command 'kubectl delete deployment intel-fpga-webhook-deployment || true' 20
}
record()
@ -83,26 +77,11 @@ screen2()
}
screen3()
{
clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '3. Deploy admission controller webhook'
command 'kubectl apply -k deployments/fpga_admissionwebhook/default'
sleep 2
out 'Check if its pod is running:'
command 'kubectl get pods -n intelfpgawebhook-system'
out 'Deploy the mappings:'
command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
out 'Check pod logs:'
command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system"
}
screen4()
{
clear
cd /srv/demo
sudo rm -rf /srv/intel.com/fpga/Arria10.dcp1.2 /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6
out '4. Create bistream storage'
out '3. Create bistream storage'
out 'Create directory for Arria10.dcp1.2 interface id:'
command 'sudo mkdir -p /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6'
out 'Create Arria10.dcp1.2 symlink for convenience:'
@ -116,30 +95,30 @@ screen4()
command 'ls -la /srv/intel.com/fpga/Arria10.dcp1.2/'
}
screen5()
screen4()
{
clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '5. Deploy FPGA plugin'
out 'Create a service account for the plugin'
command 'kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml'
out 'Set region mode for the plugin:'
command "kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region'"
out 'Create plugin daemonset:'
command 'kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml'
out 'Check if its pod is runnning:'
command 'kubectl get pod --namespace kube-system |grep intel-fpga-plugin'
out 'Check if it runs in 'region' mode:'
command "kubectl logs $(kubectl get pods --namespace kube-system |grep intel-fpga-plugin|cut -f1 -d' ') --namespace kube-system"
out '4. Deploy FPGA plugin'
command 'kubectl apply -k deployments/fpga_plugin/overlays/region'
sleep 2
out 'Check if its pods are running:'
command 'kubectl get pods -n intelfpgaplugin-system'
out 'Deploy the mappings:'
command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
out 'Check webhook pod logs:'
command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
out 'Check if the plugin runs in 'region' mode:'
command "kubectl logs $(kubectl get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
out 'Check if resource fpga.intel.com/region-<FPGA interface id> is allocatable:'
command 'kubectl describe node |grep -A5 Allocatable'
}
screen6()
screen5()
{
clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '6. Run OPAE workload that uses NLB3 bitstream'
out '5. Run OPAE workload that uses NLB3 bitstream'
out 'Program devices with a NLB0 bitstream that is not wanted by the workload:'
command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.0 pr"
command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.1 pr"
@ -154,7 +133,7 @@ screen6()
command 'cat /sys/class/*/*/*/afu_id'
}
screen7()
screen6()
{
clear
out 'Summary:' 15
@ -171,7 +150,7 @@ if [ "$1" == 'play' ] ; then
if [ -n "$2" ] ; then
screen$2
else
for n in $(seq 7) ; do screen$n ; sleep 3; done
for n in $(seq 6) ; do screen$n ; sleep 3; done
fi
elif [ "$1" == 'cleanup' ] ; then
cleanup

40
demo/screencast-fpga-preprogrammed.sh
View File

@ -39,8 +39,7 @@ cleanup()
out 'Cleanup demo artifacts' 200
command 'kubectl delete pod test-fpga-preprogrammed || true' 200
command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
command 'kubectl delete -k plugins/deployments/fpga_admissionwebhook/default || true' 200
command './plugins/scripts/deploy-fpgaplugin.sh cleanup' 200
command 'kubectl delete namespace intelfpgaplugin-system || true' 20
command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode-' 200
command 'rm -rf plugins' 200
}
@ -66,7 +65,7 @@ screen1()
sleep 2
out 'Check if docker is running k8s pods:'
command 'docker ps --format "table {{.Names}}"'
sleep 2
sleep 1
}
screen2()
@ -75,43 +74,35 @@ screen2()
rm -rf plugins
out '2. Clone Intel Device Plugins for Kubernetes repository'
command "git clone https://github.com/intel/intel-device-plugins-for-kubernetes plugins" 15
sleep 1
}
screen3()
{
clear
out '3. Deploy admission controller webhook'
command 'kubectl apply -k plugins/deployments/fpga_admissionwebhook/default'
out '3. Deploy FPGA plugin'
command 'kubectl apply -k plugins/deployments/fpga_plugin/overlays/af'
sleep 3
out 'Deploy example mappings:'
command 'kubectl apply -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml'
sleep 3
out 'Check if webhook pod is running:'
command 'kubectl get pods -n intelfpgawebhook-system'
out 'Check if the plugin pods are running:'
command 'kubectl get pods -n intelfpgaplugin-system'
sleep 2
out 'Check its logs:'
command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system"
sleep 2
}
screen4()
{
clear
out '5. Deploy FPGA plugin'
command './plugins/scripts/deploy-fpgaplugin.sh'
sleep 4
out 'Check if its pod is runnning:'
command 'kubectl get pod | grep intel-fpga-plugin'
out 'Check webhook pod logs:'
command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
out 'Check if the plugin runs in 'region' mode:'
command "kubectl logs $(kubectl get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
sleep 2
out 'Check if resource fpga.intel.com/af-<af id> is allocatable:'
command 'kubectl describe node |grep -A4 Allocatable'
sleep 2
}
screen5()
screen4()
{
clear
out '6. Run OPAE workload that uses NLB3 bitstream'
out '4. Run OPAE workload that uses NLB3 bitstream'
out 'Check if devices are programmed with NLB3:'
command 'cat /sys/class/*/*/*/afu_id'
out 'Run workload:'
@ -120,9 +111,10 @@ screen5()
sleep 5
out 'Look at the test output'
command 'kubectl logs test-fpga-preprogrammed'
sleep 2
}
screen6()
screen5()
{
clear
out 'Summary:' 15
@ -138,7 +130,7 @@ if [ "$1" == 'play' ] ; then
if [ -n "$2" ] ; then
screen$2
else
for n in $(seq 6) ; do screen$n ; sleep 3; done
for n in $(seq 5) ; do screen$n ; sleep 3; done
fi
elif [ "$1" == 'cleanup' ] ; then
cleanup

40
deployments/fpga_admissionwebhook/base/kustomization.yaml Normal file
View File

@ -0,0 +1,40 @@
bases:
- ../crd
- ../rbac
- ../manager
- ../webhook
- ../certmanager
patchesStrategicMerge:
# Enable webhook
- manager_webhook_patch.yaml
# Enable certmanager integration
- webhookcainjection_patch.yaml
vars:
- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
fieldref:
fieldpath: metadata.namespace
- name: CERTIFICATE_NAME
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
- name: SERVICE_NAMESPACE # namespace of the service
objref:
kind: Service
version: v1
name: webhook-service
fieldref:
fieldpath: metadata.namespace
- name: SERVICE_NAME
objref:
kind: Service
version: v1
name: webhook-service

2
deployments/fpga_admissionwebhook/default/manager_webhook_patch.yaml → deployments/fpga_admissionwebhook/base/manager_webhook_patch.yaml
View File

@ -1,7 +1,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
name: webhook
namespace: system
spec:
template:

0
deployments/fpga_admissionwebhook/default/webhookcainjection_patch.yaml → deployments/fpga_admissionwebhook/base/webhookcainjection_patch.yaml
View File

41
deployments/fpga_admissionwebhook/default/kustomization.yaml
View File

@ -13,42 +13,7 @@ namePrefix: intelfpgawebhook-
# someName: someValue
bases:
- ../crd
- ../rbac
- ../manager
- ../webhook
- ../certmanager
- ../base
patchesStrategicMerge:
# Enable webhook
- manager_webhook_patch.yaml
# Enable certmanager integration
- webhookcainjection_patch.yaml
vars:
- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
fieldref:
fieldpath: metadata.namespace
- name: CERTIFICATE_NAME
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
- name: SERVICE_NAMESPACE # namespace of the service
objref:
kind: Service
version: v1
name: webhook-service
fieldref:
fieldpath: metadata.namespace
- name: SERVICE_NAME
objref:
kind: Service
version: v1
name: webhook-service
resources:
- namespace.yaml

6
deployments/fpga_admissionwebhook/default/namespace.yaml Normal file
View File

@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: system

9
deployments/fpga_admissionwebhook/manager/manager.yaml
View File

@ -1,14 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: system
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
name: webhook
namespace: system
labels:
control-plane: controller-manager

28
deployments/fpga_plugin/base/intel-fpga-plugin-controller-serviceaccount.yaml
View File

@ -1,28 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: intel-fpga-plugin-controller
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
namespace: kube-system
subjects:
- kind: ServiceAccount
name: intel-fpga-plugin-controller
namespace: kube-system
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

10
deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml
View File

@ -1,8 +1,13 @@
apiVersion: v1
kind: Namespace
metadata:
name: system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: intel-fpga-plugin
namespace: kube-system
name: fpgadeviceplugin
namespace: system
labels:
app: intel-fpga-plugin
spec:
@ -14,7 +19,6 @@ spec:
labels:
app: intel-fpga-plugin
spec:
serviceAccountName: intel-fpga-plugin-controller
initContainers:
- name: intel-fpga-initcontainer
image: intel/intel-fpga-initcontainer:devel

3
deployments/fpga_plugin/base/kustomization.yaml
View File

@ -1,3 +1,4 @@
resources:
- intel-fpga-plugin-controller-serviceaccount.yaml
- role.yaml
- role_binding.yaml
- intel-fpga-plugin-daemonset.yaml

8
deployments/fpga_plugin/base/role.yaml Normal file
View File

@ -0,0 +1,8 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]

12
deployments/fpga_plugin/base/role_binding.yaml Normal file
View File

@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
subjects:
- kind: ServiceAccount
name: default
namespace: system
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

67
deployments/fpga_plugin/fpga_plugin.yaml
View File

@ -1,67 +0,0 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: intel-fpga-plugin
namespace: {namespace}
labels:
app: intel-fpga-plugin
spec:
selector:
matchLabels:
app: intel-fpga-plugin
template:
metadata:
labels:
app: intel-fpga-plugin
spec:
serviceAccountName: intel-fpga-plugin-controller
initContainers:
- name: intel-fpga-initcontainer
image: intel/intel-fpga-initcontainer:devel
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /opt/intel/fpga-sw
name: intel-fpga-sw
- mountPath: /etc/containers/oci/hooks.d
name: oci-hooks-config
containers:
- name: intel-fpga-plugin
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
image: intel/intel-fpga-plugin:devel
imagePullPolicy: IfNotPresent
terminationMessagePath: /tmp/termination-log
securityContext:
readOnlyRootFilesystem: true
volumeMounts:
- name: devfs
mountPath: /dev
readOnly: true
- name: sysfs
mountPath: /sys/class
readOnly: true
- name: kubeletsockets
mountPath: /var/lib/kubelet/device-plugins
volumes:
- name: devfs
hostPath:
path: /dev
- name: sysfs
hostPath:
path: /sys/class
- name: kubeletsockets
hostPath:
path: /var/lib/kubelet/device-plugins
- name: intel-fpga-sw
hostPath:
path: /opt/intel/fpga-sw
type: DirectoryOrCreate
- name: oci-hooks-config
hostPath:
path: /etc/containers/oci/hooks.d
type: DirectoryOrCreate

28
deployments/fpga_plugin/fpga_plugin_service_account.yaml
View File

@ -1,28 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: intel-fpga-plugin-controller
namespace: {namespace}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
namespace: kube-system
subjects:
- kind: ServiceAccount
name: intel-fpga-plugin-controller
namespace: {namespace}
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

5
deployments/fpga_plugin/overlays/af/kustomization.yaml
View File

@ -1,3 +1,6 @@
namespace: intelfpgaplugin-system
namePrefix: intelfpgaplugin-
bases:
- ../../base
- ../../../fpga_admissionwebhook/overlays/preprogrammed
- ../../../fpga_admissionwebhook/base

8
deployments/fpga_plugin/overlays/region/change-mode-region.yaml
View File

@ -1,8 +0,0 @@
# make sure to change only the -mode=af argument
- op: test
path: /spec/template/spec/containers/0/args/0
value: -mode=af
- op: replace
path: /spec/template/spec/containers/0/args/0
value: -mode=region

15
deployments/fpga_plugin/overlays/region/kustomization.yaml
View File

@ -1,12 +1,9 @@
namespace: intelfpgaplugin-system
namePrefix: intelfpgaplugin-
bases:
- ../../base
- ../../../fpga_admissionwebhook/overlays/orchestrated
- ../../../fpga_admissionwebhook/base
patchesJson6902:
- target:
group: apps
version: v1
kind: DaemonSet
name: intel-fpga-plugin
path:
change-mode-region.yaml
patchesStrategicMerge:
- mode-region.yaml

12
deployments/fpga_plugin/overlays/region/mode-region.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fpgadeviceplugin
namespace: system
spec:
template:
spec:
containers:
- name: intel-fpga-plugin
args:
- -mode=region

73
scripts/deploy-fpgaplugin.sh
View File

@ -1,73 +0,0 @@
#!/bin/sh -eu
srcroot="$(realpath $(dirname $0)/..)"
kubectl='kubectl'
namespace='default'
mode='af'
command=''
help() {
echo "Usage: $1 <options> [help|cleanup]"
echo ' Command "help" prints this message'
echo ' Command "cleanup" removes admission webhook deployment'
echo ''
echo ' If no command is given the script will deploy the webhook'
echo ''
echo ' Options:'
echo ' --kubectl <kubectl> - path to the kubectl utility'
echo ' --mode <mode> - "af" (default) or "region" mode of operation'
echo ' --namespace <name> - namespace to deploy the plugin in'
}
while [ $# -gt 0 ]; do
case ${1} in
--kubectl)
kubectl="$2"
shift
;;
--mode)
mode="$2"
shift
;;
--namespace)
namespace="$2"
shift
;;
help)
help $(basename $0)
exit 0
;;
cleanup)
command="cleanup"
;;
*)
echo "Unknown option: ${1}"
exit 1
;;
esac
shift
done
which ${kubectl} > /dev/null 2>&1 || { echo "ERROR: ${kubectl} not found"; exit 1; }
echo "Clean up previously created deployment"
${kubectl} annotate node --all fpga.intel.com/device-plugin-mode- || true
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin.yaml | ${kubectl} --namespace ${namespace} delete -f - || true
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin_service_account.yaml | ${kubectl} --namespace ${namespace} delete -f - || true
if [ "x${command}" = "xcleanup" ]; then
echo "Cleanup done. Exiting..."
exit 0
fi
if [ "x${mode}" != "xaf" -a "x${mode}" != "xregion" ]; then
echo "ERROR: supported modes are 'af' and 'region'"
exit 1
fi
echo 'Set default operation mode'
${kubectl} annotate node --overwrite --all fpga.intel.com/device-plugin-mode=${mode} || true
echo 'Create service account'
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin_service_account.yaml | ${kubectl} --namespace ${namespace} create -f - || true
echo 'Create plugin daemonset'
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin.yaml | ${kubectl} --namespace ${namespace} create -f - || true

51
test/e2e/fpga/fpga.go
View File

@ -18,6 +18,8 @@ package fpga
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"time"
@ -33,13 +35,13 @@ import (
)
const (
pluginDeployScript = "scripts/deploy-fpgaplugin.sh"
kustomizationYaml = "deployments/fpga_admissionwebhook/default/kustomization.yaml"
nlb0NodeResource = "fpga.intel.com/af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs"
nlb0PodResource = "fpga.intel.com/arria10.dcp1.2-nlb0-orchestrated"
nlb3PodResource = "fpga.intel.com/arria10.dcp1.2-nlb3-orchestrated"
nlb0PodResourceAF = "fpga.intel.com/arria10.dcp1.2-nlb0-preprogrammed"
arria10NodeResource = "fpga.intel.com/region-69528db6eb31577a8c3668f9faa081f6"
pluginKustomizationYaml = "deployments/fpga_plugin/base/kustomization.yaml"
webhookKustomizationYaml = "deployments/fpga_admissionwebhook/default/kustomization.yaml"
nlb0NodeResource = "fpga.intel.com/af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs"
nlb0PodResource = "fpga.intel.com/arria10.dcp1.2-nlb0-orchestrated"
nlb3PodResource = "fpga.intel.com/arria10.dcp1.2-nlb3-orchestrated"
nlb0PodResourceAF = "fpga.intel.com/arria10.dcp1.2-nlb0-preprogrammed"
arria10NodeResource = "fpga.intel.com/region-69528db6eb31577a8c3668f9faa081f6"
)
func init() {
@ -47,14 +49,14 @@ func init() {
}
func describe() {
kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml)
webhookKustomizationPath, err := utils.LocateRepoFile(webhookKustomizationYaml)
if err != nil {
framework.Failf("unable to locate %q: %v", kustomizationYaml, err)
framework.Failf("unable to locate %q: %v", webhookKustomizationYaml, err)
}
pluginDeployScriptPath, err := utils.LocateRepoFile(pluginDeployScript)
pluginKustomizationPath, err := utils.LocateRepoFile(pluginKustomizationYaml)
if err != nil {
framework.Failf("unable to locate %q: %v", pluginDeployScript, err)
framework.Failf("unable to locate %q: %v", pluginKustomizationYaml, err)
}
fmw := framework.NewDefaultFramework("fpgaplugin-e2e")
@ -62,23 +64,34 @@ func describe() {
ginkgo.It("Run FPGA plugin tests", func() {
// Deploy webhook
ginkgo.By(fmt.Sprintf("namespace %s: deploying webhook", fmw.Namespace.Name))
utils.DeployFpgaWebhook(fmw, kustomizationPath)
utils.DeployFpgaWebhook(fmw, webhookKustomizationPath)
ginkgo.By("deploying mappings")
framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-n", fmw.Namespace.Name, "-f", filepath.Dir(kustomizationPath)+"/../mappings-collection.yaml")
framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-n", fmw.Namespace.Name, "-f", filepath.Dir(webhookKustomizationPath)+"/../mappings-collection.yaml")
// Run region test case twice to ensure that device is reprogrammed at least once
runTestCase(fmw, pluginDeployScriptPath, "region", arria10NodeResource, nlb3PodResource, "nlb3", "nlb0")
runTestCase(fmw, pluginDeployScriptPath, "region", arria10NodeResource, nlb0PodResource, "nlb0", "nlb3")
runTestCase(fmw, pluginKustomizationPath, "region", arria10NodeResource, nlb3PodResource, "nlb3", "nlb0")
runTestCase(fmw, pluginKustomizationPath, "region", arria10NodeResource, nlb0PodResource, "nlb0", "nlb3")
// Run af test case
runTestCase(fmw, pluginDeployScriptPath, "af", nlb0NodeResource, nlb0PodResourceAF, "nlb0", "nlb3")
runTestCase(fmw, pluginKustomizationPath, "af", nlb0NodeResource, nlb0PodResourceAF, "nlb0", "nlb3")
})
}
func runTestCase(fmw *framework.Framework, pluginDeployScriptPath, pluginMode, nodeResource, podResource, cmd1, cmd2 string) {
func runTestCase(fmw *framework.Framework, pluginKustomizationPath, pluginMode, nodeResource, podResource, cmd1, cmd2 string) {
tmpDir, err := ioutil.TempDir("", "fpgaplugine2etest-"+fmw.Namespace.Name)
if err != nil {
framework.Failf("unable to create temp directory: %v", err)
}
defer os.RemoveAll(tmpDir)
err = utils.CreateKustomizationOverlay(fmw.Namespace.Name, filepath.Dir(pluginKustomizationPath)+"/../overlays/"+pluginMode, tmpDir)
if err != nil {
framework.Failf("unable to kustomization overlay: %v", err)
}
ginkgo.By(fmt.Sprintf("namespace %s: deploying FPGA plugin in %s mode", fmw.Namespace.Name, pluginMode))
_, _, err := framework.RunCmd(pluginDeployScriptPath, "--mode", pluginMode, "--namespace", fmw.Namespace.Name)
framework.ExpectNoError(err)
_, _ = framework.RunKubectl(fmw.Namespace.Name, "delete", "-k", tmpDir)
framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-k", tmpDir)
waitForPod(fmw, "intel-fpga-plugin")

5
test/e2e/utils/utils.go
View File

@ -117,7 +117,8 @@ func LocateRepoFile(repopath string) (string, error) {
return "", errors.New("no file found, try to define PLUGINS_REPO_DIR pointing to the root of the repository")
}
func createKustomizationOverlay(namespace, base, overlay string) error {
// CreateKustomizationOverlay creates an overlay with overridden namespace.
func CreateKustomizationOverlay(namespace, base, overlay string) error {
relPath := ""
for range strings.Split(overlay[1:], "/") {
relPath = relPath + "../"
@ -141,7 +142,7 @@ func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) {
}
defer os.RemoveAll(tmpDir)
err = createKustomizationOverlay(f.Namespace.Name, filepath.Dir(kustomizationPath), tmpDir)
err = CreateKustomizationOverlay(f.Namespace.Name, filepath.Dir(kustomizationPath), tmpDir)
if err != nil {
framework.Failf("unable to kustomization overlay: %v", err)
}