github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #414 from rojkov/fpga-plugin-kustomize

Browse Source 
fpga: finalize plugin kustomization
This commit is contained in:
Ed Bartosh 2020-07-01 12:33:11 +03:00 committed by GitHub
commit c923b712d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 240 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/fpga_admissionwebhook/README.md
View File

@ -98,7 +98,7 @@ mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgawebhook-mutat
clusterrole.rbac.authorization.k8s.io/intelfpgawebhook-manager-role created clusterrole.rbac.authorization.k8s.io/intelfpgawebhook-manager-role created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgawebhook-manager-rolebinding created clusterrolebinding.rbac.authorization.k8s.io/intelfpgawebhook-manager-rolebinding created
service/intelfpgawebhook-webhook-service created service/intelfpgawebhook-webhook-service created
deployment.apps/intelfpgawebhook-controller-manager created deployment.apps/intelfpgawebhook-webhook created
certificate.cert-manager.io/intelfpgawebhook-serving-cert created certificate.cert-manager.io/intelfpgawebhook-serving-cert created
issuer.cert-manager.io/intelfpgawebhook-selfsigned-issuer created issuer.cert-manager.io/intelfpgawebhook-selfsigned-issuer created
``` ```

137
cmd/fpga_plugin/README.md
View File

@ -11,17 +11,12 @@
* [Getting the source code](#getting-the-source-code) * [Getting the source code](#getting-the-source-code)
* [Verify node kubelet config](#verify-node-kubelet-config) * [Verify node kubelet config](#verify-node-kubelet-config)
* [Deploying as a DaemonSet](#deploying-as-a-daemonset) * [Deploying as a DaemonSet](#deploying-as-a-daemonset)
* [Create a service account](#create-a-service-account)
* [Deploying `region` mode](#deploying-region-mode)
* [Deploying `af` mode](#deploying-af-mode)
* [Deploy the DaemonSet](#deploy-the-daemonset)
* [Verify plugin registration](#verify-plugin-registration) * [Verify plugin registration](#verify-plugin-registration)
* [Building the plugin image](#building-the-plugin-image) * [Building the plugin image](#building-the-plugin-image)
* [Deploy by hand](#deploy-by-hand) * [Deploy by hand](#deploy-by-hand)
* [Build FPGA device plugin](#build-fpga-device-plugin) * [Build FPGA device plugin](#build-fpga-device-plugin)
* [Run FPGA device plugin in af mode](#run-fpga-device-plugin-in-af-mode) * [Run FPGA device plugin in af mode](#run-fpga-device-plugin-in-af-mode)
* [Run FPGA device plugin in region mode](#run-fpga-device-plugin-in-region-mode) * [Run FPGA device plugin in region mode](#run-fpga-device-plugin-in-region-mode)
* [Next steps](#next-steps)
# Introduction # Introduction
@ -177,82 +172,83 @@ $ ls /var/lib/kubelet/device-plugins/kubelet.sock
## Deploying as a DaemonSet ## Deploying as a DaemonSet
You can deploy the plugin in either of the modes, using the As a pre-requisite you need to have [cert-manager](https://cert-manager.io)
[DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml) up and running:
supplied. Details are in the following sections. Actions common to both deployment modes are detailed
first. Mode specific actions are then detailed. ```bash
$ kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.yaml
$ kubectl get pods -n cert-manager
NAME READY STATUS RESTARTS AGE
cert-manager-7747db9d88-bd2nl 1/1 Running 0 1m
cert-manager-cainjector-87c85c6ff-59sb5 1/1 Running 0 1m
cert-manager-webhook-64dc9fff44-29cfc 1/1 Running 0 1m
```
Depending on the FPGA mode, run either
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/af
namespace/intelfpgaplugin-system created
customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
service/intelfpgaplugin-webhook-service created
deployment.apps/intelfpgaplugin-webhook created
daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
certificate.cert-manager.io/intelfpgaplugin-serving-cert created
issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
```
or
```bash
$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin/overlays/region
namespace/intelfpgaplugin-system created
customresourcedefinition.apiextensions.k8s.io/acceleratorfunctions.fpga.intel.com created
customresourcedefinition.apiextensions.k8s.io/fpgaregions.fpga.intel.com created
mutatingwebhookconfiguration.admissionregistration.k8s.io/intelfpgaplugin-mutating-webhook-configuration created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-manager-role created
clusterrole.rbac.authorization.k8s.io/intelfpgaplugin-node-getter created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-get-nodes created
clusterrolebinding.rbac.authorization.k8s.io/intelfpgaplugin-manager-rolebinding created
service/intelfpgaplugin-webhook-service created
deployment.apps/intelfpgaplugin-webhook created
daemonset.apps/intelfpgaplugin-fpgadeviceplugin created
certificate.cert-manager.io/intelfpgaplugin-serving-cert created
issuer.cert-manager.io/intelfpgaplugin-selfsigned-issuer created
```
The command should result in two pods running:
```bash
$ kubectl get pods -n intelfpgaplugin-system
NAME READY STATUS RESTARTS AGE
intelfpgaplugin-fpgadeviceplugin-skcw5 1/1 Running 0 57s
intelfpgaplugin-webhook-7d6bcb8b57-k52b9 1/1 Running 0 57s
```
If you intend to deploy your own image, you will need to reference the If you intend to deploy your own image, you will need to reference the
[image build section](#build-the-plugin-image) first. [image build section](#build-the-plugin-image) first.
If you do not want to deploy the `devel` tagged image, you will need to edit the If you do not want to deploy the `devel` or release tagged image, you will need to create your
YAML deployment files to reference your required image. own kustomization overlay referencing your required image.
### For beta testing: new deployment model If you need the FPGA plugin on some nodes to operate in a different mode then add this
annotation to the nodes:
The FPGA plugin deployment is currently being rewritten to enable
straight-forward deployment of both `af` and
`region` modes. The deployment has two steps:
1. Run `scripts/fpga-plugin-prepare-for-kustomization.sh`. This will
create the necessary secrets: a key and a signed certificate for
the FPGA admission controller.
2. Depending on the FPGA mode, run either
```bash
$ kubectl create -k deployments/fpga_plugin/overlays/af
```
or
```bash
$ kubectl create -k deployments/fpga_plugin/overlays/region
```
This will create the service account and deploy
both the FPGA plugin and the admission controller in the chosen mode.
This deployment model is under development. The remaining part of this
document goes through the current deployment model: here for the
FPGA plugin and in the next document for the FPGA admission controller.
### Create a service account
To deploy the plugin in a production cluster, create a service account
for the plugin:
```bash ```bash
$ kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml $ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=region'
serviceaccount/intel-fpga-plugin-controller created
clusterrole.rbac.authorization.k8s.io/node-getter created
clusterrolebinding.rbac.authorization.k8s.io/get-nodes created
``` ```
or
### Deploying `region` mode
To deploy the FPGA plugin DaemonSet in `region` mode, you need to set the plugin
mode annotation on all of your nodes, otherwise the FPGA plugin will run in its default
`af` mode.
```bash ```bash
$ kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region' $ kubectl annotate node <node_name> 'fpga.intel.com/device-plugin-mode=af'
``` ```
And restart the pods on the nodes.
### Deploying `af` mode > **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
To deploy the FPGA plugin DaemonSet in `af` mode, you do not need to set the mode annotation on
your nodes, as the FPGA plugin runs in `af` mode by default.
> **Note:** The FPGA plugin [DaemonSet YAML](../../deployments/fpga_plugin/fpga_plugin.yaml)
> also deploys the [FPGA CRI-O hook](../fpga_criohook) `initcontainer` image, but it will be > also deploys the [FPGA CRI-O hook](../fpga_criohook) `initcontainer` image, but it will be
> benign (un-used) when running the FPGA plugin in `af` mode. > benign (un-used) when running the FPGA plugin in `af` mode.
### Deploy the DaemonSet
You can then use the example DaemonSet YAML file provided to deploy the plugin.
```bash
$ kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml
daemonset.apps/intel-fpga-plugin created
```
### Verify plugin registration ### Verify plugin registration
Verify the FPGA plugin has been deployed on the nodes. The below shows the output Verify the FPGA plugin has been deployed on the nodes. The below shows the output
@ -288,8 +284,8 @@ Successfully tagged intel/intel-fpga-plugin:devel
This image launches `fpga_plugin` in `af` mode by default. This image launches `fpga_plugin` in `af` mode by default.
To use your own container image, modify the To use your own container image, create you own kustomization overlay patching
[`deployments/fpga_plugin/fpga_plugin.yaml`](../../deployments/fpga_plugin/fpga_plugin.yaml) [`deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml`](../../deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml)
file. file.
## Deploy by hand ## Deploy by hand
@ -337,6 +333,3 @@ FPGA device plugin started in region mode
device-plugin start server at: /var/lib/kubelet/device-plugins/fpga.intel.com-region-ce48969398f05f33946d560708be108a.sock device-plugin start server at: /var/lib/kubelet/device-plugins/fpga.intel.com-region-ce48969398f05f33946d560708be108a.sock
device-plugin registered device-plugin registered
``` ```
# Next steps
Continue installation with the [FPGA admission controller webhook](../fpga_admissionwebhook/README.md).

63
demo/screencast-fpga-orchestrated.sh
View File

@ -39,18 +39,12 @@ cleanup()
out 'Cleanup demo artifacts' 20 out 'Cleanup demo artifacts' 20
out 'delete test pod:' 20 out 'delete test pod:' 20
command 'kubectl delete pod test-fpga-region || true' 20 command 'kubectl delete pod test-fpga-region || true' 20
out 'delete ServiceAccount:' 20 out 'delete mappings' 20
command 'kubectl delete ServiceAccount intel-fpga-plugin-controller --namespace kube-system || true' 20 command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
out 'delete ClusterRole:' 20 out 'delete namespace and all the objects in the intelfpgaplugin-system namespace:' 20
command 'kubectl delete ClusterRole node-getter --namespace kube-system || true' 20 command 'kubectl delete namespace intelfpgaplugin-system || true' 20
out 'delete ClusterRoleBinding:' 20
command 'kubectl delete ClusterRoleBinding get-nodes --namespace kube-system || true' 20
out 'delete node annotation:' 20 out 'delete node annotation:' 20
command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode- || true' 20 command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode- || true' 20
out 'delete plugin daemonset:' 20
command 'kubectl delete daemonset intel-fpga-plugin --namespace kube-system || true' 20
out 'delete webhook deployment:' 20
command 'kubectl delete deployment intel-fpga-webhook-deployment || true' 20
} }
record() record()
@ -83,26 +77,11 @@ screen2()
} }
screen3() screen3()
{
clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '3. Deploy admission controller webhook'
command 'kubectl apply -k deployments/fpga_admissionwebhook/default'
sleep 2
out 'Check if its pod is running:'
command 'kubectl get pods -n intelfpgawebhook-system'
out 'Deploy the mappings:'
command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
out 'Check pod logs:'
command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system"
}
screen4()
{ {
clear clear
cd /srv/demo cd /srv/demo
sudo rm -rf /srv/intel.com/fpga/Arria10.dcp1.2 /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6 sudo rm -rf /srv/intel.com/fpga/Arria10.dcp1.2 /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6
out '4. Create bistream storage' out '3. Create bistream storage'
out 'Create directory for Arria10.dcp1.2 interface id:' out 'Create directory for Arria10.dcp1.2 interface id:'
command 'sudo mkdir -p /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6' command 'sudo mkdir -p /srv/intel.com/fpga/69528db6eb31577a8c3668f9faa081f6'
out 'Create Arria10.dcp1.2 symlink for convenience:' out 'Create Arria10.dcp1.2 symlink for convenience:'
@ -116,30 +95,30 @@ screen4()
command 'ls -la /srv/intel.com/fpga/Arria10.dcp1.2/' command 'ls -la /srv/intel.com/fpga/Arria10.dcp1.2/'
} }
screen5() screen4()
{ {
clear clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '5. Deploy FPGA plugin' out '4. Deploy FPGA plugin'
out 'Create a service account for the plugin' command 'kubectl apply -k deployments/fpga_plugin/overlays/region'
command 'kubectl create -f deployments/fpga_plugin/fpga_plugin_service_account.yaml' sleep 2
out 'Set region mode for the plugin:' out 'Check if its pods are running:'
command "kubectl annotate node --all 'fpga.intel.com/device-plugin-mode=region'" command 'kubectl get pods -n intelfpgaplugin-system'
out 'Create plugin daemonset:' out 'Deploy the mappings:'
command 'kubectl create -f deployments/fpga_plugin/fpga_plugin.yaml' command 'kubectl apply -f deployments/fpga_admissionwebhook/mappings-collection.yaml'
out 'Check if its pod is runnning:' out 'Check webhook pod logs:'
command 'kubectl get pod --namespace kube-system |grep intel-fpga-plugin' command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
out 'Check if it runs in 'region' mode:' out 'Check if the plugin runs in 'region' mode:'
command "kubectl logs $(kubectl get pods --namespace kube-system |grep intel-fpga-plugin|cut -f1 -d' ') --namespace kube-system" command "kubectl logs $(kubectl get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
out 'Check if resource fpga.intel.com/region-<FPGA interface id> is allocatable:' out 'Check if resource fpga.intel.com/region-<FPGA interface id> is allocatable:'
command 'kubectl describe node |grep -A5 Allocatable' command 'kubectl describe node |grep -A5 Allocatable'
} }
screen6() screen5()
{ {
clear clear
cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
out '6. Run OPAE workload that uses NLB3 bitstream' out '5. Run OPAE workload that uses NLB3 bitstream'
out 'Program devices with a NLB0 bitstream that is not wanted by the workload:' out 'Program devices with a NLB0 bitstream that is not wanted by the workload:'
command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.0 pr" command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.0 pr"
command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.1 pr" command "sudo /opt/intel/fpga-sw/fpgatool -b /srv/intel.com/fpga/Arria10.dcp1.2/nlb0.gbs -d ${DEVICE_PREFIX}.1 pr"
@ -154,7 +133,7 @@ screen6()
command 'cat /sys/class/*/*/*/afu_id' command 'cat /sys/class/*/*/*/afu_id'
} }
screen7() screen6()
{ {
clear clear
out 'Summary:' 15 out 'Summary:' 15
@ -171,7 +150,7 @@ if [ "$1" == 'play' ] ; then
if [ -n "$2" ] ; then if [ -n "$2" ] ; then
screen$2 screen$2
else else
for n in $(seq 7) ; do screen$n ; sleep 3; done for n in $(seq 6) ; do screen$n ; sleep 3; done
fi fi
elif [ "$1" == 'cleanup' ] ; then elif [ "$1" == 'cleanup' ] ; then
cleanup cleanup

40
demo/screencast-fpga-preprogrammed.sh
View File

@ -39,8 +39,7 @@ cleanup()
out 'Cleanup demo artifacts' 200 out 'Cleanup demo artifacts' 200
command 'kubectl delete pod test-fpga-preprogrammed || true' 200 command 'kubectl delete pod test-fpga-preprogrammed || true' 200
command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200 command 'kubectl delete -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml || true' 200
command 'kubectl delete -k plugins/deployments/fpga_admissionwebhook/default || true' 200 command 'kubectl delete namespace intelfpgaplugin-system || true' 20
command './plugins/scripts/deploy-fpgaplugin.sh cleanup' 200
command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode-' 200 command 'kubectl annotate node --all fpga.intel.com/device-plugin-mode-' 200
command 'rm -rf plugins' 200 command 'rm -rf plugins' 200
} }
@ -66,7 +65,7 @@ screen1()
sleep 2 sleep 2
out 'Check if docker is running k8s pods:' out 'Check if docker is running k8s pods:'
command 'docker ps --format "table {{.Names}}"' command 'docker ps --format "table {{.Names}}"'
sleep 2 sleep 1
} }
screen2() screen2()
@ -75,43 +74,35 @@ screen2()
rm -rf plugins rm -rf plugins
out '2. Clone Intel Device Plugins for Kubernetes repository' out '2. Clone Intel Device Plugins for Kubernetes repository'
command "git clone https://github.com/intel/intel-device-plugins-for-kubernetes plugins" 15 command "git clone https://github.com/intel/intel-device-plugins-for-kubernetes plugins" 15
sleep 1
} }
screen3() screen3()
{ {
clear clear
out '3. Deploy admission controller webhook' out '3. Deploy FPGA plugin'
command 'kubectl apply -k plugins/deployments/fpga_admissionwebhook/default' command 'kubectl apply -k plugins/deployments/fpga_plugin/overlays/af'
sleep 3 sleep 3
out 'Deploy example mappings:' out 'Deploy example mappings:'
command 'kubectl apply -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml' command 'kubectl apply -f plugins/deployments/fpga_admissionwebhook/mappings-collection.yaml'
sleep 3 sleep 3
out 'Check if webhook pod is running:' out 'Check if the plugin pods are running:'
command 'kubectl get pods -n intelfpgawebhook-system' command 'kubectl get pods -n intelfpgaplugin-system'
sleep 2 sleep 2
out 'Check its logs:' out 'Check webhook pod logs:'
command "kubectl logs $(kubectl get pods -n intelfpgawebhook-system| grep intelfpgawebhook-controller-manager | awk '{print $1}') -n intelfpgawebhook-system" command "kubectl logs $(kubectl get pods -n intelfpgaplugin-system| grep intelfpgaplugin-webhook | awk '{print $1}') -n intelfpgaplugin-system"
sleep 2 out 'Check if the plugin runs in 'region' mode:'
} command "kubectl logs $(kubectl get pods --namespace intelfpgaplugin-system |grep fpgadeviceplugin|cut -f1 -d' ') --namespace intelfpgaplugin-system"
screen4()
{
clear
out '5. Deploy FPGA plugin'
command './plugins/scripts/deploy-fpgaplugin.sh'
sleep 4
out 'Check if its pod is runnning:'
command 'kubectl get pod | grep intel-fpga-plugin'
sleep 2 sleep 2
out 'Check if resource fpga.intel.com/af-<af id> is allocatable:' out 'Check if resource fpga.intel.com/af-<af id> is allocatable:'
command 'kubectl describe node |grep -A4 Allocatable' command 'kubectl describe node |grep -A4 Allocatable'
sleep 2 sleep 2
} }
screen5() screen4()
{ {
clear clear
out '6. Run OPAE workload that uses NLB3 bitstream' out '4. Run OPAE workload that uses NLB3 bitstream'
out 'Check if devices are programmed with NLB3:' out 'Check if devices are programmed with NLB3:'
command 'cat /sys/class/*/*/*/afu_id' command 'cat /sys/class/*/*/*/afu_id'
out 'Run workload:' out 'Run workload:'
@ -120,9 +111,10 @@ screen5()
sleep 5 sleep 5
out 'Look at the test output' out 'Look at the test output'
command 'kubectl logs test-fpga-preprogrammed' command 'kubectl logs test-fpga-preprogrammed'
sleep 2
} }
screen6() screen5()
{ {
clear clear
out 'Summary:' 15 out 'Summary:' 15
@ -138,7 +130,7 @@ if [ "$1" == 'play' ] ; then
if [ -n "$2" ] ; then if [ -n "$2" ] ; then
screen$2 screen$2
else else
for n in $(seq 6) ; do screen$n ; sleep 3; done for n in $(seq 5) ; do screen$n ; sleep 3; done
fi fi
elif [ "$1" == 'cleanup' ] ; then elif [ "$1" == 'cleanup' ] ; then
cleanup cleanup

40
deployments/fpga_admissionwebhook/base/kustomization.yaml Normal file
View File

@ -0,0 +1,40 @@
bases:
- ../crd
- ../rbac
- ../manager
- ../webhook
- ../certmanager
patchesStrategicMerge:
# Enable webhook
- manager_webhook_patch.yaml
# Enable certmanager integration
- webhookcainjection_patch.yaml
vars:
- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
fieldref:
fieldpath: metadata.namespace
- name: CERTIFICATE_NAME
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
- name: SERVICE_NAMESPACE # namespace of the service
objref:
kind: Service
version: v1
name: webhook-service
fieldref:
fieldpath: metadata.namespace
- name: SERVICE_NAME
objref:
kind: Service
version: v1
name: webhook-service

2
deployments/fpga_admissionwebhook/default/manager_webhook_patch.yaml → deployments/fpga_admissionwebhook/base/manager_webhook_patch.yaml
View File

@ -1,7 +1,7 @@
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: controller-manager name: webhook
namespace: system namespace: system
spec: spec:
template: template:

0
deployments/fpga_admissionwebhook/default/webhookcainjection_patch.yaml → deployments/fpga_admissionwebhook/base/webhookcainjection_patch.yaml
View File

41
deployments/fpga_admissionwebhook/default/kustomization.yaml
View File

@ -13,42 +13,7 @@ namePrefix: intelfpgawebhook-
# someName: someValue # someName: someValue
bases: bases:
- ../crd - ../base
- ../rbac
- ../manager
- ../webhook
- ../certmanager
patchesStrategicMerge: resources:
# Enable webhook - namespace.yaml
- manager_webhook_patch.yaml
# Enable certmanager integration
- webhookcainjection_patch.yaml
vars:
- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
fieldref:
fieldpath: metadata.namespace
- name: CERTIFICATE_NAME
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
- name: SERVICE_NAMESPACE # namespace of the service
objref:
kind: Service
version: v1
name: webhook-service
fieldref:
fieldpath: metadata.namespace
- name: SERVICE_NAME
objref:
kind: Service
version: v1
name: webhook-service

6
deployments/fpga_admissionwebhook/default/namespace.yaml Normal file
View File

@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: system

9
deployments/fpga_admissionwebhook/manager/manager.yaml
View File

@ -1,14 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: system
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: controller-manager name: webhook
namespace: system namespace: system
labels: labels:
control-plane: controller-manager control-plane: controller-manager

28
deployments/fpga_plugin/base/intel-fpga-plugin-controller-serviceaccount.yaml
View File

@ -1,28 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: intel-fpga-plugin-controller
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
namespace: kube-system
subjects:
- kind: ServiceAccount
name: intel-fpga-plugin-controller
namespace: kube-system
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

10
deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml
View File

@ -1,8 +1,13 @@
apiVersion: v1
kind: Namespace
metadata:
name: system
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: DaemonSet kind: DaemonSet
metadata: metadata:
name: intel-fpga-plugin name: fpgadeviceplugin
namespace: kube-system namespace: system
labels: labels:
app: intel-fpga-plugin app: intel-fpga-plugin
spec: spec:
@ -14,7 +19,6 @@ spec:
labels: labels:
app: intel-fpga-plugin app: intel-fpga-plugin
spec: spec:
serviceAccountName: intel-fpga-plugin-controller
initContainers: initContainers:
- name: intel-fpga-initcontainer - name: intel-fpga-initcontainer
image: intel/intel-fpga-initcontainer:devel image: intel/intel-fpga-initcontainer:devel

3
deployments/fpga_plugin/base/kustomization.yaml
View File

@ -1,3 +1,4 @@
resources: resources:
- intel-fpga-plugin-controller-serviceaccount.yaml - role.yaml
- role_binding.yaml
- intel-fpga-plugin-daemonset.yaml - intel-fpga-plugin-daemonset.yaml

8
deployments/fpga_plugin/base/role.yaml Normal file
View File

@ -0,0 +1,8 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]

12
deployments/fpga_plugin/base/role_binding.yaml Normal file
View File

@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
subjects:
- kind: ServiceAccount
name: default
namespace: system
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

67
deployments/fpga_plugin/fpga_plugin.yaml
View File

@ -1,67 +0,0 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: intel-fpga-plugin
namespace: {namespace}
labels:
app: intel-fpga-plugin
spec:
selector:
matchLabels:
app: intel-fpga-plugin
template:
metadata:
labels:
app: intel-fpga-plugin
spec:
serviceAccountName: intel-fpga-plugin-controller
initContainers:
- name: intel-fpga-initcontainer
image: intel/intel-fpga-initcontainer:devel
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /opt/intel/fpga-sw
name: intel-fpga-sw
- mountPath: /etc/containers/oci/hooks.d
name: oci-hooks-config
containers:
- name: intel-fpga-plugin
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
image: intel/intel-fpga-plugin:devel
imagePullPolicy: IfNotPresent
terminationMessagePath: /tmp/termination-log
securityContext:
readOnlyRootFilesystem: true
volumeMounts:
- name: devfs
mountPath: /dev
readOnly: true
- name: sysfs
mountPath: /sys/class
readOnly: true
- name: kubeletsockets
mountPath: /var/lib/kubelet/device-plugins
volumes:
- name: devfs
hostPath:
path: /dev
- name: sysfs
hostPath:
path: /sys/class
- name: kubeletsockets
hostPath:
path: /var/lib/kubelet/device-plugins
- name: intel-fpga-sw
hostPath:
path: /opt/intel/fpga-sw
type: DirectoryOrCreate
- name: oci-hooks-config
hostPath:
path: /etc/containers/oci/hooks.d
type: DirectoryOrCreate

28
deployments/fpga_plugin/fpga_plugin_service_account.yaml
View File

@ -1,28 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: intel-fpga-plugin-controller
namespace: {namespace}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: node-getter
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: get-nodes
namespace: kube-system
subjects:
- kind: ServiceAccount
name: intel-fpga-plugin-controller
namespace: {namespace}
roleRef:
kind: ClusterRole
name: node-getter
apiGroup: rbac.authorization.k8s.io

5
deployments/fpga_plugin/overlays/af/kustomization.yaml
View File

@ -1,3 +1,6 @@
namespace: intelfpgaplugin-system
namePrefix: intelfpgaplugin-
bases: bases:
- ../../base - ../../base
- ../../../fpga_admissionwebhook/overlays/preprogrammed - ../../../fpga_admissionwebhook/base

8
deployments/fpga_plugin/overlays/region/change-mode-region.yaml
View File

@ -1,8 +0,0 @@
# make sure to change only the -mode=af argument
- op: test
path: /spec/template/spec/containers/0/args/0
value: -mode=af
- op: replace
path: /spec/template/spec/containers/0/args/0
value: -mode=region

15
deployments/fpga_plugin/overlays/region/kustomization.yaml
View File

@ -1,12 +1,9 @@
namespace: intelfpgaplugin-system
namePrefix: intelfpgaplugin-
bases: bases:
- ../../base - ../../base
- ../../../fpga_admissionwebhook/overlays/orchestrated - ../../../fpga_admissionwebhook/base
patchesJson6902: patchesStrategicMerge:
- target: - mode-region.yaml
group: apps
version: v1
kind: DaemonSet
name: intel-fpga-plugin
path:
change-mode-region.yaml

12
deployments/fpga_plugin/overlays/region/mode-region.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fpgadeviceplugin
namespace: system
spec:
template:
spec:
containers:
- name: intel-fpga-plugin
args:
- -mode=region

73
scripts/deploy-fpgaplugin.sh
View File

@ -1,73 +0,0 @@
#!/bin/sh -eu
srcroot="$(realpath $(dirname $0)/..)"
kubectl='kubectl'
namespace='default'
mode='af'
command=''
help() {
echo "Usage: $1 <options> [help|cleanup]"
echo ' Command "help" prints this message'
echo ' Command "cleanup" removes admission webhook deployment'
echo ''
echo ' If no command is given the script will deploy the webhook'
echo ''
echo ' Options:'
echo ' --kubectl <kubectl> - path to the kubectl utility'
echo ' --mode <mode> - "af" (default) or "region" mode of operation'
echo ' --namespace <name> - namespace to deploy the plugin in'
}
while [ $# -gt 0 ]; do
case ${1} in
--kubectl)
kubectl="$2"
shift
;;
--mode)
mode="$2"
shift
;;
--namespace)
namespace="$2"
shift
;;
help)
help $(basename $0)
exit 0
;;
cleanup)
command="cleanup"
;;
*)
echo "Unknown option: ${1}"
exit 1
;;
esac
shift
done
which ${kubectl} > /dev/null 2>&1 || { echo "ERROR: ${kubectl} not found"; exit 1; }
echo "Clean up previously created deployment"
${kubectl} annotate node --all fpga.intel.com/device-plugin-mode- || true
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin.yaml | ${kubectl} --namespace ${namespace} delete -f - || true
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin_service_account.yaml | ${kubectl} --namespace ${namespace} delete -f - || true
if [ "x${command}" = "xcleanup" ]; then
echo "Cleanup done. Exiting..."
exit 0
fi
if [ "x${mode}" != "xaf" -a "x${mode}" != "xregion" ]; then
echo "ERROR: supported modes are 'af' and 'region'"
exit 1
fi
echo 'Set default operation mode'
${kubectl} annotate node --overwrite --all fpga.intel.com/device-plugin-mode=${mode} || true
echo 'Create service account'
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin_service_account.yaml | ${kubectl} --namespace ${namespace} create -f - || true
echo 'Create plugin daemonset'
sed -e "s/{namespace}/${namespace}/g" ${srcroot}/deployments/fpga_plugin/fpga_plugin.yaml | ${kubectl} --namespace ${namespace} create -f - || true

51
test/e2e/fpga/fpga.go
View File

@ -18,6 +18,8 @@ package fpga
import ( import (
"context" "context"
"fmt" "fmt"
"io/ioutil"
"os"
"path/filepath" "path/filepath"
"time" "time"
@ -33,13 +35,13 @@ import (
) )
const ( const (
pluginDeployScript = "scripts/deploy-fpgaplugin.sh" pluginKustomizationYaml = "deployments/fpga_plugin/base/kustomization.yaml"
kustomizationYaml = "deployments/fpga_admissionwebhook/default/kustomization.yaml" webhookKustomizationYaml = "deployments/fpga_admissionwebhook/default/kustomization.yaml"
nlb0NodeResource = "fpga.intel.com/af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs" nlb0NodeResource = "fpga.intel.com/af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs"
nlb0PodResource = "fpga.intel.com/arria10.dcp1.2-nlb0-orchestrated" nlb0PodResource = "fpga.intel.com/arria10.dcp1.2-nlb0-orchestrated"
nlb3PodResource = "fpga.intel.com/arria10.dcp1.2-nlb3-orchestrated" nlb3PodResource = "fpga.intel.com/arria10.dcp1.2-nlb3-orchestrated"
nlb0PodResourceAF = "fpga.intel.com/arria10.dcp1.2-nlb0-preprogrammed" nlb0PodResourceAF = "fpga.intel.com/arria10.dcp1.2-nlb0-preprogrammed"
arria10NodeResource = "fpga.intel.com/region-69528db6eb31577a8c3668f9faa081f6" arria10NodeResource = "fpga.intel.com/region-69528db6eb31577a8c3668f9faa081f6"
) )
func init() { func init() {
@ -47,14 +49,14 @@ func init() {
} }
func describe() { func describe() {
kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml) webhookKustomizationPath, err := utils.LocateRepoFile(webhookKustomizationYaml)
if err != nil { if err != nil {
framework.Failf("unable to locate %q: %v", kustomizationYaml, err) framework.Failf("unable to locate %q: %v", webhookKustomizationYaml, err)
} }
pluginDeployScriptPath, err := utils.LocateRepoFile(pluginDeployScript) pluginKustomizationPath, err := utils.LocateRepoFile(pluginKustomizationYaml)
if err != nil { if err != nil {
framework.Failf("unable to locate %q: %v", pluginDeployScript, err) framework.Failf("unable to locate %q: %v", pluginKustomizationYaml, err)
} }
fmw := framework.NewDefaultFramework("fpgaplugin-e2e") fmw := framework.NewDefaultFramework("fpgaplugin-e2e")
@ -62,23 +64,34 @@ func describe() {
ginkgo.It("Run FPGA plugin tests", func() { ginkgo.It("Run FPGA plugin tests", func() {
// Deploy webhook // Deploy webhook
ginkgo.By(fmt.Sprintf("namespace %s: deploying webhook", fmw.Namespace.Name)) ginkgo.By(fmt.Sprintf("namespace %s: deploying webhook", fmw.Namespace.Name))
utils.DeployFpgaWebhook(fmw, kustomizationPath) utils.DeployFpgaWebhook(fmw, webhookKustomizationPath)
ginkgo.By("deploying mappings") ginkgo.By("deploying mappings")
framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-n", fmw.Namespace.Name, "-f", filepath.Dir(kustomizationPath)+"/../mappings-collection.yaml") framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-n", fmw.Namespace.Name, "-f", filepath.Dir(webhookKustomizationPath)+"/../mappings-collection.yaml")
// Run region test case twice to ensure that device is reprogrammed at least once // Run region test case twice to ensure that device is reprogrammed at least once
runTestCase(fmw, pluginDeployScriptPath, "region", arria10NodeResource, nlb3PodResource, "nlb3", "nlb0") runTestCase(fmw, pluginKustomizationPath, "region", arria10NodeResource, nlb3PodResource, "nlb3", "nlb0")
runTestCase(fmw, pluginDeployScriptPath, "region", arria10NodeResource, nlb0PodResource, "nlb0", "nlb3") runTestCase(fmw, pluginKustomizationPath, "region", arria10NodeResource, nlb0PodResource, "nlb0", "nlb3")
// Run af test case // Run af test case
runTestCase(fmw, pluginDeployScriptPath, "af", nlb0NodeResource, nlb0PodResourceAF, "nlb0", "nlb3") runTestCase(fmw, pluginKustomizationPath, "af", nlb0NodeResource, nlb0PodResourceAF, "nlb0", "nlb3")
}) })
} }
func runTestCase(fmw *framework.Framework, pluginDeployScriptPath, pluginMode, nodeResource, podResource, cmd1, cmd2 string) { func runTestCase(fmw *framework.Framework, pluginKustomizationPath, pluginMode, nodeResource, podResource, cmd1, cmd2 string) {
tmpDir, err := ioutil.TempDir("", "fpgaplugine2etest-"+fmw.Namespace.Name)
if err != nil {
framework.Failf("unable to create temp directory: %v", err)
}
defer os.RemoveAll(tmpDir)
err = utils.CreateKustomizationOverlay(fmw.Namespace.Name, filepath.Dir(pluginKustomizationPath)+"/../overlays/"+pluginMode, tmpDir)
if err != nil {
framework.Failf("unable to kustomization overlay: %v", err)
}
ginkgo.By(fmt.Sprintf("namespace %s: deploying FPGA plugin in %s mode", fmw.Namespace.Name, pluginMode)) ginkgo.By(fmt.Sprintf("namespace %s: deploying FPGA plugin in %s mode", fmw.Namespace.Name, pluginMode))
_, _, err := framework.RunCmd(pluginDeployScriptPath, "--mode", pluginMode, "--namespace", fmw.Namespace.Name) _, _ = framework.RunKubectl(fmw.Namespace.Name, "delete", "-k", tmpDir)
framework.ExpectNoError(err) framework.RunKubectlOrDie(fmw.Namespace.Name, "apply", "-k", tmpDir)
waitForPod(fmw, "intel-fpga-plugin") waitForPod(fmw, "intel-fpga-plugin")

5
test/e2e/utils/utils.go
View File

@ -117,7 +117,8 @@ func LocateRepoFile(repopath string) (string, error) {
return "", errors.New("no file found, try to define PLUGINS_REPO_DIR pointing to the root of the repository") return "", errors.New("no file found, try to define PLUGINS_REPO_DIR pointing to the root of the repository")
} }
func createKustomizationOverlay(namespace, base, overlay string) error { // CreateKustomizationOverlay creates an overlay with overridden namespace.
func CreateKustomizationOverlay(namespace, base, overlay string) error {
relPath := "" relPath := ""
for range strings.Split(overlay[1:], "/") { for range strings.Split(overlay[1:], "/") {
relPath = relPath + "../" relPath = relPath + "../"
@ -141,7 +142,7 @@ func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) {
} }
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
err = createKustomizationOverlay(f.Namespace.Name, filepath.Dir(kustomizationPath), tmpDir) err = CreateKustomizationOverlay(f.Namespace.Name, filepath.Dir(kustomizationPath), tmpDir)
if err != nil { if err != nil {
framework.Failf("unable to kustomization overlay: %v", err) framework.Failf("unable to kustomization overlay: %v", err)
} }