Merge pull request #96 from bart0sh/PR0039-crihook-use-tools-from-opt-intel

CRI hook: use tools installed by initcontainer
This commit is contained in:
Dmitry Rozhkov 2018-09-04 23:11:25 +03:00 committed by GitHub
commit 868ed2dce9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 280 additions and 98 deletions

View File

@ -21,20 +21,14 @@ $ cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes
$ make fpga_crihook
```
### Install CRI-O hook:
### Download 'Acceleration Stack for Runtime' tarball
```
$ sudo cp cmd/fpga_crihook/fpga_crihook /usr/local/bin/
Download a10_gx_pac_ias_1_1_pv_rte_installer.tar.gz from https://www.intel.com/content/www/us/en/programmable/solutions/acceleration-hub/downloads.html into $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin directory
```
### Configure CRI-O to run the hook:
### Build init container that contains CRI hook and all its dependencies:
```
$ cd $GOPATH/src/github.com/intel/intel-device-plugins-for-kubernetes/deployments/fpga_plugin
$ ./build-initcontainer-image.sh
```
$ sudo cat << EOF > /etc/containers/oci/hooks.d/prestart.json
{
"hook" : "/usr/local/bin/fpga_crihook",
"stage" : [ "prestart" ],
"annotation": [ "intel.com/fpga-region" ]
}
EOF
$ sudo systemctl restart crio
```

View File

@ -21,7 +21,7 @@ import (
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
@ -31,10 +31,12 @@ import (
const (
fpgaBitStreamDirectory = "/srv/intel.com/fpga"
packager = "/opt/intel/fpga-sw/opae/bin/packager"
fpgaconf = "/opt/intel/fpga-sw/opae/fpgaconf-wrapper"
aocl = "/opt/intel/fpga-sw/opencl/aocl-wrapper"
configJSON = "config.json"
fpgaRegionEnv = "FPGA_REGION"
fpgaAfuEnv = "FPGA_AFU"
fpgaBitStreamExt = ".gbs"
fpgaDevRegexp = `\/dev\/intel-fpga-port.(\d)$`
afuIDTemplate = "/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id"
annotationName = "com.intel.fpga.mode"
@ -61,6 +63,89 @@ type fpgaParams struct {
devNum string
}
type fpgaBitStream interface {
validate() error
program() error
}
type opaeBitStream struct {
path string
params *fpgaParams
execer utilsexec.Interface
}
func (bitStream *opaeBitStream) validate() error {
region, afu := bitStream.params.region, bitStream.params.afu
output, err := bitStream.execer.Command(packager, "gbs-info", "--gbs", bitStream.path).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "%s/%s: can't get bitstream info", region, afu)
}
reader := bytes.NewBuffer(output)
content, err := decodeJSONStream(reader)
if err != nil {
return errors.WithMessage(err, fmt.Sprintf("%s/%s: can't decode 'packager gbs-info' output", region, afu))
}
afuImage, ok := content["afu-image"]
if !ok {
return errors.Errorf("%s/%s: 'afu-image' field not found in the 'packager gbs-info' output", region, afu)
}
interfaceUUID, ok := afuImage.(map[string]interface{})["interface-uuid"]
if !ok {
return errors.Errorf("%s/%s: 'interface-uuid' field not found in the 'packager gbs-info' output", region, afu)
}
acceleratorClusters, ok := afuImage.(map[string]interface{})["accelerator-clusters"]
if !ok {
return errors.Errorf("%s/%s: 'accelerator-clusters' field not found in the 'packager gbs-info' output", region, afu)
}
if canonize(interfaceUUID.(string)) != region {
return errors.Errorf("bitstream is not for this device: region(%s) and interface-uuid(%s) don't match", region, interfaceUUID)
}
acceleratorTypeUUID, ok := acceleratorClusters.([]interface{})[0].(map[string]interface{})["accelerator-type-uuid"]
if !ok {
return errors.Errorf("%s/%s: 'accelerator-type-uuid' field not found in the 'packager gbs-info' output", region, afu)
}
if canonize(acceleratorTypeUUID.(string)) != afu {
return errors.Errorf("incorrect bitstream: AFU(%s) and accelerator-type-uuid(%s) don't match", afu, acceleratorTypeUUID)
}
return nil
}
func (bitStream *opaeBitStream) program() error {
output, err := bitStream.execer.Command(fpgaconf, "-S", bitStream.params.devNum, bitStream.path).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "failed to program AFU %s to socket %s, region %s: output: %s", bitStream.params.afu, bitStream.params.devNum, bitStream.params.region, string(output))
}
return nil
}
type openCLBitStream struct {
path string
params *fpgaParams
execer utilsexec.Interface
}
func (bitStream *openCLBitStream) validate() error {
return nil
}
func (bitStream *openCLBitStream) program() error {
output, err := bitStream.execer.Command(aocl, "program", "acl"+bitStream.params.devNum, bitStream.path).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "failed to program AFU %s to socket %s, region %s: output: %s", bitStream.params.afu, bitStream.params.devNum, bitStream.params.region, string(output))
}
return nil
}
func newHookEnv(bitStreamDir string, config string, execer utilsexec.Interface, afuIDTemplate string) *hookEnv {
return &hookEnv{
bitStreamDir,
@ -80,7 +165,7 @@ func (he *hookEnv) getFPGAParams(content map[string]interface{}) (*fpgaParams, e
return nil, errors.New("no 'bundle' field in the configuration")
}
configPath := path.Join(fmt.Sprint(bundle), he.config)
configPath := filepath.Join(fmt.Sprint(bundle), he.config)
configFile, err := os.Open(configPath)
if err != nil {
return nil, errors.WithStack(err)
@ -140,65 +225,26 @@ func (he *hookEnv) getFPGAParams(content map[string]interface{}) (*fpgaParams, e
}
func (he *hookEnv) validateBitStream(params *fpgaParams, fpgaBitStreamPath string) error {
output, err := he.execer.Command("packager", "gbs-info", "--gbs", fpgaBitStreamPath).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "%s/%s: can't get bitstream info", params.region, params.afu)
func (he *hookEnv) getBitStream(params *fpgaParams) (fpgaBitStream, error) {
bitStreamPath := ""
for _, ext := range []string{".gbs", ".aocx"} {
bitStreamPath = filepath.Join(he.bitStreamDir, params.region, params.afu+ext)
_, err := os.Stat(bitStreamPath)
if os.IsNotExist(err) {
continue
}
if err != nil {
return nil, errors.Errorf("%s: stat error: %v", bitStreamPath, err)
}
if ext == ".gbs" {
return &opaeBitStream{bitStreamPath, params, he.execer}, nil
} else if ext == ".aocx" {
return &openCLBitStream{bitStreamPath, params, he.execer}, nil
}
}
reader := bytes.NewBuffer(output)
content, err := decodeJSONStream(reader)
if err != nil {
return errors.WithMessage(err, fmt.Sprintf("%s/%s: can't decode 'packager gbs-info' output", params.region, params.afu))
}
afuImage, ok := content["afu-image"]
if !ok {
return errors.Errorf("%s/%s: 'afu-image' field not found in the 'packager gbs-info' output", params.region, params.afu)
}
interfaceUUID, ok := afuImage.(map[string]interface{})["interface-uuid"]
if !ok {
return errors.Errorf("%s/%s: 'interface-uuid' field not found in the 'packager gbs-info' output", params.region, params.afu)
}
acceleratorClusters, ok := afuImage.(map[string]interface{})["accelerator-clusters"]
if !ok {
return errors.Errorf("%s/%s: 'accelerator-clusters' field not found in the 'packager gbs-info' output", params.region, params.afu)
}
if canonize(interfaceUUID.(string)) != params.region {
return errors.Errorf("bitstream is not for this device: region(%s) and interface-uuid(%s) don't match", params.region, interfaceUUID)
}
acceleratorTypeUUID, ok := acceleratorClusters.([]interface{})[0].(map[string]interface{})["accelerator-type-uuid"]
if !ok {
return errors.Errorf("%s/%s: 'accelerator-type-uuid' field not found in the 'packager gbs-info' output", params.region, params.afu)
}
if canonize(acceleratorTypeUUID.(string)) != params.afu {
return errors.Errorf("incorrect bitstream: AFU(%s) and accelerator-type-uuid(%s) don't match", params.afu, acceleratorTypeUUID)
}
return nil
}
func (he *hookEnv) programBitStream(params *fpgaParams, fpgaBitStreamPath string) error {
output, err := he.execer.Command("fpgaconf", "-S", params.devNum, fpgaBitStreamPath).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "failed to program AFU %s to socket %s, region %s: output: %s", params.afu, params.devNum, params.region, string(output))
}
programmedAfu, err := he.getProgrammedAfu(params.devNum)
if err != nil {
return err
}
if programmedAfu != params.afu {
return errors.Errorf("programmed function %s instead of %s", programmedAfu, params.afu)
}
return nil
return nil, errors.Errorf("%s/%s: bitstream not found", params.region, params.afu)
}
func (he *hookEnv) getProgrammedAfu(deviceNum string) (string, error) {
@ -249,21 +295,30 @@ func (he *hookEnv) process(reader io.Reader) error {
return nil
}
fpgaBitStreamPath := path.Join(he.bitStreamDir, params.region, params.afu+fpgaBitStreamExt)
if _, err = os.Stat(fpgaBitStreamPath); os.IsNotExist(err) {
return errors.Errorf("%s/%s: bitstream is not found", params.region, params.afu)
}
err = he.validateBitStream(params, fpgaBitStreamPath)
bitStream, err := he.getBitStream(params)
if err != nil {
return err
}
err = he.programBitStream(params, fpgaBitStreamPath)
err = bitStream.validate()
if err != nil {
return err
}
err = bitStream.program()
if err != nil {
return err
}
programmedAfu, err = he.getProgrammedAfu(params.devNum)
if err != nil {
return err
}
if programmedAfu != params.afu {
return errors.Errorf("programmed function %s instead of %s", programmedAfu, params.afu)
}
return nil
}

View File

@ -147,7 +147,8 @@ func genFakeActions(fcmd *fakeexec.FakeCmd, num int) []fakeexec.FakeCommandActio
return actions
}
func TestValidateBitstream(t *testing.T) {
func TestValidate(t *testing.T) {
var fpgaBitStreamDir = "testdata/intel.com/fpga"
tcases := []struct {
params *fpgaParams
expectedErr bool
@ -165,14 +166,29 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "d7724dc4a4a3c413f89e433683f9040b"},
expectedErr: false,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
return nil, nil
},
},
},
{
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 1} },
},
},
{
params: &fpgaParams{region: "", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -181,7 +197,9 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -190,7 +208,9 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -199,7 +219,9 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -208,7 +230,9 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "this should not match", afu: ""},
params: &fpgaParams{
region: "ce48969398f05fxxxxxxxxxxxxxxxxxx",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -217,7 +241,9 @@ func TestValidateBitstream(t *testing.T) {
},
},
{
params: &fpgaParams{region: "ce48969398f05f33946d560708be108a", afu: ""},
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -228,7 +254,7 @@ func TestValidateBitstream(t *testing.T) {
{
params: &fpgaParams{
region: "ce48969398f05f33946d560708be108a",
afu: "this should not match"},
afu: "d8424dc4a4a3c413f89e433683f9040b"},
expectedErr: true,
fakeAction: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) {
@ -241,10 +267,15 @@ func TestValidateBitstream(t *testing.T) {
for _, tc := range tcases {
fcmd := fakeexec.FakeCmd{CombinedOutputScript: tc.fakeAction}
execer := fakeexec.FakeExec{CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript))}
he := newHookEnv("", "", &execer, "")
err := he.validateBitStream(tc.params, "")
he := newHookEnv(fpgaBitStreamDir, "", &execer, "")
bitStream, err := he.getBitStream(tc.params)
if err != nil && !tc.expectedErr {
t.Errorf("unexpected error: %+v", err)
t.Errorf("unexpected error: unable to get bitstream: %+v", err)
continue
}
err = bitStream.validate()
if err != nil && !tc.expectedErr {
t.Errorf("unexpected error: bitstream validation failed: %+v", err)
}
}
}
@ -259,7 +290,8 @@ func genFpgaConfAction(he *hookEnv, afuIDTemplate string, returnError bool) fake
}
}
func TestProgramBitStream(t *testing.T) {
func TestProgram(t *testing.T) {
var fpgaBitStreamDir = "testdata/intel.com/fpga"
tcases := []struct {
params *fpgaParams
afuIDTemplate string
@ -276,7 +308,33 @@ func TestProgramBitStream(t *testing.T) {
newAFUIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_f7df405cbd7acf7222f144b0b93acd18",
},
{
params: &fpgaParams{"", "", ""},
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "d7724dc4a4a3c413f89e433683f9040b"},
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
newAFUIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d7724dc4a4a3c413f89e433683f9040b",
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7dfaaacbd7acf7222f144b0b93acd18"},
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
expectedErr: true,
fpgaconfErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "18b7bffa2eb54aa096ef4230dafacb5a"},
expectedErr: true,
fpgaconfErr: true,
},
@ -300,13 +358,20 @@ func TestProgramBitStream(t *testing.T) {
}
for _, tc := range tcases {
he := newHookEnv("", "", nil, tc.afuIDTemplate)
he := newHookEnv(fpgaBitStreamDir, "", nil, tc.afuIDTemplate)
actions := []fakeexec.FakeCombinedOutputAction{genFpgaConfAction(he, tc.newAFUIDTemplate, tc.fpgaconfErr)}
fcmd := fakeexec.FakeCmd{CombinedOutputScript: actions}
he.execer = &fakeexec.FakeExec{CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript))}
err := he.programBitStream(tc.params, "")
bitStream, err := he.getBitStream(tc.params)
if err != nil {
if !tc.expectedErr {
t.Errorf("unexpected error: unable to get bitstream: %+v", err)
}
continue
}
err = bitStream.program()
if err != nil && !tc.expectedErr {
t.Errorf("unexpected error: %+v", err)
t.Errorf("unexpected error: programming bitstream failed: %+v", err)
}
}
}
@ -348,39 +413,71 @@ func TestProcess(t *testing.T) {
},
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-broken-json.json",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-no-annotations.json",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-no-intel-annotation.json",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-incorrect-intel-annotation.json",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-correct.json",
configJSON: "config-no-afu.json",
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-correct.json",
configJSON: "config-correct.json",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-correct.json",
configJSON: "config-non-existing-bitstream.json",
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
expectedErr: true,
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-correct.json",
configJSON: "config-correct.json",
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
@ -402,6 +499,34 @@ func TestProcess(t *testing.T) {
return ioutil.ReadFile("testdata/gbs-info-correct.json")
},
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "d8424dc4a4a3c413f89e433683f9040b"},
stdinJSON: "stdin-correct.json",
configJSON: "config-correct.json",
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
expectedErr: true,
fpgaconfErr: true,
gbsInfoAction: func() ([]byte, error) {
return ioutil.ReadFile("testdata/gbs-info-correct.json")
},
},
{
params: &fpgaParams{
devNum: "0",
region: "ce48969398f05f33946d560708be108a",
afu: "f7df405cbd7acf7222f144b0b93acd18"},
stdinJSON: "stdin-correct.json",
configJSON: "config-correct.json",
afuIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
newAFUIDTemplate: "testdata/sys/class/fpga/intel-fpga-dev.%s/intel-fpga-port.%s/afu_id_d8424dc4a4a3c413f89e433683f9040b",
expectedErr: true,
gbsInfoAction: func() ([]byte, error) {
return ioutil.ReadFile("testdata/gbs-info-correct.json")
},
},
}
for _, tc := range tcases {

View File

@ -7,7 +7,7 @@
"io.kubernetes.pod.namespace": "default",
"io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.pod.uid": "942e94c1-72d3-11e8-b221-c81f66f62fcc",
"com.intel.fpga.mode": "intel.com/fpga-region"
"com.intel.fpga.mode": "fpga.intel.com/region"
},
"bundle": "testdata",
"id": "1c40dd8efd268a47d7fb9f75d00f50c20af49d07d8d3c5fb948e68abb6d5ecf9",

View File

@ -13,6 +13,13 @@ spec:
afuId: f7df405cbd7acf7222f144b0b93acd18
---
apiVersion: fpga.intel.com/v1
kind: AcceleratorFunction
metadata:
name: arria10-compress
spec:
afuId: 18b79ffa2ee54aa096ef4230dafacb5f
---
apiVersion: fpga.intel.com/v1
kind: FpgaRegion
metadata:
name: arria10