diff --git a/cmd/fpga_plugin/fpga_plugin.go b/cmd/fpga_plugin/fpga_plugin.go index c7ef5b45..f1985f2d 100644 --- a/cmd/fpga_plugin/fpga_plugin.go +++ b/cmd/fpga_plugin/fpga_plugin.go @@ -39,12 +39,32 @@ const ( devfsDirectory = "/dev" deviceRE = `^intel-fpga-dev.[0-9]+$` portRE = `^intel-fpga-port.[0-9]+$` + fmeRE = `^intel-fpga-fme.[0-9]+$` // Device plugin settings. pluginEndpointPrefix = "intel-fpga" resourceNamePrefix = "intel.com/fpga" ) +type pluginMode int + +const ( + wrongMode pluginMode = iota + afMode + regionMode +) + +func parseMode(input string) pluginMode { + switch input { + case "af": + return afMode + case "region": + return regionMode + } + + return wrongMode +} + // deviceManager manages Intel FPGA devices. type deviceManager struct { srv deviceplugin.Server @@ -52,21 +72,24 @@ type deviceManager struct { name string devices map[string]deviceplugin.DeviceInfo root string + mode pluginMode } -func newDeviceManager(resourceName string, fpgaId string, rootDir string) *deviceManager { +func newDeviceManager(resourceName string, fpgaId string, rootDir string, mode pluginMode) *deviceManager { return &deviceManager{ fpgaId: fpgaId, name: resourceName, devices: make(map[string]deviceplugin.DeviceInfo), root: rootDir, + mode: mode, } } // Discovers all FPGA devices available on the local node by walking `/sys/class/fpga` directory. -func discoverFPGAs(sysfsDir string, devfsDir string) (map[string]map[string]deviceplugin.DeviceInfo, error) { +func discoverFPGAs(sysfsDir string, devfsDir string, mode pluginMode) (map[string]map[string]deviceplugin.DeviceInfo, error) { deviceReg := regexp.MustCompile(deviceRE) portReg := regexp.MustCompile(portRE) + fmeReg := regexp.MustCompile(fmeRE) result := make(map[string]map[string]deviceplugin.DeviceInfo) @@ -78,22 +101,53 @@ func discoverFPGAs(sysfsDir string, devfsDir string) (map[string]map[string]devi for _, fpgaFile := range fpgaFiles { fname := fpgaFile.Name() if deviceReg.MatchString(fname) { + var interfaceId string + deviceFolder := path.Join(sysfsDir, fname) deviceFiles, err := ioutil.ReadDir(deviceFolder) if err != nil { return nil, err } fpgaNodes := make(map[string][]string) + + if mode == regionMode { + for _, deviceFile := range deviceFiles { + name := deviceFile.Name() + if fmeReg.MatchString(name) { + if len(interfaceId) > 0 { + return nil, fmt.Errorf("Detected more than one FPGA region for device %s. Only one region per FPGA device is supported", fname) + } + interfaceIdFile := path.Join(deviceFolder, name, "pr", "interface_id") + data, err := ioutil.ReadFile(interfaceIdFile) + if err != nil { + return nil, err + } + interfaceId = strings.TrimSpace(string(data)) + fpgaNodes[interfaceId] = append(fpgaNodes[interfaceId], name) + } + } + } + for _, deviceFile := range deviceFiles { name := deviceFile.Name() if portReg.MatchString(name) { - afuFile := path.Join(deviceFolder, name, "afu_id") - data, err := ioutil.ReadFile(afuFile) - if err != nil { - return nil, err + switch mode { + case regionMode: + if len(interfaceId) == 0 { + return nil, fmt.Errorf("No FPGA region found for %s", fname) + } + fpgaNodes[interfaceId] = append(fpgaNodes[interfaceId], name) + case afMode: + afuFile := path.Join(deviceFolder, name, "afu_id") + data, err := ioutil.ReadFile(afuFile) + if err != nil { + return nil, err + } + afuID := strings.TrimSpace(string(data)) + fpgaNodes[afuID] = append(fpgaNodes[afuID], name) + default: + glog.Fatal("Unsupported mode") } - afuID := strings.TrimSpace(string(data)) - fpgaNodes[afuID] = append(fpgaNodes[afuID], name) } } if len(fpgaNodes) == 0 { @@ -133,7 +187,7 @@ func (dm *deviceManager) ListAndWatch(empty *pluginapi.Empty, stream pluginapi.D sysfsDir := path.Join(dm.root, sysfsDirectory) devfsDir := path.Join(dm.root, devfsDirectory) for { - devs, err := discoverFPGAs(sysfsDir, devfsDir) + devs, err := discoverFPGAs(sysfsDir, devfsDir, dm.mode) if err != nil { dm.srv.Stop() return fmt.Errorf("Device discovery failed: %+v", err) @@ -173,10 +227,19 @@ func (dm *deviceManager) PreStartContainer(ctx context.Context, rqt *pluginapi.P } func main() { - flag.Parse() - fmt.Println("FPGA device plugin started") + var modeStr string - devs, err := discoverFPGAs(sysfsDirectory, devfsDirectory) + flag.StringVar(&modeStr, "mode", "af", "device plugin mode: 'af' (default) or 'region'") + flag.Parse() + + mode := parseMode(modeStr) + if mode == wrongMode { + glog.Fatal("Wrong mode: ", modeStr) + } + + fmt.Println("FPGA device plugin started in", modeStr, "mode") + + devs, err := discoverFPGAs(sysfsDirectory, devfsDirectory, mode) if err != nil { glog.Fatalf("Device discovery failed: %+v", err) } @@ -190,7 +253,7 @@ func main() { for fpgaId, _ := range devs { resourceName := resourceNamePrefix + "-" + fpgaId pPrefix := pluginEndpointPrefix + "-" + fpgaId - dm := newDeviceManager(resourceName, fpgaId, "/") + dm := newDeviceManager(resourceName, fpgaId, "/", mode) go func() { ch <- dm.srv.Serve(dm, resourceName, pPrefix) diff --git a/cmd/fpga_plugin/fpga_plugin_test.go b/cmd/fpga_plugin/fpga_plugin_test.go index 25a9eb6f..9d7d45d1 100644 --- a/cmd/fpga_plugin/fpga_plugin_test.go +++ b/cmd/fpga_plugin/fpga_plugin_test.go @@ -66,20 +66,24 @@ func TestDiscoverFPGAs(t *testing.T) { sysfsfiles map[string][]byte expectedResult map[string]map[string]deviceplugin.DeviceInfo expectedErr bool + mode pluginMode }{ { expectedResult: nil, expectedErr: true, + mode: afMode, }, { sysfsdirs: []string{"intel-fpga-dev.0"}, expectedResult: nil, expectedErr: true, + mode: afMode, }, { sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-port.0"}, expectedResult: nil, expectedErr: true, + mode: afMode, }, { sysfsdirs: []string{ @@ -90,6 +94,7 @@ func TestDiscoverFPGAs(t *testing.T) { }, expectedResult: nil, expectedErr: true, + mode: afMode, }, { sysfsdirs: []string{ @@ -135,6 +140,81 @@ func TestDiscoverFPGAs(t *testing.T) { }, }, expectedErr: false, + mode: afMode, + }, + { + sysfsdirs: []string{ + "intel-fpga-dev.0/intel-fpga-fme.0/pr", + "intel-fpga-dev.0/intel-fpga-fme.1/pr", + }, + sysfsfiles: map[string][]byte{ + "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "intel-fpga-dev.0/intel-fpga-fme.1/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + }, + expectedResult: nil, + expectedErr: true, + mode: regionMode, + }, + { + sysfsdirs: []string{ + "intel-fpga-dev.0/intel-fpga-port.0", + "intel-fpga-dev.1/intel-fpga-port.1", + }, + expectedResult: nil, + expectedErr: true, + mode: regionMode, + }, + { + sysfsdirs: []string{ + "intel-fpga-dev.0/intel-fpga-port.0", + "intel-fpga-dev.0/intel-fpga-fme.0/pr", + "intel-fpga-dev.1/intel-fpga-port.1", + "intel-fpga-dev.1/intel-fpga-fme.1/pr", + "intel-fpga-dev.2/intel-fpga-port.2", + "intel-fpga-dev.2/intel-fpga-fme.2/pr", + }, + sysfsfiles: map[string][]byte{ + "intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "intel-fpga-dev.2/intel-fpga-port.2/afu_id": []byte("47595d0fae972fbed0c51b4a41c7a349\n"), + "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), + "intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), + "intel-fpga-dev.2/intel-fpga-fme.2/pr/interface_id": []byte("fd967345645f05f338462a0748be0091\n"), + }, + devfsdirs: []string{ + "intel-fpga-port.0", "intel-fpga-fme.0", + "intel-fpga-port.1", "intel-fpga-fme.1", + "intel-fpga-port.2", "intel-fpga-fme.2", + }, + expectedResult: map[string]map[string]deviceplugin.DeviceInfo{ + "ce48969398f05f33946d560708be108a": map[string]deviceplugin.DeviceInfo{ + "intel-fpga-dev.0": deviceplugin.DeviceInfo{ + State: "Healthy", + Nodes: []string{ + path.Join(tmpdir, "/dev/intel-fpga-fme.0"), + path.Join(tmpdir, "/dev/intel-fpga-port.0"), + }, + }, + "intel-fpga-dev.1": deviceplugin.DeviceInfo{ + State: "Healthy", + Nodes: []string{ + path.Join(tmpdir, "/dev/intel-fpga-fme.1"), + path.Join(tmpdir, "/dev/intel-fpga-port.1"), + }, + }, + }, + "fd967345645f05f338462a0748be0091": map[string]deviceplugin.DeviceInfo{ + "intel-fpga-dev.2": deviceplugin.DeviceInfo{ + State: "Healthy", + Nodes: []string{ + path.Join(tmpdir, "/dev/intel-fpga-fme.2"), + path.Join(tmpdir, "/dev/intel-fpga-port.2"), + }, + }, + }, + }, + expectedErr: false, + mode: regionMode, }, } @@ -144,7 +224,7 @@ func TestDiscoverFPGAs(t *testing.T) { t.Error(err) } - result, err := discoverFPGAs(sysfs, devfs) + result, err := discoverFPGAs(sysfs, devfs, tcase.mode) if tcase.expectedErr && err == nil { t.Error("Expected error hasn't been triggered") } @@ -257,7 +337,7 @@ func TestListAndWatch(t *testing.T) { } resourceName := resourceNamePrefix + "-" + afuID - testDM := newDeviceManager(resourceName, afuID, tmpdir) + testDM := newDeviceManager(resourceName, afuID, tmpdir, afMode) if testDM == nil { t.Fatal("Failed to create a deviceManager") } @@ -306,7 +386,7 @@ func TestListAndWatch(t *testing.T) { } func TestAllocate(t *testing.T) { - testDM := newDeviceManager("", "", "") + testDM := newDeviceManager("", "", "", afMode) if testDM == nil { t.Fatal("Failed to create a deviceManager") } @@ -329,3 +409,29 @@ func TestAllocate(t *testing.T) { t.Fatal("Allocated wrong number of devices") } } + +func TestParseMode(t *testing.T) { + tcases := []struct { + input string + output pluginMode + }{ + { + input: "af", + output: afMode, + }, + { + input: "region", + output: regionMode, + }, + { + input: "unparsable", + output: wrongMode, + }, + } + + for _, tcase := range tcases { + if parseMode(tcase.input) != tcase.output { + t.Error("Wrong output", tcase.output, "for the given input", tcase.input) + } + } +}