mirror of
https://github.com/intel/intel-device-plugins-for-kubernetes.git
synced 2025-06-03 03:59:37 +00:00
fpga: add mode CLI switch
By default the fpga plugin announce regions' interface IDs. With added `-mode af` switch the plugins announces IDs of accelerator functions instead of regions.
This commit is contained in:
Dmitry Rozhkov
parent
7e830d7953
commit
49840e9720
@ -39,12 +39,32 @@ const (
|
||||
devfsDirectory = "/dev"
|
||||
deviceRE = `^intel-fpga-dev.[0-9]+$`
|
||||
portRE = `^intel-fpga-port.[0-9]+$`
|
||||
fmeRE = `^intel-fpga-fme.[0-9]+$`
|
||||
|
||||
// Device plugin settings.
|
||||
pluginEndpointPrefix = "intel-fpga"
|
||||
resourceNamePrefix = "intel.com/fpga"
|
||||
)
|
||||
|
||||
type pluginMode int
|
||||
|
||||
const (
|
||||
wrongMode pluginMode = iota
|
||||
afMode
|
||||
regionMode
|
||||
)
|
||||
|
||||
func parseMode(input string) pluginMode {
|
||||
switch input {
|
||||
case "af":
|
||||
return afMode
|
||||
case "region":
|
||||
return regionMode
|
||||
}
|
||||
|
||||
return wrongMode
|
||||
}
|
||||
|
||||
// deviceManager manages Intel FPGA devices.
|
||||
type deviceManager struct {
|
||||
srv deviceplugin.Server
|
||||
@ -52,21 +72,24 @@ type deviceManager struct {
|
||||
name string
|
||||
devices map[string]deviceplugin.DeviceInfo
|
||||
root string
|
||||
mode pluginMode
|
||||
}
|
||||
|
||||
func newDeviceManager(resourceName string, fpgaId string, rootDir string) *deviceManager {
|
||||
func newDeviceManager(resourceName string, fpgaId string, rootDir string, mode pluginMode) *deviceManager {
|
||||
return &deviceManager{
|
||||
fpgaId: fpgaId,
|
||||
name: resourceName,
|
||||
devices: make(map[string]deviceplugin.DeviceInfo),
|
||||
root: rootDir,
|
||||
mode: mode,
|
||||
}
|
||||
}
|
||||
|
||||
// Discovers all FPGA devices available on the local node by walking `/sys/class/fpga` directory.
|
||||
func discoverFPGAs(sysfsDir string, devfsDir string) (map[string]map[string]deviceplugin.DeviceInfo, error) {
|
||||
func discoverFPGAs(sysfsDir string, devfsDir string, mode pluginMode) (map[string]map[string]deviceplugin.DeviceInfo, error) {
|
||||
deviceReg := regexp.MustCompile(deviceRE)
|
||||
portReg := regexp.MustCompile(portRE)
|
||||
fmeReg := regexp.MustCompile(fmeRE)
|
||||
|
||||
result := make(map[string]map[string]deviceplugin.DeviceInfo)
|
||||
|
||||
@ -78,22 +101,53 @@ func discoverFPGAs(sysfsDir string, devfsDir string) (map[string]map[string]devi
|
||||
for _, fpgaFile := range fpgaFiles {
|
||||
fname := fpgaFile.Name()
|
||||
if deviceReg.MatchString(fname) {
|
||||
var interfaceId string
|
||||
|
||||
deviceFolder := path.Join(sysfsDir, fname)
|
||||
deviceFiles, err := ioutil.ReadDir(deviceFolder)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fpgaNodes := make(map[string][]string)
|
||||
|
||||
if mode == regionMode {
|
||||
for _, deviceFile := range deviceFiles {
|
||||
name := deviceFile.Name()
|
||||
if fmeReg.MatchString(name) {
|
||||
if len(interfaceId) > 0 {
|
||||
return nil, fmt.Errorf("Detected more than one FPGA region for device %s. Only one region per FPGA device is supported", fname)
|
||||
}
|
||||
interfaceIdFile := path.Join(deviceFolder, name, "pr", "interface_id")
|
||||
data, err := ioutil.ReadFile(interfaceIdFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
interfaceId = strings.TrimSpace(string(data))
|
||||
fpgaNodes[interfaceId] = append(fpgaNodes[interfaceId], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, deviceFile := range deviceFiles {
|
||||
name := deviceFile.Name()
|
||||
if portReg.MatchString(name) {
|
||||
afuFile := path.Join(deviceFolder, name, "afu_id")
|
||||
data, err := ioutil.ReadFile(afuFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
switch mode {
|
||||
case regionMode:
|
||||
if len(interfaceId) == 0 {
|
||||
return nil, fmt.Errorf("No FPGA region found for %s", fname)
|
||||
}
|
||||
fpgaNodes[interfaceId] = append(fpgaNodes[interfaceId], name)
|
||||
case afMode:
|
||||
afuFile := path.Join(deviceFolder, name, "afu_id")
|
||||
data, err := ioutil.ReadFile(afuFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
afuID := strings.TrimSpace(string(data))
|
||||
fpgaNodes[afuID] = append(fpgaNodes[afuID], name)
|
||||
default:
|
||||
glog.Fatal("Unsupported mode")
|
||||
}
|
||||
afuID := strings.TrimSpace(string(data))
|
||||
fpgaNodes[afuID] = append(fpgaNodes[afuID], name)
|
||||
}
|
||||
}
|
||||
if len(fpgaNodes) == 0 {
|
||||
@ -133,7 +187,7 @@ func (dm *deviceManager) ListAndWatch(empty *pluginapi.Empty, stream pluginapi.D
|
||||
sysfsDir := path.Join(dm.root, sysfsDirectory)
|
||||
devfsDir := path.Join(dm.root, devfsDirectory)
|
||||
for {
|
||||
devs, err := discoverFPGAs(sysfsDir, devfsDir)
|
||||
devs, err := discoverFPGAs(sysfsDir, devfsDir, dm.mode)
|
||||
if err != nil {
|
||||
dm.srv.Stop()
|
||||
return fmt.Errorf("Device discovery failed: %+v", err)
|
||||
@ -173,10 +227,19 @@ func (dm *deviceManager) PreStartContainer(ctx context.Context, rqt *pluginapi.P
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
fmt.Println("FPGA device plugin started")
|
||||
var modeStr string
|
||||
|
||||
devs, err := discoverFPGAs(sysfsDirectory, devfsDirectory)
|
||||
flag.StringVar(&modeStr, "mode", "af", "device plugin mode: 'af' (default) or 'region'")
|
||||
flag.Parse()
|
||||
|
||||
mode := parseMode(modeStr)
|
||||
if mode == wrongMode {
|
||||
glog.Fatal("Wrong mode: ", modeStr)
|
||||
}
|
||||
|
||||
fmt.Println("FPGA device plugin started in", modeStr, "mode")
|
||||
|
||||
devs, err := discoverFPGAs(sysfsDirectory, devfsDirectory, mode)
|
||||
if err != nil {
|
||||
glog.Fatalf("Device discovery failed: %+v", err)
|
||||
}
|
||||
@ -190,7 +253,7 @@ func main() {
|
||||
for fpgaId, _ := range devs {
|
||||
resourceName := resourceNamePrefix + "-" + fpgaId
|
||||
pPrefix := pluginEndpointPrefix + "-" + fpgaId
|
||||
dm := newDeviceManager(resourceName, fpgaId, "/")
|
||||
dm := newDeviceManager(resourceName, fpgaId, "/", mode)
|
||||
|
||||
go func() {
|
||||
ch <- dm.srv.Serve(dm, resourceName, pPrefix)
|
||||
|
||||
@ -66,20 +66,24 @@ func TestDiscoverFPGAs(t *testing.T) {
|
||||
sysfsfiles map[string][]byte
|
||||
expectedResult map[string]map[string]deviceplugin.DeviceInfo
|
||||
expectedErr bool
|
||||
mode pluginMode
|
||||
}{
|
||||
{
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: afMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{"intel-fpga-dev.0"},
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: afMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-port.0"},
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: afMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{
|
||||
@ -90,6 +94,7 @@ func TestDiscoverFPGAs(t *testing.T) {
|
||||
},
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: afMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{
|
||||
@ -135,6 +140,81 @@ func TestDiscoverFPGAs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expectedErr: false,
|
||||
mode: afMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{
|
||||
"intel-fpga-dev.0/intel-fpga-fme.0/pr",
|
||||
"intel-fpga-dev.0/intel-fpga-fme.1/pr",
|
||||
},
|
||||
sysfsfiles: map[string][]byte{
|
||||
"intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"),
|
||||
"intel-fpga-dev.0/intel-fpga-fme.1/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"),
|
||||
},
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: regionMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{
|
||||
"intel-fpga-dev.0/intel-fpga-port.0",
|
||||
"intel-fpga-dev.1/intel-fpga-port.1",
|
||||
},
|
||||
expectedResult: nil,
|
||||
expectedErr: true,
|
||||
mode: regionMode,
|
||||
},
|
||||
{
|
||||
sysfsdirs: []string{
|
||||
"intel-fpga-dev.0/intel-fpga-port.0",
|
||||
"intel-fpga-dev.0/intel-fpga-fme.0/pr",
|
||||
"intel-fpga-dev.1/intel-fpga-port.1",
|
||||
"intel-fpga-dev.1/intel-fpga-fme.1/pr",
|
||||
"intel-fpga-dev.2/intel-fpga-port.2",
|
||||
"intel-fpga-dev.2/intel-fpga-fme.2/pr",
|
||||
},
|
||||
sysfsfiles: map[string][]byte{
|
||||
"intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"),
|
||||
"intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"),
|
||||
"intel-fpga-dev.2/intel-fpga-port.2/afu_id": []byte("47595d0fae972fbed0c51b4a41c7a349\n"),
|
||||
"intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"),
|
||||
"intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"),
|
||||
"intel-fpga-dev.2/intel-fpga-fme.2/pr/interface_id": []byte("fd967345645f05f338462a0748be0091\n"),
|
||||
},
|
||||
devfsdirs: []string{
|
||||
"intel-fpga-port.0", "intel-fpga-fme.0",
|
||||
"intel-fpga-port.1", "intel-fpga-fme.1",
|
||||
"intel-fpga-port.2", "intel-fpga-fme.2",
|
||||
},
|
||||
expectedResult: map[string]map[string]deviceplugin.DeviceInfo{
|
||||
"ce48969398f05f33946d560708be108a": map[string]deviceplugin.DeviceInfo{
|
||||
"intel-fpga-dev.0": deviceplugin.DeviceInfo{
|
||||
State: "Healthy",
|
||||
Nodes: []string{
|
||||
path.Join(tmpdir, "/dev/intel-fpga-fme.0"),
|
||||
path.Join(tmpdir, "/dev/intel-fpga-port.0"),
|
||||
},
|
||||
},
|
||||
"intel-fpga-dev.1": deviceplugin.DeviceInfo{
|
||||
State: "Healthy",
|
||||
Nodes: []string{
|
||||
path.Join(tmpdir, "/dev/intel-fpga-fme.1"),
|
||||
path.Join(tmpdir, "/dev/intel-fpga-port.1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
"fd967345645f05f338462a0748be0091": map[string]deviceplugin.DeviceInfo{
|
||||
"intel-fpga-dev.2": deviceplugin.DeviceInfo{
|
||||
State: "Healthy",
|
||||
Nodes: []string{
|
||||
path.Join(tmpdir, "/dev/intel-fpga-fme.2"),
|
||||
path.Join(tmpdir, "/dev/intel-fpga-port.2"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedErr: false,
|
||||
mode: regionMode,
|
||||
},
|
||||
}
|
||||
|
||||
@ -144,7 +224,7 @@ func TestDiscoverFPGAs(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
result, err := discoverFPGAs(sysfs, devfs)
|
||||
result, err := discoverFPGAs(sysfs, devfs, tcase.mode)
|
||||
if tcase.expectedErr && err == nil {
|
||||
t.Error("Expected error hasn't been triggered")
|
||||
}
|
||||
@ -257,7 +337,7 @@ func TestListAndWatch(t *testing.T) {
|
||||
}
|
||||
|
||||
resourceName := resourceNamePrefix + "-" + afuID
|
||||
testDM := newDeviceManager(resourceName, afuID, tmpdir)
|
||||
testDM := newDeviceManager(resourceName, afuID, tmpdir, afMode)
|
||||
if testDM == nil {
|
||||
t.Fatal("Failed to create a deviceManager")
|
||||
}
|
||||
@ -306,7 +386,7 @@ func TestListAndWatch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAllocate(t *testing.T) {
|
||||
testDM := newDeviceManager("", "", "")
|
||||
testDM := newDeviceManager("", "", "", afMode)
|
||||
if testDM == nil {
|
||||
t.Fatal("Failed to create a deviceManager")
|
||||
}
|
||||
@ -329,3 +409,29 @@ func TestAllocate(t *testing.T) {
|
||||
t.Fatal("Allocated wrong number of devices")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMode(t *testing.T) {
|
||||
tcases := []struct {
|
||||
input string
|
||||
output pluginMode
|
||||
}{
|
||||
{
|
||||
input: "af",
|
||||
output: afMode,
|
||||
},
|
||||
{
|
||||
input: "region",
|
||||
output: regionMode,
|
||||
},
|
||||
{
|
||||
input: "unparsable",
|
||||
output: wrongMode,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tcase := range tcases {
|
||||
if parseMode(tcase.input) != tcase.output {
|
||||
t.Error("Wrong output", tcase.output, "for the given input", tcase.input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user