From 1f4928790fd964e9e82fc6d35fc094448d8f9ee7 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 5 Feb 2020 17:32:20 +0200 Subject: [PATCH] Implement function for DeviceInfo creation - Made DeviceInfo fields private - Implement NewDeviceInfo constructor --- cmd/fpga_plugin/dfl_test.go | 255 ++++++++++++-------------- cmd/fpga_plugin/fpga_plugin.go | 27 +-- cmd/fpga_plugin/opae_test.go | 180 +++++++++--------- cmd/gpu_plugin/gpu_plugin.go | 5 +- cmd/qat_plugin/dpdkdrv/dpdkdrv.go | 18 +- cmd/qat_plugin/kerneldrv/kerneldrv.go | 20 +- cmd/vpu_plugin/vpu_plugin.go | 9 +- pkg/deviceplugin/api.go | 25 ++- pkg/deviceplugin/manager_test.go | 32 ++-- pkg/deviceplugin/server.go | 14 +- pkg/deviceplugin/server_test.go | 32 ++-- 11 files changed, 295 insertions(+), 322 deletions(-) diff --git a/cmd/fpga_plugin/dfl_test.go b/cmd/fpga_plugin/dfl_test.go index e5a740c2..c6b52a99 100644 --- a/cmd/fpga_plugin/dfl_test.go +++ b/cmd/fpga_plugin/dfl_test.go @@ -124,61 +124,57 @@ func getDevicesDFL() []device { func TestGetRegionDevelTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.0", - ContainerPath: "/dev/dfl-fme.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.1", - ContainerPath: "/dev/dfl-fme.1", - Permissions: "rw", - }, + { + HostPath: "/dev/dfl-fme.0", + ContainerPath: "/dev/dfl-fme.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.2", - ContainerPath: "/dev/dfl-fme.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-fme.1", + ContainerPath: "/dev/dfl-fme.1", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-fme.2", + ContainerPath: "/dev/dfl-fme.2", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionDevelTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { @@ -188,46 +184,42 @@ func TestGetRegionDevelTreeDFL(t *testing.T) { func TestGetRegionTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { @@ -237,56 +229,51 @@ func TestGetRegionTreeDFL(t *testing.T) { func TestGetAfuTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, + } + + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", }, - }) + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) result := getAfuTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { diff --git a/cmd/fpga_plugin/fpga_plugin.go b/cmd/fpga_plugin/fpga_plugin.go index 6f8fa0b6..a8a45769 100644 --- a/cmd/fpga_plugin/fpga_plugin.go +++ b/cmd/fpga_plugin/fpga_plugin.go @@ -82,10 +82,8 @@ func getRegionDevelTree(devices []device) dpapi.DeviceTree { ContainerPath: region.devNode, Permissions: "rw", } - regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{ - State: health, - Nodes: devNodes, - }) + + regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } @@ -111,10 +109,7 @@ func getRegionTree(devices []device) dpapi.DeviceTree { Permissions: "rw", } } - regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{ - State: health, - Nodes: devNodes, - }) + regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } @@ -133,16 +128,14 @@ func getAfuTree(devices []device) dpapi.DeviceTree { health = pluginapi.Unhealthy } devType := fmt.Sprintf("%s-%s", afMode, afu.afuID) - afuTree.AddDevice(devType, afu.id, dpapi.DeviceInfo{ - State: health, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: afu.devNode, - ContainerPath: afu.devNode, - Permissions: "rw", - }, + devNodes := []pluginapi.DeviceSpec{ + { + HostPath: afu.devNode, + ContainerPath: afu.devNode, + Permissions: "rw", }, - }) + } + afuTree.AddDevice(devType, afu.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } } diff --git a/cmd/fpga_plugin/opae_test.go b/cmd/fpga_plugin/opae_test.go index 6e6ba942..12fa788f 100644 --- a/cmd/fpga_plugin/opae_test.go +++ b/cmd/fpga_plugin/opae_test.go @@ -114,51 +114,47 @@ func getDevicesOPAE() []device { func TestGetRegionDevelTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.0", - ContainerPath: "/dev/intel-fpga-fme.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.1", - ContainerPath: "/dev/intel-fpga-fme.1", - Permissions: "rw", - }, + { + HostPath: "/dev/intel-fpga-fme.0", + ContainerPath: "/dev/intel-fpga-fme.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.2", - ContainerPath: "/dev/intel-fpga-fme.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) + { + HostPath: "/dev/intel-fpga-fme.1", + ContainerPath: "/dev/intel-fpga-fme.1", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", + }, + { + HostPath: "/dev/intel-fpga-fme.2", + ContainerPath: "/dev/intel-fpga-fme.2", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionDevelTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { @@ -168,36 +164,32 @@ func TestGetRegionDevelTreeOPAE(t *testing.T) { func TestGetRegionTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { @@ -207,36 +199,32 @@ func TestGetRegionTreeOPAE(t *testing.T) { func TestGetAfuTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getAfuTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { diff --git a/cmd/gpu_plugin/gpu_plugin.go b/cmd/gpu_plugin/gpu_plugin.go index 591f91cf..b174d572 100644 --- a/cmd/gpu_plugin/gpu_plugin.go +++ b/cmd/gpu_plugin/gpu_plugin.go @@ -131,10 +131,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { devID := fmt.Sprintf("%s-%d", f.Name(), i) // Currently only one device type (i915) is supported. // TODO: check model ID to differentiate device models. - devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: nodes, - }) + devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) } } } diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go index 125f3ada..894aa950 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -28,7 +28,7 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ) const ( @@ -81,7 +81,7 @@ func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVf } // Scan implements Scanner interface for vfio based QAT plugin. -func (dp *DevicePlugin) Scan(notifier deviceplugin.Notifier) error { +func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error { for { devTree, err := dp.scan() if err != nil { @@ -258,8 +258,8 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error return nil } -func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { - devTree := deviceplugin.NewDeviceTree() +func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { + devTree := dpapi.NewDeviceTree() n := 0 for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) @@ -304,14 +304,10 @@ func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { return nil, err } - devinfo := deviceplugin.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: devNodes, - Mounts: devMounts, - Envs: map[string]string{ - fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), - }, + envs := map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), } + devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, devMounts, envs) devTree.AddDevice("generic", vfpciaddr, devinfo) } diff --git a/cmd/qat_plugin/kerneldrv/kerneldrv.go b/cmd/qat_plugin/kerneldrv/kerneldrv.go index 59a87c69..3cd06ccf 100644 --- a/cmd/qat_plugin/kerneldrv/kerneldrv.go +++ b/cmd/qat_plugin/kerneldrv/kerneldrv.go @@ -95,17 +95,15 @@ func getDevTree(sysfs string, qatDevs []device, config map[string]section) (dpap devType = fmt.Sprintf("cy%d_dc%d", svalue.cryptoEngines, svalue.compressionEngines) for _, ep := range svalue.endpoints { for i := 0; i < ep.processes; i++ { - devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: devs, - Envs: map[string]string{ - fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, - // This env variable may get overriden if a container requests more than one QAT process. - // But we keep this code since the majority of pod workloads run only one QAT process. - // The rest should use QAT_SECTION_NAME_XXX variables. - "QAT_SECTION_NAME": sname, - }, - }) + envs := map[string]string{ + fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, + // This env variable may get overriden if a container requests more than one QAT process. + // But we keep this code since the majority of pod workloads run only one QAT process. + // The rest should use QAT_SECTION_NAME_XXX variables. + "QAT_SECTION_NAME": sname, + } + deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, envs) + devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), deviceInfo) uniqID++ } diff --git a/cmd/vpu_plugin/vpu_plugin.go b/cmd/vpu_plugin/vpu_plugin.go index e2f20dca..7feb05b3 100644 --- a/cmd/vpu_plugin/vpu_plugin.go +++ b/cmd/vpu_plugin/vpu_plugin.go @@ -17,10 +17,11 @@ package main import ( "flag" "fmt" - "github.com/google/gousb" "os" "time" + "github.com/google/gousb" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" @@ -144,11 +145,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { ContainerPath: hddlServicePath2, }, } - devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: nodes, - Mounts: mounts, - }) + devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil)) } } diff --git a/pkg/deviceplugin/api.go b/pkg/deviceplugin/api.go index f7a4f777..3e579e85 100644 --- a/pkg/deviceplugin/api.go +++ b/pkg/deviceplugin/api.go @@ -20,10 +20,27 @@ import ( // DeviceInfo contains information about device maintained by Device Plugin type DeviceInfo struct { - State string - Nodes []pluginapi.DeviceSpec - Mounts []pluginapi.Mount - Envs map[string]string + state string + nodes []pluginapi.DeviceSpec + mounts []pluginapi.Mount + envs map[string]string + topology *pluginapi.TopologyInfo +} + +// NewDeviceInfo makes DeviceInfo struct and adds topology information to it +func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string) DeviceInfo { + deviceInfo := DeviceInfo{ + state: state, + nodes: nodes, + mounts: mounts, + envs: envs, + } + devPaths := []string{} + for _, node := range nodes { + devPaths = append(devPaths, node.HostPath) + } + + return deviceInfo } // DeviceTree contains a tree-like structure of device type -> device ID -> device info. diff --git a/pkg/deviceplugin/manager_test.go b/pkg/deviceplugin/manager_test.go index fb5a7c0c..0ebe059a 100644 --- a/pkg/deviceplugin/manager_test.go +++ b/pkg/deviceplugin/manager_test.go @@ -43,8 +43,8 @@ func TestNotify(t *testing.T) { newmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -61,8 +61,8 @@ func TestNotify(t *testing.T) { oldmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -75,8 +75,8 @@ func TestNotify(t *testing.T) { newmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", @@ -93,8 +93,8 @@ func TestNotify(t *testing.T) { oldmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -150,8 +150,8 @@ type devicePluginStub struct{} func (*devicePluginStub) Scan(n Notifier) error { tree := NewDeviceTree() tree.AddDevice("testdevice", "dev1", DeviceInfo{ - State: pluginapi.Healthy, - Nodes: make([]pluginapi.DeviceSpec, 0), + state: pluginapi.Healthy, + nodes: make([]pluginapi.DeviceSpec, 0), }) n.Notify(tree) return nil @@ -179,8 +179,8 @@ func TestHandleUpdate(t *testing.T) { Added: map[string]map[string]DeviceInfo{ "ce48969398f05f33946d560708be108a": { "intel-fpga-fme.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -194,8 +194,8 @@ func TestHandleUpdate(t *testing.T) { }, }, "intel-fpga-fme.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", @@ -222,8 +222,8 @@ func TestHandleUpdate(t *testing.T) { Updated: map[string]map[string]DeviceInfo{ "ce48969398f05f33946d560708be108a": { "intel-fpga-fme.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", diff --git a/pkg/deviceplugin/server.go b/pkg/deviceplugin/server.go index c7758dcd..e3af1753 100644 --- a/pkg/deviceplugin/server.go +++ b/pkg/deviceplugin/server.go @@ -83,7 +83,7 @@ func (srv *server) sendDevices(stream pluginapi.DevicePlugin_ListAndWatchServer) for id, device := range srv.devices { resp.Devices = append(resp.Devices, &pluginapi.Device{ ID: id, - Health: device.State, + Health: device.state, }) } debug.Print("Sending to kubelet", resp.Devices) @@ -120,16 +120,16 @@ func (srv *server) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) if !ok { return nil, errors.Errorf("Invalid allocation request with non-existing device %s", id) } - if dev.State != pluginapi.Healthy { + if dev.state != pluginapi.Healthy { return nil, errors.Errorf("Invalid allocation request with unhealthy device %s", id) } - for i := range dev.Nodes { - cresp.Devices = append(cresp.Devices, &dev.Nodes[i]) + for i := range dev.nodes { + cresp.Devices = append(cresp.Devices, &dev.nodes[i]) } - for i := range dev.Mounts { - cresp.Mounts = append(cresp.Mounts, &dev.Mounts[i]) + for i := range dev.mounts { + cresp.Mounts = append(cresp.Mounts, &dev.mounts[i]) } - for key, value := range dev.Envs { + for key, value := range dev.envs { if cresp.Envs == nil { cresp.Envs = make(map[string]string) } diff --git a/pkg/deviceplugin/server_test.go b/pkg/deviceplugin/server_test.go index 82989449..3f3fb0a3 100644 --- a/pkg/deviceplugin/server_test.go +++ b/pkg/deviceplugin/server_test.go @@ -120,10 +120,10 @@ func TestSetupAndServe(t *testing.T) { devType: "testtype", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, + state: pluginapi.Healthy, }, "dev2": { - State: pluginapi.Healthy, + state: pluginapi.Healthy, }, }, updatesCh: make(chan map[string]DeviceInfo), @@ -247,8 +247,8 @@ func TestAllocate(t *testing.T) { name: "Allocate unhealthy devices", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Unhealthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -263,8 +263,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -279,8 +279,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device with postAllocate hook", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -292,7 +292,7 @@ func TestAllocate(t *testing.T) { Permissions: "rw", }, }, - Mounts: []pluginapi.Mount{ + mounts: []pluginapi.Mount{ { HostPath: "/dev", ContainerPath: "/dev", @@ -302,7 +302,7 @@ func TestAllocate(t *testing.T) { ContainerPath: "/mnt", }, }, - Envs: map[string]string{ + envs: map[string]string{ "testname": "testvalue", }, }, @@ -316,8 +316,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device with failing postAllocate hook", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -418,8 +418,8 @@ func TestListAndWatch(t *testing.T) { updates: []map[string]DeviceInfo{ { "fake_id": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -435,8 +435,8 @@ func TestListAndWatch(t *testing.T) { updates: []map[string]DeviceInfo{ { "fake_id": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",