diff --git a/cmd/fpga_plugin/dfl_test.go b/cmd/fpga_plugin/dfl_test.go index e5a740c2..c6b52a99 100644 --- a/cmd/fpga_plugin/dfl_test.go +++ b/cmd/fpga_plugin/dfl_test.go @@ -124,61 +124,57 @@ func getDevicesDFL() []device { func TestGetRegionDevelTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.0", - ContainerPath: "/dev/dfl-fme.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.1", - ContainerPath: "/dev/dfl-fme.1", - Permissions: "rw", - }, + { + HostPath: "/dev/dfl-fme.0", + ContainerPath: "/dev/dfl-fme.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-fme.2", - ContainerPath: "/dev/dfl-fme.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-fme.1", + ContainerPath: "/dev/dfl-fme.1", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-fme.2", + ContainerPath: "/dev/dfl-fme.2", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionDevelTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { @@ -188,46 +184,42 @@ func TestGetRegionDevelTreeDFL(t *testing.T) { func TestGetRegionTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", + }, + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { @@ -237,56 +229,51 @@ func TestGetRegionTreeDFL(t *testing.T) { func TestGetAfuTreeDFL(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.0", - ContainerPath: "/dev/dfl-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.0", + ContainerPath: "/dev/dfl-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.1", - ContainerPath: "/dev/dfl-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.1", + ContainerPath: "/dev/dfl-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.2", - ContainerPath: "/dev/dfl-port.2", - Permissions: "rw", - }, + } + + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.2", + ContainerPath: "/dev/dfl-port.2", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.3", - ContainerPath: "/dev/dfl-port.3", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.3", + ContainerPath: "/dev/dfl-port.3", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/dfl-port.4", - ContainerPath: "/dev/dfl-port.4", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/dfl-port.4", + ContainerPath: "/dev/dfl-port.4", + Permissions: "rw", }, - }) + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) result := getAfuTree(getDevicesDFL()) if !reflect.DeepEqual(result, expected) { diff --git a/cmd/fpga_plugin/fpga_plugin.go b/cmd/fpga_plugin/fpga_plugin.go index 6f8fa0b6..a8a45769 100644 --- a/cmd/fpga_plugin/fpga_plugin.go +++ b/cmd/fpga_plugin/fpga_plugin.go @@ -82,10 +82,8 @@ func getRegionDevelTree(devices []device) dpapi.DeviceTree { ContainerPath: region.devNode, Permissions: "rw", } - regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{ - State: health, - Nodes: devNodes, - }) + + regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } @@ -111,10 +109,7 @@ func getRegionTree(devices []device) dpapi.DeviceTree { Permissions: "rw", } } - regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{ - State: health, - Nodes: devNodes, - }) + regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } @@ -133,16 +128,14 @@ func getAfuTree(devices []device) dpapi.DeviceTree { health = pluginapi.Unhealthy } devType := fmt.Sprintf("%s-%s", afMode, afu.afuID) - afuTree.AddDevice(devType, afu.id, dpapi.DeviceInfo{ - State: health, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: afu.devNode, - ContainerPath: afu.devNode, - Permissions: "rw", - }, + devNodes := []pluginapi.DeviceSpec{ + { + HostPath: afu.devNode, + ContainerPath: afu.devNode, + Permissions: "rw", }, - }) + } + afuTree.AddDevice(devType, afu.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil)) } } } diff --git a/cmd/fpga_plugin/opae_test.go b/cmd/fpga_plugin/opae_test.go index 6e6ba942..12fa788f 100644 --- a/cmd/fpga_plugin/opae_test.go +++ b/cmd/fpga_plugin/opae_test.go @@ -114,51 +114,47 @@ func getDevicesOPAE() []device { func TestGetRegionDevelTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.0", - ContainerPath: "/dev/intel-fpga-fme.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.1", - ContainerPath: "/dev/intel-fpga-fme.1", - Permissions: "rw", - }, + { + HostPath: "/dev/intel-fpga-fme.0", + ContainerPath: "/dev/intel-fpga-fme.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, - { - HostPath: "/dev/intel-fpga-fme.2", - ContainerPath: "/dev/intel-fpga-fme.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) + { + HostPath: "/dev/intel-fpga-fme.1", + ContainerPath: "/dev/intel-fpga-fme.1", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", + }, + { + HostPath: "/dev/intel-fpga-fme.2", + ContainerPath: "/dev/intel-fpga-fme.2", + Permissions: "rw", + }, + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionDevelTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { @@ -168,36 +164,32 @@ func TestGetRegionDevelTreeOPAE(t *testing.T) { func TestGetRegionTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getRegionTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { @@ -207,36 +199,32 @@ func TestGetRegionTreeOPAE(t *testing.T) { func TestGetAfuTreeOPAE(t *testing.T) { expected := dpapi.NewDeviceTree() - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.0", - ContainerPath: "/dev/intel-fpga-port.0", - Permissions: "rw", - }, + nodes := []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.0", + ContainerPath: "/dev/intel-fpga-port.0", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.1", - ContainerPath: "/dev/intel-fpga-port.1", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.1", + ContainerPath: "/dev/intel-fpga-port.1", + Permissions: "rw", }, - }) - expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.DeviceInfo{ - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ - { - HostPath: "/dev/intel-fpga-port.2", - ContainerPath: "/dev/intel-fpga-port.2", - Permissions: "rw", - }, + } + expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) + + nodes = []pluginapi.DeviceSpec{ + { + HostPath: "/dev/intel-fpga-port.2", + ContainerPath: "/dev/intel-fpga-port.2", + Permissions: "rw", }, - }) + } + expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil)) result := getAfuTree(getDevicesOPAE()) if !reflect.DeepEqual(result, expected) { diff --git a/cmd/gpu_plugin/gpu_plugin.go b/cmd/gpu_plugin/gpu_plugin.go index 591f91cf..b174d572 100644 --- a/cmd/gpu_plugin/gpu_plugin.go +++ b/cmd/gpu_plugin/gpu_plugin.go @@ -131,10 +131,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { devID := fmt.Sprintf("%s-%d", f.Name(), i) // Currently only one device type (i915) is supported. // TODO: check model ID to differentiate device models. - devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: nodes, - }) + devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil)) } } } diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go index 125f3ada..894aa950 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -28,7 +28,7 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ) const ( @@ -81,7 +81,7 @@ func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVf } // Scan implements Scanner interface for vfio based QAT plugin. -func (dp *DevicePlugin) Scan(notifier deviceplugin.Notifier) error { +func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error { for { devTree, err := dp.scan() if err != nil { @@ -258,8 +258,8 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error return nil } -func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { - devTree := deviceplugin.NewDeviceTree() +func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { + devTree := dpapi.NewDeviceTree() n := 0 for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) @@ -304,14 +304,10 @@ func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { return nil, err } - devinfo := deviceplugin.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: devNodes, - Mounts: devMounts, - Envs: map[string]string{ - fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), - }, + envs := map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), } + devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, devMounts, envs) devTree.AddDevice("generic", vfpciaddr, devinfo) } diff --git a/cmd/qat_plugin/kerneldrv/kerneldrv.go b/cmd/qat_plugin/kerneldrv/kerneldrv.go index 59a87c69..3cd06ccf 100644 --- a/cmd/qat_plugin/kerneldrv/kerneldrv.go +++ b/cmd/qat_plugin/kerneldrv/kerneldrv.go @@ -95,17 +95,15 @@ func getDevTree(sysfs string, qatDevs []device, config map[string]section) (dpap devType = fmt.Sprintf("cy%d_dc%d", svalue.cryptoEngines, svalue.compressionEngines) for _, ep := range svalue.endpoints { for i := 0; i < ep.processes; i++ { - devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: devs, - Envs: map[string]string{ - fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, - // This env variable may get overriden if a container requests more than one QAT process. - // But we keep this code since the majority of pod workloads run only one QAT process. - // The rest should use QAT_SECTION_NAME_XXX variables. - "QAT_SECTION_NAME": sname, - }, - }) + envs := map[string]string{ + fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, + // This env variable may get overriden if a container requests more than one QAT process. + // But we keep this code since the majority of pod workloads run only one QAT process. + // The rest should use QAT_SECTION_NAME_XXX variables. + "QAT_SECTION_NAME": sname, + } + deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, envs) + devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), deviceInfo) uniqID++ } diff --git a/cmd/vpu_plugin/vpu_plugin.go b/cmd/vpu_plugin/vpu_plugin.go index e2f20dca..7feb05b3 100644 --- a/cmd/vpu_plugin/vpu_plugin.go +++ b/cmd/vpu_plugin/vpu_plugin.go @@ -17,10 +17,11 @@ package main import ( "flag" "fmt" - "github.com/google/gousb" "os" "time" + "github.com/google/gousb" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" @@ -144,11 +145,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { ContainerPath: hddlServicePath2, }, } - devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: nodes, - Mounts: mounts, - }) + devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil)) } } diff --git a/go.mod b/go.mod index d24acb13..f8e561fc 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-ini/ini v1.46.0 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750 + github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e github.com/pkg/errors v0.8.1 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 diff --git a/go.sum b/go.sum index 098bdc7a..d7575ef2 100644 --- a/go.sum +++ b/go.sum @@ -281,6 +281,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e h1:+pD7vYnhjGklatQ/Zs98yB/l6ETL+79WiI3IGM9Bmqk= +github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e/go.mod h1:98ghzNsdMHeewnXWSFGx+Sf1AbZEVcL+mOLvE6DI4OQ= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/pkg/deviceplugin/api.go b/pkg/deviceplugin/api.go index f7a4f777..180e6015 100644 --- a/pkg/deviceplugin/api.go +++ b/pkg/deviceplugin/api.go @@ -15,15 +15,83 @@ package deviceplugin import ( + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/intel/cri-resource-manager/pkg/topology" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) // DeviceInfo contains information about device maintained by Device Plugin type DeviceInfo struct { - State string - Nodes []pluginapi.DeviceSpec - Mounts []pluginapi.Mount - Envs map[string]string + state string + nodes []pluginapi.DeviceSpec + mounts []pluginapi.Mount + envs map[string]string + topology *pluginapi.TopologyInfo +} + +// getTopologyInfo returns topology information for the list of device nodes +func getTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) { + var result pluginapi.TopologyInfo + nodeIDs := map[int64]struct{}{} + for _, dev := range devs { + sysfsDevice, err := topology.FindSysFsDevice(dev) + if err != nil { + return nil, err + } + + if sysfsDevice == "" { + return nil, errors.Errorf("device %s doesn't exist", dev) + } + + hints, err := topology.NewTopologyHints(sysfsDevice) + if err != nil { + return nil, err + } + + for _, hint := range hints { + for _, nNode := range strings.Split(hint.NUMAs, ",") { + nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode) + } + if nNodeID < 0 { + return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID) + } + if _, ok := nodeIDs[nNodeID]; !ok { + result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID}) + nodeIDs[nNodeID] = struct{}{} + } + } + } + } + sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID }) + return &result, nil +} + +// NewDeviceInfo makes DeviceInfo struct and adds topology information to it +func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string) DeviceInfo { + deviceInfo := DeviceInfo{ + state: state, + nodes: nodes, + mounts: mounts, + envs: envs, + } + devPaths := []string{} + for _, node := range nodes { + devPaths = append(devPaths, node.HostPath) + } + + topologyInfo, err := getTopologyInfo(devPaths) + if err == nil { + deviceInfo.topology = topologyInfo + } + + return deviceInfo } // DeviceTree contains a tree-like structure of device type -> device ID -> device info. diff --git a/pkg/deviceplugin/manager_test.go b/pkg/deviceplugin/manager_test.go index fb5a7c0c..0ebe059a 100644 --- a/pkg/deviceplugin/manager_test.go +++ b/pkg/deviceplugin/manager_test.go @@ -43,8 +43,8 @@ func TestNotify(t *testing.T) { newmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -61,8 +61,8 @@ func TestNotify(t *testing.T) { oldmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -75,8 +75,8 @@ func TestNotify(t *testing.T) { newmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", @@ -93,8 +93,8 @@ func TestNotify(t *testing.T) { oldmap: map[string]map[string]DeviceInfo{ "someDeviceType": { "intel-fpga-port.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -150,8 +150,8 @@ type devicePluginStub struct{} func (*devicePluginStub) Scan(n Notifier) error { tree := NewDeviceTree() tree.AddDevice("testdevice", "dev1", DeviceInfo{ - State: pluginapi.Healthy, - Nodes: make([]pluginapi.DeviceSpec, 0), + state: pluginapi.Healthy, + nodes: make([]pluginapi.DeviceSpec, 0), }) n.Notify(tree) return nil @@ -179,8 +179,8 @@ func TestHandleUpdate(t *testing.T) { Added: map[string]map[string]DeviceInfo{ "ce48969398f05f33946d560708be108a": { "intel-fpga-fme.0": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -194,8 +194,8 @@ func TestHandleUpdate(t *testing.T) { }, }, "intel-fpga-fme.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", @@ -222,8 +222,8 @@ func TestHandleUpdate(t *testing.T) { Updated: map[string]map[string]DeviceInfo{ "ce48969398f05f33946d560708be108a": { "intel-fpga-fme.1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1", diff --git a/pkg/deviceplugin/server.go b/pkg/deviceplugin/server.go index c7758dcd..5e365221 100644 --- a/pkg/deviceplugin/server.go +++ b/pkg/deviceplugin/server.go @@ -82,8 +82,9 @@ func (srv *server) sendDevices(stream pluginapi.DevicePlugin_ListAndWatchServer) resp := new(pluginapi.ListAndWatchResponse) for id, device := range srv.devices { resp.Devices = append(resp.Devices, &pluginapi.Device{ - ID: id, - Health: device.State, + ID: id, + Health: device.state, + Topology: device.topology, }) } debug.Print("Sending to kubelet", resp.Devices) @@ -120,16 +121,16 @@ func (srv *server) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) if !ok { return nil, errors.Errorf("Invalid allocation request with non-existing device %s", id) } - if dev.State != pluginapi.Healthy { + if dev.state != pluginapi.Healthy { return nil, errors.Errorf("Invalid allocation request with unhealthy device %s", id) } - for i := range dev.Nodes { - cresp.Devices = append(cresp.Devices, &dev.Nodes[i]) + for i := range dev.nodes { + cresp.Devices = append(cresp.Devices, &dev.nodes[i]) } - for i := range dev.Mounts { - cresp.Mounts = append(cresp.Mounts, &dev.Mounts[i]) + for i := range dev.mounts { + cresp.Mounts = append(cresp.Mounts, &dev.mounts[i]) } - for key, value := range dev.Envs { + for key, value := range dev.envs { if cresp.Envs == nil { cresp.Envs = make(map[string]string) } diff --git a/pkg/deviceplugin/server_test.go b/pkg/deviceplugin/server_test.go index 82989449..3f3fb0a3 100644 --- a/pkg/deviceplugin/server_test.go +++ b/pkg/deviceplugin/server_test.go @@ -120,10 +120,10 @@ func TestSetupAndServe(t *testing.T) { devType: "testtype", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, + state: pluginapi.Healthy, }, "dev2": { - State: pluginapi.Healthy, + state: pluginapi.Healthy, }, }, updatesCh: make(chan map[string]DeviceInfo), @@ -247,8 +247,8 @@ func TestAllocate(t *testing.T) { name: "Allocate unhealthy devices", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Unhealthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Unhealthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -263,8 +263,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -279,8 +279,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device with postAllocate hook", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -292,7 +292,7 @@ func TestAllocate(t *testing.T) { Permissions: "rw", }, }, - Mounts: []pluginapi.Mount{ + mounts: []pluginapi.Mount{ { HostPath: "/dev", ContainerPath: "/dev", @@ -302,7 +302,7 @@ func TestAllocate(t *testing.T) { ContainerPath: "/mnt", }, }, - Envs: map[string]string{ + envs: map[string]string{ "testname": "testvalue", }, }, @@ -316,8 +316,8 @@ func TestAllocate(t *testing.T) { name: "Allocate healthy device with failing postAllocate hook", devices: map[string]DeviceInfo{ "dev1": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/dev1", ContainerPath: "/dev/dev1", @@ -418,8 +418,8 @@ func TestListAndWatch(t *testing.T) { updates: []map[string]DeviceInfo{ { "fake_id": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0", @@ -435,8 +435,8 @@ func TestListAndWatch(t *testing.T) { updates: []map[string]DeviceInfo{ { "fake_id": { - State: pluginapi.Healthy, - Nodes: []pluginapi.DeviceSpec{ + state: pluginapi.Healthy, + nodes: []pluginapi.DeviceSpec{ { HostPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",