github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #277 from bart0sh/PR0072-topology-hints

Browse Source 
Topology hints
This commit is contained in:
Dmitry Rozhkov 2020-02-10 16:43:48 +02:00 committed by GitHub
commit 2eae086f6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 351 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
cmd/fpga_plugin/dfl_test.go
View File

@ -124,61 +124,57 @@ func getDevicesDFL() []device {
func TestGetRegionDevelTreeDFL(t *testing.T) { func TestGetRegionDevelTreeDFL(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/dfl-port.0",
{ ContainerPath: "/dev/dfl-port.0",
HostPath: "/dev/dfl-port.0", Permissions: "rw",
ContainerPath: "/dev/dfl-port.0",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-fme.0",
ContainerPath: "/dev/dfl-fme.0",
Permissions: "rw",
},
}, },
}) {
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ HostPath: "/dev/dfl-fme.0",
State: pluginapi.Healthy, ContainerPath: "/dev/dfl-fme.0",
Nodes: []pluginapi.DeviceSpec{ Permissions: "rw",
{
HostPath: "/dev/dfl-port.1",
ContainerPath: "/dev/dfl-port.1",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.2",
ContainerPath: "/dev/dfl-port.2",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-fme.1",
ContainerPath: "/dev/dfl-fme.1",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dfl-port.3", HostPath: "/dev/dfl-port.1",
ContainerPath: "/dev/dfl-port.3", ContainerPath: "/dev/dfl-port.1",
Permissions: "rw", Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.4",
ContainerPath: "/dev/dfl-port.4",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-fme.2",
ContainerPath: "/dev/dfl-fme.2",
Permissions: "rw",
},
}, },
}) {
HostPath: "/dev/dfl-port.2",
ContainerPath: "/dev/dfl-port.2",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-fme.1",
ContainerPath: "/dev/dfl-fme.1",
Permissions: "rw",
},
}
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
nodes = []pluginapi.DeviceSpec{
{
HostPath: "/dev/dfl-port.3",
ContainerPath: "/dev/dfl-port.3",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.4",
ContainerPath: "/dev/dfl-port.4",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-fme.2",
ContainerPath: "/dev/dfl-fme.2",
Permissions: "rw",
},
}
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
result := getRegionDevelTree(getDevicesDFL()) result := getRegionDevelTree(getDevicesDFL())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {
@ -188,46 +184,42 @@ func TestGetRegionDevelTreeDFL(t *testing.T) {
func TestGetRegionTreeDFL(t *testing.T) { func TestGetRegionTreeDFL(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/dfl-port.0",
{ ContainerPath: "/dev/dfl-port.0",
HostPath: "/dev/dfl-port.0", Permissions: "rw",
ContainerPath: "/dev/dfl-port.0",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.DeviceInfo{ expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dfl-port.1", HostPath: "/dev/dfl-port.1",
ContainerPath: "/dev/dfl-port.1", ContainerPath: "/dev/dfl-port.1",
Permissions: "rw", Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.2",
ContainerPath: "/dev/dfl-port.2",
Permissions: "rw",
},
}, },
}) {
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.DeviceInfo{ HostPath: "/dev/dfl-port.2",
State: pluginapi.Unhealthy, ContainerPath: "/dev/dfl-port.2",
Nodes: []pluginapi.DeviceSpec{ Permissions: "rw",
{
HostPath: "/dev/dfl-port.3",
ContainerPath: "/dev/dfl-port.3",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.4",
ContainerPath: "/dev/dfl-port.4",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
nodes = []pluginapi.DeviceSpec{
{
HostPath: "/dev/dfl-port.3",
ContainerPath: "/dev/dfl-port.3",
Permissions: "rw",
},
{
HostPath: "/dev/dfl-port.4",
ContainerPath: "/dev/dfl-port.4",
Permissions: "rw",
},
}
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
result := getRegionTree(getDevicesDFL()) result := getRegionTree(getDevicesDFL())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {
@ -237,56 +229,51 @@ func TestGetRegionTreeDFL(t *testing.T) {
func TestGetAfuTreeDFL(t *testing.T) { func TestGetAfuTreeDFL(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/dfl-port.0",
{ ContainerPath: "/dev/dfl-port.0",
HostPath: "/dev/dfl-port.0", Permissions: "rw",
ContainerPath: "/dev/dfl-port.0",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.DeviceInfo{ expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dfl-port.1", HostPath: "/dev/dfl-port.1",
ContainerPath: "/dev/dfl-port.1", ContainerPath: "/dev/dfl-port.1",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.DeviceInfo{
State: pluginapi.Healthy, expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
Nodes: []pluginapi.DeviceSpec{
{ nodes = []pluginapi.DeviceSpec{
HostPath: "/dev/dfl-port.2", {
ContainerPath: "/dev/dfl-port.2", HostPath: "/dev/dfl-port.2",
Permissions: "rw", ContainerPath: "/dev/dfl-port.2",
}, Permissions: "rw",
}, },
}) }
expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.DeviceInfo{ expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dfl-port.3", HostPath: "/dev/dfl-port.3",
ContainerPath: "/dev/dfl-port.3", ContainerPath: "/dev/dfl-port.3",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.DeviceInfo{ expected.AddDevice(afMode+"-"+unhealthyAfuID, "dfl-port.3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dfl-port.4", HostPath: "/dev/dfl-port.4",
ContainerPath: "/dev/dfl-port.4", ContainerPath: "/dev/dfl-port.4",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "dfl-port.4", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
result := getAfuTree(getDevicesDFL()) result := getAfuTree(getDevicesDFL())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {

27
cmd/fpga_plugin/fpga_plugin.go
View File

@ -82,10 +82,8 @@ func getRegionDevelTree(devices []device) dpapi.DeviceTree {
ContainerPath: region.devNode, ContainerPath: region.devNode,
Permissions: "rw", Permissions: "rw",
} }
regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{
State: health, regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil))
Nodes: devNodes,
})
} }
} }
@ -111,10 +109,7 @@ func getRegionTree(devices []device) dpapi.DeviceTree {
Permissions: "rw", Permissions: "rw",
} }
} }
regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{ regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil))
State: health,
Nodes: devNodes,
})
} }
} }
@ -133,16 +128,14 @@ func getAfuTree(devices []device) dpapi.DeviceTree {
health = pluginapi.Unhealthy health = pluginapi.Unhealthy
} }
devType := fmt.Sprintf("%s-%s", afMode, afu.afuID) devType := fmt.Sprintf("%s-%s", afMode, afu.afuID)
afuTree.AddDevice(devType, afu.id, dpapi.DeviceInfo{ devNodes := []pluginapi.DeviceSpec{
State: health, {
Nodes: []pluginapi.DeviceSpec{ HostPath: afu.devNode,
{ ContainerPath: afu.devNode,
HostPath: afu.devNode, Permissions: "rw",
ContainerPath: afu.devNode,
Permissions: "rw",
},
}, },
}) }
afuTree.AddDevice(devType, afu.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil))
} }
} }
} }

180
cmd/fpga_plugin/opae_test.go
View File

@ -114,51 +114,47 @@ func getDevicesOPAE() []device {
func TestGetRegionDevelTreeOPAE(t *testing.T) { func TestGetRegionDevelTreeOPAE(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/intel-fpga-port.0",
{ ContainerPath: "/dev/intel-fpga-port.0",
HostPath: "/dev/intel-fpga-port.0", Permissions: "rw",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.0",
ContainerPath: "/dev/intel-fpga-fme.0",
Permissions: "rw",
},
}, },
}) {
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ HostPath: "/dev/intel-fpga-fme.0",
State: pluginapi.Healthy, ContainerPath: "/dev/intel-fpga-fme.0",
Nodes: []pluginapi.DeviceSpec{ Permissions: "rw",
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.1",
ContainerPath: "/dev/intel-fpga-fme.1",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.2", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.2", ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw", Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.2",
ContainerPath: "/dev/intel-fpga-fme.2",
Permissions: "rw",
},
}, },
}) {
HostPath: "/dev/intel-fpga-fme.1",
ContainerPath: "/dev/intel-fpga-fme.1",
Permissions: "rw",
},
}
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
nodes = []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.2",
ContainerPath: "/dev/intel-fpga-fme.2",
Permissions: "rw",
},
}
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
result := getRegionDevelTree(getDevicesOPAE()) result := getRegionDevelTree(getDevicesOPAE())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {
@ -168,36 +164,32 @@ func TestGetRegionDevelTreeOPAE(t *testing.T) {
func TestGetRegionTreeOPAE(t *testing.T) { func TestGetRegionTreeOPAE(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/intel-fpga-port.0",
{ ContainerPath: "/dev/intel-fpga-port.0",
HostPath: "/dev/intel-fpga-port.0", Permissions: "rw",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{ expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.1", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{ expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.2", HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2", ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
result := getRegionTree(getDevicesOPAE()) result := getRegionTree(getDevicesOPAE())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {
@ -207,36 +199,32 @@ func TestGetRegionTreeOPAE(t *testing.T) {
func TestGetAfuTreeOPAE(t *testing.T) { func TestGetAfuTreeOPAE(t *testing.T) {
expected := dpapi.NewDeviceTree() expected := dpapi.NewDeviceTree()
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.DeviceInfo{ nodes := []pluginapi.DeviceSpec{
State: pluginapi.Healthy, {
Nodes: []pluginapi.DeviceSpec{ HostPath: "/dev/intel-fpga-port.0",
{ ContainerPath: "/dev/intel-fpga-port.0",
HostPath: "/dev/intel-fpga-port.0", Permissions: "rw",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.DeviceInfo{ expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.1", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.DeviceInfo{ expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes = []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.2", HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2", ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw", Permissions: "rw",
},
}, },
}) }
expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil))
result := getAfuTree(getDevicesOPAE()) result := getAfuTree(getDevicesOPAE())
if !reflect.DeepEqual(result, expected) { if !reflect.DeepEqual(result, expected) {

5
cmd/gpu_plugin/gpu_plugin.go
View File

@ -131,10 +131,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
devID := fmt.Sprintf("%s-%d", f.Name(), i) devID := fmt.Sprintf("%s-%d", f.Name(), i)
// Currently only one device type (i915) is supported. // Currently only one device type (i915) is supported.
// TODO: check model ID to differentiate device models. // TODO: check model ID to differentiate device models.
devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
State: pluginapi.Healthy,
Nodes: nodes,
})
} }
} }
} }

18
cmd/qat_plugin/dpdkdrv/dpdkdrv.go
View File

@ -28,7 +28,7 @@ import (
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
) )
const ( const (
@ -81,7 +81,7 @@ func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVf
} }
// Scan implements Scanner interface for vfio based QAT plugin. // Scan implements Scanner interface for vfio based QAT plugin.
func (dp *DevicePlugin) Scan(notifier deviceplugin.Notifier) error { func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
for { for {
devTree, err := dp.scan() devTree, err := dp.scan()
if err != nil { if err != nil {
@ -258,8 +258,8 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error
return nil return nil
} }
func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
devTree := deviceplugin.NewDeviceTree() devTree := dpapi.NewDeviceTree()
n := 0 n := 0
for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) {
files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver))
@ -304,14 +304,10 @@ func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) {
return nil, err return nil, err
} }
devinfo := deviceplugin.DeviceInfo{ envs := map[string]string{
State: pluginapi.Healthy, fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(),
Nodes: devNodes,
Mounts: devMounts,
Envs: map[string]string{
fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(),
},
} }
devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, devMounts, envs)
devTree.AddDevice("generic", vfpciaddr, devinfo) devTree.AddDevice("generic", vfpciaddr, devinfo)
} }

20
cmd/qat_plugin/kerneldrv/kerneldrv.go
View File

@ -95,17 +95,15 @@ func getDevTree(sysfs string, qatDevs []device, config map[string]section) (dpap
devType = fmt.Sprintf("cy%d_dc%d", svalue.cryptoEngines, svalue.compressionEngines) devType = fmt.Sprintf("cy%d_dc%d", svalue.cryptoEngines, svalue.compressionEngines)
for _, ep := range svalue.endpoints { for _, ep := range svalue.endpoints {
for i := 0; i < ep.processes; i++ { for i := 0; i < ep.processes; i++ {
devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), dpapi.DeviceInfo{ envs := map[string]string{
State: pluginapi.Healthy, fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname,
Nodes: devs, // This env variable may get overriden if a container requests more than one QAT process.
Envs: map[string]string{ // But we keep this code since the majority of pod workloads run only one QAT process.
fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, // The rest should use QAT_SECTION_NAME_XXX variables.
// This env variable may get overriden if a container requests more than one QAT process. "QAT_SECTION_NAME": sname,
// But we keep this code since the majority of pod workloads run only one QAT process. }
// The rest should use QAT_SECTION_NAME_XXX variables. deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, envs)
"QAT_SECTION_NAME": sname, devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), deviceInfo)
},
})
uniqID++ uniqID++
} }

9
cmd/vpu_plugin/vpu_plugin.go
View File

@ -17,10 +17,11 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"github.com/google/gousb"
"os" "os"
"time" "time"
"github.com/google/gousb"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug"
dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
@ -144,11 +145,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
ContainerPath: hddlServicePath2, ContainerPath: hddlServicePath2,
}, },
} }
devTree.AddDevice(deviceType, devID, dpapi.DeviceInfo{ devTree.AddDevice(deviceType, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil))
State: pluginapi.Healthy,
Nodes: nodes,
Mounts: mounts,
})
} }
} }

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/go-ini/ini v1.46.0 github.com/go-ini/ini v1.46.0
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750 github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e
github.com/pkg/errors v0.8.1 github.com/pkg/errors v0.8.1
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456

2
go.sum
View File

@ -281,6 +281,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e h1:+pD7vYnhjGklatQ/Zs98yB/l6ETL+79WiI3IGM9Bmqk=
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e/go.mod h1:98ghzNsdMHeewnXWSFGx+Sf1AbZEVcL+mOLvE6DI4OQ=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

76
pkg/deviceplugin/api.go
View File

@ -15,15 +15,83 @@
package deviceplugin package deviceplugin
import ( import (
"sort"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/intel/cri-resource-manager/pkg/topology"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
) )
// DeviceInfo contains information about device maintained by Device Plugin // DeviceInfo contains information about device maintained by Device Plugin
type DeviceInfo struct { type DeviceInfo struct {
State string state string
Nodes []pluginapi.DeviceSpec nodes []pluginapi.DeviceSpec
Mounts []pluginapi.Mount mounts []pluginapi.Mount
Envs map[string]string envs map[string]string
topology *pluginapi.TopologyInfo
}
// getTopologyInfo returns topology information for the list of device nodes
func getTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) {
var result pluginapi.TopologyInfo
nodeIDs := map[int64]struct{}{}
for _, dev := range devs {
sysfsDevice, err := topology.FindSysFsDevice(dev)
if err != nil {
return nil, err
}
if sysfsDevice == "" {
return nil, errors.Errorf("device %s doesn't exist", dev)
}
hints, err := topology.NewTopologyHints(sysfsDevice)
if err != nil {
return nil, err
}
for _, hint := range hints {
for _, nNode := range strings.Split(hint.NUMAs, ",") {
nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode)
}
if nNodeID < 0 {
return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID)
}
if _, ok := nodeIDs[nNodeID]; !ok {
result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID})
nodeIDs[nNodeID] = struct{}{}
}
}
}
}
sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID })
return &result, nil
}
// NewDeviceInfo makes DeviceInfo struct and adds topology information to it
func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string) DeviceInfo {
deviceInfo := DeviceInfo{
state: state,
nodes: nodes,
mounts: mounts,
envs: envs,
}
devPaths := []string{}
for _, node := range nodes {
devPaths = append(devPaths, node.HostPath)
}
topologyInfo, err := getTopologyInfo(devPaths)
if err == nil {
deviceInfo.topology = topologyInfo
}
return deviceInfo
} }
// DeviceTree contains a tree-like structure of device type -> device ID -> device info. // DeviceTree contains a tree-like structure of device type -> device ID -> device info.

32
pkg/deviceplugin/manager_test.go
View File

@ -43,8 +43,8 @@ func TestNotify(t *testing.T) {
newmap: map[string]map[string]DeviceInfo{ newmap: map[string]map[string]DeviceInfo{
"someDeviceType": { "someDeviceType": {
"intel-fpga-port.0": { "intel-fpga-port.0": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",
@ -61,8 +61,8 @@ func TestNotify(t *testing.T) {
oldmap: map[string]map[string]DeviceInfo{ oldmap: map[string]map[string]DeviceInfo{
"someDeviceType": { "someDeviceType": {
"intel-fpga-port.0": { "intel-fpga-port.0": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",
@ -75,8 +75,8 @@ func TestNotify(t *testing.T) {
newmap: map[string]map[string]DeviceInfo{ newmap: map[string]map[string]DeviceInfo{
"someDeviceType": { "someDeviceType": {
"intel-fpga-port.1": { "intel-fpga-port.1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.1", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1",
@ -93,8 +93,8 @@ func TestNotify(t *testing.T) {
oldmap: map[string]map[string]DeviceInfo{ oldmap: map[string]map[string]DeviceInfo{
"someDeviceType": { "someDeviceType": {
"intel-fpga-port.0": { "intel-fpga-port.0": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",
@ -150,8 +150,8 @@ type devicePluginStub struct{}
func (*devicePluginStub) Scan(n Notifier) error { func (*devicePluginStub) Scan(n Notifier) error {
tree := NewDeviceTree() tree := NewDeviceTree()
tree.AddDevice("testdevice", "dev1", DeviceInfo{ tree.AddDevice("testdevice", "dev1", DeviceInfo{
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: make([]pluginapi.DeviceSpec, 0), nodes: make([]pluginapi.DeviceSpec, 0),
}) })
n.Notify(tree) n.Notify(tree)
return nil return nil
@ -179,8 +179,8 @@ func TestHandleUpdate(t *testing.T) {
Added: map[string]map[string]DeviceInfo{ Added: map[string]map[string]DeviceInfo{
"ce48969398f05f33946d560708be108a": { "ce48969398f05f33946d560708be108a": {
"intel-fpga-fme.0": { "intel-fpga-fme.0": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",
@ -194,8 +194,8 @@ func TestHandleUpdate(t *testing.T) {
}, },
}, },
"intel-fpga-fme.1": { "intel-fpga-fme.1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.1", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1",
@ -222,8 +222,8 @@ func TestHandleUpdate(t *testing.T) {
Updated: map[string]map[string]DeviceInfo{ Updated: map[string]map[string]DeviceInfo{
"ce48969398f05f33946d560708be108a": { "ce48969398f05f33946d560708be108a": {
"intel-fpga-fme.1": { "intel-fpga-fme.1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.1", HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1", ContainerPath: "/dev/intel-fpga-port.1",

17
pkg/deviceplugin/server.go
View File

@ -82,8 +82,9 @@ func (srv *server) sendDevices(stream pluginapi.DevicePlugin_ListAndWatchServer)
resp := new(pluginapi.ListAndWatchResponse) resp := new(pluginapi.ListAndWatchResponse)
for id, device := range srv.devices { for id, device := range srv.devices {
resp.Devices = append(resp.Devices, &pluginapi.Device{ resp.Devices = append(resp.Devices, &pluginapi.Device{
ID: id, ID: id,
Health: device.State, Health: device.state,
Topology: device.topology,
}) })
} }
debug.Print("Sending to kubelet", resp.Devices) debug.Print("Sending to kubelet", resp.Devices)
@ -120,16 +121,16 @@ func (srv *server) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest)
if !ok { if !ok {
return nil, errors.Errorf("Invalid allocation request with non-existing device %s", id) return nil, errors.Errorf("Invalid allocation request with non-existing device %s", id)
} }
if dev.State != pluginapi.Healthy { if dev.state != pluginapi.Healthy {
return nil, errors.Errorf("Invalid allocation request with unhealthy device %s", id) return nil, errors.Errorf("Invalid allocation request with unhealthy device %s", id)
} }
for i := range dev.Nodes { for i := range dev.nodes {
cresp.Devices = append(cresp.Devices, &dev.Nodes[i]) cresp.Devices = append(cresp.Devices, &dev.nodes[i])
} }
for i := range dev.Mounts { for i := range dev.mounts {
cresp.Mounts = append(cresp.Mounts, &dev.Mounts[i]) cresp.Mounts = append(cresp.Mounts, &dev.mounts[i])
} }
for key, value := range dev.Envs { for key, value := range dev.envs {
if cresp.Envs == nil { if cresp.Envs == nil {
cresp.Envs = make(map[string]string) cresp.Envs = make(map[string]string)
} }

32
pkg/deviceplugin/server_test.go
View File

@ -120,10 +120,10 @@ func TestSetupAndServe(t *testing.T) {
devType: "testtype", devType: "testtype",
devices: map[string]DeviceInfo{ devices: map[string]DeviceInfo{
"dev1": { "dev1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
}, },
"dev2": { "dev2": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
}, },
}, },
updatesCh: make(chan map[string]DeviceInfo), updatesCh: make(chan map[string]DeviceInfo),
@ -247,8 +247,8 @@ func TestAllocate(t *testing.T) {
name: "Allocate unhealthy devices", name: "Allocate unhealthy devices",
devices: map[string]DeviceInfo{ devices: map[string]DeviceInfo{
"dev1": { "dev1": {
State: pluginapi.Unhealthy, state: pluginapi.Unhealthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dev1", HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1", ContainerPath: "/dev/dev1",
@ -263,8 +263,8 @@ func TestAllocate(t *testing.T) {
name: "Allocate healthy device", name: "Allocate healthy device",
devices: map[string]DeviceInfo{ devices: map[string]DeviceInfo{
"dev1": { "dev1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dev1", HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1", ContainerPath: "/dev/dev1",
@ -279,8 +279,8 @@ func TestAllocate(t *testing.T) {
name: "Allocate healthy device with postAllocate hook", name: "Allocate healthy device with postAllocate hook",
devices: map[string]DeviceInfo{ devices: map[string]DeviceInfo{
"dev1": { "dev1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dev1", HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1", ContainerPath: "/dev/dev1",
@ -292,7 +292,7 @@ func TestAllocate(t *testing.T) {
Permissions: "rw", Permissions: "rw",
}, },
}, },
Mounts: []pluginapi.Mount{ mounts: []pluginapi.Mount{
{ {
HostPath: "/dev", HostPath: "/dev",
ContainerPath: "/dev", ContainerPath: "/dev",
@ -302,7 +302,7 @@ func TestAllocate(t *testing.T) {
ContainerPath: "/mnt", ContainerPath: "/mnt",
}, },
}, },
Envs: map[string]string{ envs: map[string]string{
"testname": "testvalue", "testname": "testvalue",
}, },
}, },
@ -316,8 +316,8 @@ func TestAllocate(t *testing.T) {
name: "Allocate healthy device with failing postAllocate hook", name: "Allocate healthy device with failing postAllocate hook",
devices: map[string]DeviceInfo{ devices: map[string]DeviceInfo{
"dev1": { "dev1": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/dev1", HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1", ContainerPath: "/dev/dev1",
@ -418,8 +418,8 @@ func TestListAndWatch(t *testing.T) {
updates: []map[string]DeviceInfo{ updates: []map[string]DeviceInfo{
{ {
"fake_id": { "fake_id": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",
@ -435,8 +435,8 @@ func TestListAndWatch(t *testing.T) {
updates: []map[string]DeviceInfo{ updates: []map[string]DeviceInfo{
{ {
"fake_id": { "fake_id": {
State: pluginapi.Healthy, state: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{ nodes: []pluginapi.DeviceSpec{
{ {
HostPath: "/dev/intel-fpga-port.0", HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0", ContainerPath: "/dev/intel-fpga-port.0",