github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

extend API to receive full specs instead of strings

Browse Source
This commit is contained in:
Dmitry Rozhkov 2019-01-21 17:15:27 +02:00
parent 2ce62fbc7e
commit 7662cb9154
8 changed files with 263 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/fpga_plugin/fpga_plugin.go
View File

@ -72,11 +72,19 @@ func getRegionDevelTree(devices []device) dpapi.DeviceTree {
health = pluginapi.Unhealthy
}
devType := fmt.Sprintf("%s-%s", regionMode, region.interfaceID)
devNodes := make([]string, len(region.afus)+1)
devNodes := make([]pluginapi.DeviceSpec, len(region.afus)+1)
for num, afu := range region.afus {
devNodes[num] = afu.devNode
devNodes[num] = pluginapi.DeviceSpec{
HostPath: afu.devNode,
ContainerPath: afu.devNode,
Permissions: "rw",
}
}
devNodes[len(region.afus)] = pluginapi.DeviceSpec{
HostPath: region.devNode,
ContainerPath: region.devNode,
Permissions: "rw",
}
devNodes[len(region.afus)] = region.devNode
regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{
State: health,
Nodes: devNodes,
@ -98,9 +106,13 @@ func getRegionTree(devices []device) dpapi.DeviceTree {
health = pluginapi.Unhealthy
}
devType := fmt.Sprintf("%s-%s", regionMode, region.interfaceID)
devNodes := make([]string, len(region.afus))
devNodes := make([]pluginapi.DeviceSpec, len(region.afus))
for num, afu := range region.afus {
devNodes[num] = afu.devNode
devNodes[num] = pluginapi.DeviceSpec{
HostPath: afu.devNode,
ContainerPath: afu.devNode,
Permissions: "rw",
}
}
regionTree.AddDevice(devType, region.id, dpapi.DeviceInfo{
State: health,
@ -126,7 +138,13 @@ func getAfuTree(devices []device) dpapi.DeviceTree {
devType := fmt.Sprintf("%s-%s", afMode, afu.afuID)
afuTree.AddDevice(devType, afu.id, dpapi.DeviceInfo{
State: health,
Nodes: []string{afu.devNode},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: afu.devNode,
ContainerPath: afu.devNode,
Permissions: "rw",
},
},
})
}
}

87
cmd/fpga_plugin/fpga_plugin_test.go
View File

@ -150,15 +150,48 @@ func TestGetRegionDevelTree(t *testing.T) {
expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0", "/dev/intel-fpga-fme.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.0",
ContainerPath: "/dev/intel-fpga-fme.0",
Permissions: "rw",
},
},
})
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1", "/dev/intel-fpga-fme.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.1",
ContainerPath: "/dev/intel-fpga-fme.1",
Permissions: "rw",
},
},
})
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{
State: pluginapi.Unhealthy,
Nodes: []string{"/dev/intel-fpga-port.2", "/dev/intel-fpga-fme.2"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.2",
ContainerPath: "/dev/intel-fpga-fme.2",
Permissions: "rw",
},
},
})
result := getRegionDevelTree(getDevices())
@ -171,15 +204,33 @@ func TestGetRegionTree(t *testing.T) {
expected := dpapi.NewDeviceTree()
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
})
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
},
})
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.DeviceInfo{
State: pluginapi.Unhealthy,
Nodes: []string{"/dev/intel-fpga-port.2"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw",
},
},
})
result := getRegionTree(getDevices())
@ -192,15 +243,33 @@ func TestGetAfuTree(t *testing.T) {
expected := dpapi.NewDeviceTree()
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.0", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
})
expected.AddDevice(afMode+"-d8424dc4a4a3c413f89e433683f9040b", "intel-fpga-port.1", dpapi.DeviceInfo{
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
},
})
expected.AddDevice(afMode+"-"+unhealthyAfuID, "intel-fpga-port.2", dpapi.DeviceInfo{
State: pluginapi.Unhealthy,
Nodes: []string{"/dev/intel-fpga-port.2"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.2",
ContainerPath: "/dev/intel-fpga-port.2",
Permissions: "rw",
},
},
})
result := getAfuTree(getDevices())

8
cmd/gpu_plugin/gpu_plugin.go
View File

@ -93,7 +93,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
}
if strings.TrimSpace(string(dat)) == vendorString {
var nodes []string
var nodes []pluginapi.DeviceSpec
drmFiles, err := ioutil.ReadDir(path.Join(dp.sysfsDir, f.Name(), "device/drm"))
if err != nil {
@ -111,7 +111,11 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
}
debug.Printf("Adding %s to GPU %s", devPath, f.Name())
nodes = append(nodes, devPath)
nodes = append(nodes, pluginapi.DeviceSpec{
HostPath: devPath,
ContainerPath: devPath,
Permissions: "rw",
})
}
if len(nodes) > 0 {

44
cmd/qat_plugin/qat_plugin.go
View File

@ -110,10 +110,10 @@ func (dp *devicePlugin) getDpdkDevice(id string) (string, error) {
return "", errors.New("Unknown DPDK driver")
}
func (dp *devicePlugin) getDpdkDeviceNames(id string) ([]string, error) {
func (dp *devicePlugin) getDpdkDeviceSpecs(id string) ([]pluginapi.DeviceSpec, error) {
dpdkDeviceName, err := dp.getDpdkDevice(id)
if err != nil {
return []string{}, err
return nil, err
}
fmt.Printf("%s device: corresponding DPDK device detected is %s\n", id, dpdkDeviceName)
@ -122,31 +122,53 @@ func (dp *devicePlugin) getDpdkDeviceNames(id string) ([]string, error) {
case "igb_uio":
//Setting up with uio
uioDev := path.Join(uioDevicePath, dpdkDeviceName)
return []string{uioDev}, nil
return []pluginapi.DeviceSpec{
{
HostPath: uioDev,
ContainerPath: uioDev,
Permissions: "rw",
},
}, nil
case "vfio-pci":
//Setting up with vfio
vfioDev1 := path.Join(vfioDevicePath, dpdkDeviceName)
vfioDev2 := path.Join(vfioDevicePath, "/vfio")
return []string{vfioDev1, vfioDev2}, nil
return []pluginapi.DeviceSpec{
{
HostPath: vfioDev1,
ContainerPath: vfioDev1,
Permissions: "rw",
},
{
HostPath: vfioDev2,
ContainerPath: vfioDev2,
Permissions: "rw",
},
}, nil
}
return []string{}, errors.New("Unknown DPDK driver")
return nil, errors.New("Unknown DPDK driver")
}
func (dp *devicePlugin) getDpdkMountPaths(id string) ([]string, error) {
func (dp *devicePlugin) getDpdkMounts(id string) ([]pluginapi.Mount, error) {
dpdkDeviceName, err := dp.getDpdkDevice(id)
if err != nil {
return []string{}, err
return nil, err
}
switch dp.dpdkDriver {
case "igb_uio":
//Setting up with uio mountpoints
uioMountPoint := path.Join(uioMountPath, dpdkDeviceName, "/device")
return []string{uioMountPoint}, nil
return []pluginapi.Mount{
{
HostPath: uioMountPoint,
ContainerPath: uioMountPath,
},
}, nil
case "vfio-pci":
//No mountpoint for vfio needs to be populated
return []string{}, nil
return nil, nil
}
return nil, errors.New("Unknown DPDK driver")
@ -259,11 +281,11 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
}
}
devNodes, err := dp.getDpdkDeviceNames(vfpciaddr)
devNodes, err := dp.getDpdkDeviceSpecs(vfpciaddr)
if err != nil {
return nil, err
}
devMounts, err := dp.getDpdkMountPaths(vfpciaddr)
devMounts, err := dp.getDpdkMounts(vfpciaddr)
if err != nil {
return nil, err
}

4
internal/deviceplugin/api.go
View File

@ -21,8 +21,8 @@ import (
// DeviceInfo contains information about device maintained by Device Plugin
type DeviceInfo struct {
State string
Nodes []string
Mounts []string
Nodes []pluginapi.DeviceSpec
Mounts []pluginapi.Mount
Envs map[string]string
}

73
internal/deviceplugin/manager_test.go
View File

@ -44,7 +44,13 @@ func TestNotify(t *testing.T) {
"someDeviceType": {
"intel-fpga-port.0": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
},
},
},
@ -56,7 +62,13 @@ func TestNotify(t *testing.T) {
"someDeviceType": {
"intel-fpga-port.0": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
},
},
},
@ -64,7 +76,13 @@ func TestNotify(t *testing.T) {
"someDeviceType": {
"intel-fpga-port.1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
},
},
},
},
@ -76,7 +94,13 @@ func TestNotify(t *testing.T) {
"someDeviceType": {
"intel-fpga-port.0": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
},
},
},
@ -127,7 +151,7 @@ func (*devicePluginStub) Scan(n Notifier) error {
tree := NewDeviceTree()
tree.AddDevice("testdevice", "dev1", DeviceInfo{
State: pluginapi.Healthy,
Nodes: make([]string, 0),
Nodes: make([]pluginapi.DeviceSpec, 0),
})
n.Notify(tree)
return nil
@ -156,11 +180,33 @@ func TestHandleUpdate(t *testing.T) {
"ce48969398f05f33946d560708be108a": {
"intel-fpga-fme.0": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0", "/dev/intel-fpga-fme.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.0",
ContainerPath: "/dev/intel-fpga-fme.0",
Permissions: "rw",
},
},
},
"intel-fpga-fme.1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1", "/dev/intel-fpga-fme.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.1",
ContainerPath: "/dev/intel-fpga-fme.1",
Permissions: "rw",
},
},
},
},
},
@ -177,7 +223,18 @@ func TestHandleUpdate(t *testing.T) {
"ce48969398f05f33946d560708be108a": {
"intel-fpga-fme.1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.1", "/dev/intel-fpga-fme.1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.1",
ContainerPath: "/dev/intel-fpga-port.1",
Permissions: "rw",
},
{
HostPath: "/dev/intel-fpga-fme.1",
ContainerPath: "/dev/intel-fpga-fme.1",
Permissions: "rw",
},
},
},
},
},

17
internal/deviceplugin/server.go
View File

@ -68,7 +68,10 @@ func (srv *server) GetDevicePluginOptions(ctx context.Context, empty *pluginapi.
func (srv *server) sendDevices(stream pluginapi.DevicePlugin_ListAndWatchServer) error {
resp := new(pluginapi.ListAndWatchResponse)
for id, device := range srv.devices {
resp.Devices = append(resp.Devices, &pluginapi.Device{id, device.State})
resp.Devices = append(resp.Devices, &pluginapi.Device{
ID: id,
Health: device.State,
})
}
debug.Print("Sending to kubelet", resp.Devices)
if err := stream.Send(resp); err != nil {
@ -108,18 +111,10 @@ func (srv *server) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest)
return nil, errors.Errorf("Invalid allocation request with unhealthy device %s", id)
}
for _, devnode := range dev.Nodes {
cresp.Devices = append(cresp.Devices, &pluginapi.DeviceSpec{
HostPath: devnode,
ContainerPath: devnode,
Permissions: "mrw",
})
cresp.Devices = append(cresp.Devices, &devnode)
}
for _, devmount := range dev.Mounts {
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: devmount,
ContainerPath: devmount,
ReadOnly: false,
})
cresp.Mounts = append(cresp.Mounts, &devmount)
}
for key, value := range dev.Envs {
if cresp.Envs == nil {

57
internal/deviceplugin/server_test.go
View File

@ -247,7 +247,13 @@ func TestAllocate(t *testing.T) {
devices: map[string]DeviceInfo{
"dev1": {
State: pluginapi.Unhealthy,
Nodes: []string{"/dev/dev1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1",
Permissions: "rw",
},
},
},
},
expectedErr: true,
@ -257,7 +263,13 @@ func TestAllocate(t *testing.T) {
devices: map[string]DeviceInfo{
"dev1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/dev1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1",
Permissions: "rw",
},
},
},
},
expectedAllocated: 1,
@ -266,9 +278,20 @@ func TestAllocate(t *testing.T) {
name: "Allocate healthy device with postAllocate hook",
devices: map[string]DeviceInfo{
"dev1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/dev1"},
Mounts: []string{"/dev"},
State: pluginapi.Healthy,
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1",
Permissions: "rw",
},
},
Mounts: []pluginapi.Mount{
{
HostPath: "/dev",
ContainerPath: "/dev",
},
},
Envs: map[string]string{
"testname": "testvalue",
},
@ -284,7 +307,13 @@ func TestAllocate(t *testing.T) {
devices: map[string]DeviceInfo{
"dev1": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/dev1"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/dev1",
ContainerPath: "/dev/dev1",
Permissions: "rw",
},
},
},
},
postAllocate: func(resp *pluginapi.AllocateResponse) error {
@ -375,7 +404,13 @@ func TestListAndWatch(t *testing.T) {
{
"fake_id": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
},
},
},
@ -386,7 +421,13 @@ func TestListAndWatch(t *testing.T) {
{
"fake_id": {
State: pluginapi.Healthy,
Nodes: []string{"/dev/intel-fpga-port.0"},
Nodes: []pluginapi.DeviceSpec{
{
HostPath: "/dev/intel-fpga-port.0",
ContainerPath: "/dev/intel-fpga-port.0",
Permissions: "rw",
},
},
},
},
},