github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

qat: rework bus-device-function handling

Browse Source 
The code was stripping out "0000:" (bus) and then adding
it back in several places.

That's not necessary so this change simplifies QAT VF addr
handling by operating using full BDF IDs.

Moveover, simplify function calls: use getDpdkDevice() once
for each VF device.

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>
This commit is contained in:
Mikko Ylinen 2020-12-07 15:40:49 +02:00
parent cc027857dd
commit ad8bbcea21
1 changed files with 33 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
cmd/qat_plugin/dpdkdrv/dpdkdrv.go
View File

@ -36,6 +36,7 @@ import (
const ( const (
uioDevicePath = "/dev" uioDevicePath = "/dev"
vfioDevicePath = "/dev/vfio" vfioDevicePath = "/dev/vfio"
vfioCtrlDevicePath = vfioDevicePath + "/vfio"
uioMountPath = "/sys/class/uio" uioMountPath = "/sys/class/uio"
pciDeviceDirectory = "/sys/bus/pci/devices" pciDeviceDirectory = "/sys/bus/pci/devices"
pciDriverDirectory = "/sys/bus/pci/drivers" pciDriverDirectory = "/sys/bus/pci/drivers"
@ -99,12 +100,10 @@ func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
} }
} }
func (dp *DevicePlugin) getDpdkDevice(id string) (string, error) { func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) {
devicePCIAdd := "0000:" + id
switch dp.dpdkDriver { switch dp.dpdkDriver {
// TODO: case "pci-generic" and "kernel":
case igbUio: case igbUio:
uioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, uioSuffix) uioDirPath := path.Join(dp.pciDeviceDir, vfBdf, uioSuffix)
files, err := ioutil.ReadDir(uioDirPath) files, err := ioutil.ReadDir(uioDirPath)
if err != nil { if err != nil {
return "", err return "", err
@ -115,7 +114,7 @@ func (dp *DevicePlugin) getDpdkDevice(id string) (string, error) {
return files[0].Name(), nil return files[0].Name(), nil
case vfioPci: case vfioPci:
vfioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, iommuGroupSuffix) vfioDirPath := path.Join(dp.pciDeviceDir, vfBdf, iommuGroupSuffix)
group, err := filepath.EvalSymlinks(vfioDirPath) group, err := filepath.EvalSymlinks(vfioDirPath)
if err != nil { if err != nil {
return "", errors.WithStack(err) return "", errors.WithStack(err)
@ -128,15 +127,8 @@ func (dp *DevicePlugin) getDpdkDevice(id string) (string, error) {
return "", errors.New("Unknown DPDK driver") return "", errors.New("Unknown DPDK driver")
} }
func (dp *DevicePlugin) getDpdkDeviceSpecs(id string) ([]pluginapi.DeviceSpec, error) { func (dp *DevicePlugin) getDpdkDeviceSpecs(dpdkDeviceName string) []pluginapi.DeviceSpec {
dpdkDeviceName, err := dp.getDpdkDevice(id)
if err != nil {
return nil, err
}
klog.V(1).Infof("%s device: corresponding DPDK device detected is %s", id, dpdkDeviceName)
switch dp.dpdkDriver { switch dp.dpdkDriver {
// TODO: case "pci-generic" and "kernel":
case igbUio: case igbUio:
//Setting up with uio //Setting up with uio
uioDev := path.Join(uioDevicePath, dpdkDeviceName) uioDev := path.Join(uioDevicePath, dpdkDeviceName)
@ -146,34 +138,28 @@ func (dp *DevicePlugin) getDpdkDeviceSpecs(id string) ([]pluginapi.DeviceSpec, e
ContainerPath: uioDev, ContainerPath: uioDev,
Permissions: "rw", Permissions: "rw",
}, },
}, nil }
case vfioPci: case vfioPci:
//Setting up with vfio //Setting up with vfio
vfioDev1 := path.Join(vfioDevicePath, dpdkDeviceName) vfioDev := path.Join(vfioDevicePath, dpdkDeviceName)
vfioDev2 := path.Join(vfioDevicePath, "/vfio")
return []pluginapi.DeviceSpec{ return []pluginapi.DeviceSpec{
{ {
HostPath: vfioDev1, HostPath: vfioDev,
ContainerPath: vfioDev1, ContainerPath: vfioDev,
Permissions: "rw", Permissions: "rw",
}, },
{ {
HostPath: vfioDev2, HostPath: vfioCtrlDevicePath,
ContainerPath: vfioDev2, ContainerPath: vfioCtrlDevicePath,
Permissions: "rw", Permissions: "rw",
}, },
}, nil
} }
default:
return nil, errors.New("Unknown DPDK driver") return nil
}
} }
func (dp *DevicePlugin) getDpdkMounts(id string) ([]pluginapi.Mount, error) { func (dp *DevicePlugin) getDpdkMounts(dpdkDeviceName string) []pluginapi.Mount {
dpdkDeviceName, err := dp.getDpdkDevice(id)
if err != nil {
return nil, err
}
switch dp.dpdkDriver { switch dp.dpdkDriver {
case igbUio: case igbUio:
//Setting up with uio mountpoints //Setting up with uio mountpoints
@ -183,13 +169,13 @@ func (dp *DevicePlugin) getDpdkMounts(id string) ([]pluginapi.Mount, error) {
HostPath: uioMountPoint, HostPath: uioMountPoint,
ContainerPath: uioMountPoint, ContainerPath: uioMountPoint,
}, },
}, nil }
case vfioPci: case vfioPci:
//No mountpoint for vfio needs to be populated //No mountpoint for vfio needs to be populated
return nil, nil return nil
default:
return nil
} }
return nil, errors.New("Unknown DPDK driver")
} }
func (dp *DevicePlugin) getDeviceID(pciAddr string) (string, error) { func (dp *DevicePlugin) getDeviceID(pciAddr string) (string, error) {
@ -202,16 +188,15 @@ func (dp *DevicePlugin) getDeviceID(pciAddr string) (string, error) {
} }
// bindDevice unbinds given device from kernel driver and binds to DPDK driver. // bindDevice unbinds given device from kernel driver and binds to DPDK driver.
func (dp *DevicePlugin) bindDevice(id string) error { func (dp *DevicePlugin) bindDevice(vfBdf string) error {
devicePCIAddr := "0000:" + id unbindDevicePath := path.Join(dp.pciDeviceDir, vfBdf, driverUnbindSuffix)
unbindDevicePath := path.Join(dp.pciDeviceDir, devicePCIAddr, driverUnbindSuffix)
// Unbind from the kernel driver // Unbind from the kernel driver
err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0600) err := ioutil.WriteFile(unbindDevicePath, []byte(vfBdf), 0600)
if err != nil { if err != nil {
return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id) return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", vfBdf)
} }
vfdevID, err := dp.getDeviceID(devicePCIAddr) vfdevID, err := dp.getDeviceID(vfBdf)
if err != nil { if err != nil {
return err return err
} }
@ -219,7 +204,7 @@ func (dp *DevicePlugin) bindDevice(id string) error {
//Bind to the the dpdk driver //Bind to the the dpdk driver
err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0600) err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0600)
if err != nil { if err != nil {
return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id) return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", vfBdf)
} }
return nil return nil
} }
@ -253,7 +238,7 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error
for _, cresp := range response.ContainerResponses { for _, cresp := range response.ContainerResponses {
counter := 0 counter := 0
for k := range cresp.Envs { for k := range cresp.Envs {
tempMap[strings.Join([]string{"QAT", strconv.Itoa(counter)}, "")] = cresp.Envs[k] tempMap[strings.Join([]string{envVarPrefix, strconv.Itoa(counter)}, "")] = cresp.Envs[k]
counter++ counter++
} }
cresp.Envs = tempMap cresp.Envs = tempMap
@ -288,31 +273,28 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
break break
} }
vfpciaddr := strings.TrimPrefix(file.Name(), "0000:")
// initialize newly found devices which aren't bound to DPDK driver yet // initialize newly found devices which aren't bound to DPDK driver yet
if driver != dp.dpdkDriver { if driver != dp.dpdkDriver {
err = dp.bindDevice(vfpciaddr) err = dp.bindDevice(file.Name())
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
devNodes, err := dp.getDpdkDeviceSpecs(vfpciaddr) dpdkDeviceName, err := dp.getDpdkDevice(file.Name())
if err != nil {
return nil, err
}
devMounts, err := dp.getDpdkMounts(vfpciaddr)
if err != nil { if err != nil {
return nil, err return nil, err
} }
klog.V(1).Infof("%s device: corresponding DPDK device detected is %s", file.Name(), dpdkDeviceName)
envs := map[string]string{ envs := map[string]string{
fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(),
} }
devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, devMounts, envs)
devTree.AddDevice("generic", vfpciaddr, devinfo) devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs)
devTree.AddDevice("generic", file.Name(), devinfo)
} }
} }