github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing minor bugs with QAT plugin

Browse Source 
- To ensure that the device plugin continues to run in case a certain QAT device family is not detected on the node
- Updated the extended resource name to qat/generic in demo/crypto-perf-dpdk-pod-requesting-qat.yaml file
- Added explicit check for QAT device ID to avoid counting other DPDK bound devices as QAT devices
This commit is contained in:
ssehgal 2018-08-08 17:33:10 +01:00
parent be07ed777c
commit 8e37adfb42
3 changed files with 213 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/qat_plugin/qat_plugin.go
View File

@ -34,20 +34,18 @@ import (
)
const (
uioDevicePath = "/dev"
vfioDevicePath = "/dev/vfio"
uioMountPath = "/sys/class/uio"
pciDeviceDirectory = "/sys/bus/pci/devices"
pciDriverDirectory = "/sys/bus/pci/drivers"
uioSuffix = "uio"
iommuGroupSuffix = "iommu_group"
sysfsIommuGroupPrefix = "/sys/kernel/iommu_groups"
newIDSuffix = "new_id"
driverUnbindSuffix = "driver/unbind"
qatDeviceRE = "[0-9|a-f][0-9|a-f]:[0-9|a-f][0-9|a-f]\\.[0-9|a-f].*"
vendorPrefix = "8086 "
uioDevicePath = "/dev"
vfioDevicePath = "/dev/vfio"
uioMountPath = "/sys/class/uio"
pciDeviceDirectory = "/sys/bus/pci/devices"
pciDriverDirectory = "/sys/bus/pci/drivers"
uioSuffix = "uio"
iommuGroupSuffix = "iommu_group"
newIDSuffix = "new_id"
driverUnbindSuffix = "driver/unbind"
vendorPrefix = "8086 "
namespace = "qat"
namespace = "qat.intel.com"
)
type devicePlugin struct {
@ -103,7 +101,7 @@ func (dp *devicePlugin) getDpdkDevice(id string) (string, error) {
if err != nil {
return "", errors.WithStack(err)
}
s := strings.TrimPrefix(group, sysfsIommuGroupPrefix)
s := path.Base(group)
fmt.Printf("The vfio device group detected is %v\n", s)
return s, nil
}
@ -170,10 +168,9 @@ func (dp *devicePlugin) bindDevice(id string) error {
// Unbind from the kernel driver
err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0644)
if err != nil {
return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s: %v", id)
return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id)
}
vfdevID, err := dp.getDeviceID(devicePCIAddr)
if err != nil {
return err
@ -182,9 +179,8 @@ func (dp *devicePlugin) bindDevice(id string) error {
//Bind to the the dpdk driver
err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0644)
if err != nil {
return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s: %v", id)
return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id)
}
return nil
}
@ -203,6 +199,13 @@ func isValidDpdkDeviceDriver(dpdkDriver string) bool {
}
return false
}
func isValidVfDeviceID(vfDevID string) bool {
switch vfDevID {
case "0442", "0443", "37c9", "19e3":
return true
}
return false
}
func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
devTree := deviceplugin.NewDeviceTree()
@ -210,7 +213,8 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
for _, driver := range append(dp.kernelVfDrivers, dp.dpdkDriver) {
files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver))
if err != nil {
return nil, errors.Wrapf(err, "Can't read sysfs for driver %s", driver)
fmt.Printf("Can't read sysfs for driver as Driver %s is not available: Skipping\n", driver)
continue
}
n := 0
@ -218,6 +222,13 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
if !strings.HasPrefix(file.Name(), "0000:") {
continue
}
vfdevID, err := dp.getDeviceID(file.Name())
if err != nil {
return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name())
}
if !isValidVfDeviceID(vfdevID) {
continue
}
n = n + 1 // increment after all junk got filtered out
if n > dp.maxDevices {
@ -228,7 +239,7 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
// initialize newly found devices which aren't bound to DPDK driver yet
if driver != dp.dpdkDriver {
err := dp.bindDevice(vfpciaddr)
err = dp.bindDevice(vfpciaddr)
if err != nil {
return nil, err
}
@ -242,13 +253,14 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
if err != nil {
return nil, err
}
deviceName := strings.TrimSuffix(namespace, ".intel.com")
devinfo := deviceplugin.DeviceInfo{
State: pluginapi.Healthy,
Nodes: devNodes,
Mounts: devMounts,
Envs: map[string]string{
fmt.Sprintf("%s%d", namespace, n): file.Name(),
fmt.Sprintf("%s%d", strings.ToUpper(deviceName), n): file.Name(),
},
}

192
cmd/qat_plugin/qat_plugin_test.go
View File

@ -31,20 +31,34 @@ func init() {
debug.Activate()
}
func createTestFiles(prefix string, dirs []string, files map[string][]byte) error {
func createTestFiles(prefix string, dirs []string, files map[string][]byte, symlinks map[string]string) error {
for _, dir := range dirs {
err := os.MkdirAll(path.Join(prefix, dir), 0755)
if err != nil {
return errors.Wrap(err, "Failed to create fake device directory")
}
}
for filename, body := range files {
err := ioutil.WriteFile(path.Join(prefix, filename), body, 0644)
if err != nil {
return errors.Wrap(err, "Failed to create fake vendor file")
}
}
}
for link, target := range symlinks {
err := os.MkdirAll(path.Join(prefix, target), 0755)
if err != nil {
return errors.Wrap(err, "Failed to create fake symlink target directory")
}
err = os.Symlink(path.Join(prefix, target), path.Join(prefix, link))
if err != nil {
return errors.Wrap(err, "Failed to create fake symlink")
}
}
return nil
}
@ -58,18 +72,19 @@ func TestScanPrivate(t *testing.T) {
kernelVfDrivers []string
dirs []string
files map[string][]byte
symlinks map[string]string
expectedErr bool
maxDevNum int
expectedDevNum int
}{
{
name: "Return error for uninitialized device plugin",
expectedErr: true,
name: "No error returned for uninitialized device plugin",
},
{
name: "Only DPDK driver is set and no devs allowed",
dpdkDriver: "igb_uio",
dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"},
name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined",
dpdkDriver: "igb_uio",
dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"},
expectedErr: true,
},
{
name: "Only DPDK driver is set and no dev exists",
@ -93,6 +108,16 @@ func TestScanPrivate(t *testing.T) {
maxDevNum: 1,
expectedErr: true,
},
{
name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined",
dpdkDriver: "igb_uio",
dirs: []string{
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
},
maxDevNum: 1,
expectedErr: true,
},
{
name: "igb_uio DPDKdriver with one DPDK bound device",
dpdkDriver: "igb_uio",
@ -100,8 +125,11 @@ func TestScanPrivate(t *testing.T) {
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
},
maxDevNum: 1,
expectedDevNum: 1,
expectedDevNum: 0,
},
{
name: "igb_uio DPDKdriver with one kernel bound device, but unbindable",
@ -142,11 +170,11 @@ func TestScanPrivate(t *testing.T) {
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
},
maxDevNum: 1,
expectedErr: true,
maxDevNum: 1,
expectedDevNum: 0,
},
{
name: "igb_uio DPDKdriver with one kernel bound device",
name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
@ -161,16 +189,152 @@ func TestScanPrivate(t *testing.T) {
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedDevNum: 0,
},
{
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/igb_uio",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
"sys/bus/pci/devices/0000:02:00.0/driver",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedDevNum: 1,
},
{
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/igb_uio",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/driver",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedErr: true,
},
{
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but no uio device is found",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/igb_uio",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/uio",
"sys/bus/pci/devices/0000:02:00.0/driver",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedErr: true,
},
{
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where the available devices on the system are 2 but maxNumDevices=1] ",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/igb_uio",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
"sys/bus/pci/devices/0000:02:00.0/driver",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.1",
"sys/bus/pci/devices/0000:02:00.1/uio/sometestfile",
"sys/bus/pci/devices/0000:02:00.1/driver",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"),
},
maxDevNum: 1,
expectedDevNum: 1,
},
{
name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
dpdkDriver: "igb_uio",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/vfio-pci",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/driver",
"sys/bus/pci/devices/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedDevNum: 0,
},
{
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)",
dpdkDriver: "vfio-pci",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/vfio-pci",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/driver",
"sys/bus/pci/devices/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
},
symlinks: map[string]string{
"sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
},
maxDevNum: 1,
expectedDevNum: 1,
},
{
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but symlink is broken",
dpdkDriver: "vfio-pci",
kernelVfDrivers: []string{"c6xxvf"},
dirs: []string{
"sys/bus/pci/drivers/vfio-pci",
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/driver",
"sys/bus/pci/devices/0000:02:00.0",
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
},
files: map[string][]byte{
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
},
maxDevNum: 1,
expectedErr: true,
},
}
for _, tt := range tcases {
if err := os.MkdirAll(tmpdir, 0755); err != nil {
t.Fatal(err)
}
if err := createTestFiles(tmpdir, tt.dirs, tt.files); err != nil {
if err := createTestFiles(tmpdir, tt.dirs, tt.files, tt.symlinks); err != nil {
t.Fatalf("%+v", err)
}

4
demo/crypto-perf-dpdk-pod-requesting-qat.yaml
View File

@ -16,12 +16,12 @@ spec:
requests:
cpu: "3"
memory: "128Mi"
intel.com/qat: '4'
qat.intel.com/generic: '4'
hugepages-2Mi: "1Gi"
limits:
cpu: "3"
memory: "128Mi"
intel.com/qat: '4'
qat.intel.com/generic: '4'
hugepages-2Mi: "1Gi"
securityContext:
capabilities: