diff --git a/cmd/qat_plugin/qat_plugin.go b/cmd/qat_plugin/qat_plugin.go index 504cc7fb..150d2b09 100644 --- a/cmd/qat_plugin/qat_plugin.go +++ b/cmd/qat_plugin/qat_plugin.go @@ -34,20 +34,18 @@ import ( ) const ( - uioDevicePath = "/dev" - vfioDevicePath = "/dev/vfio" - uioMountPath = "/sys/class/uio" - pciDeviceDirectory = "/sys/bus/pci/devices" - pciDriverDirectory = "/sys/bus/pci/drivers" - uioSuffix = "uio" - iommuGroupSuffix = "iommu_group" - sysfsIommuGroupPrefix = "/sys/kernel/iommu_groups" - newIDSuffix = "new_id" - driverUnbindSuffix = "driver/unbind" - qatDeviceRE = "[0-9|a-f][0-9|a-f]:[0-9|a-f][0-9|a-f]\\.[0-9|a-f].*" - vendorPrefix = "8086 " + uioDevicePath = "/dev" + vfioDevicePath = "/dev/vfio" + uioMountPath = "/sys/class/uio" + pciDeviceDirectory = "/sys/bus/pci/devices" + pciDriverDirectory = "/sys/bus/pci/drivers" + uioSuffix = "uio" + iommuGroupSuffix = "iommu_group" + newIDSuffix = "new_id" + driverUnbindSuffix = "driver/unbind" + vendorPrefix = "8086 " - namespace = "qat" + namespace = "qat.intel.com" ) type devicePlugin struct { @@ -103,7 +101,7 @@ func (dp *devicePlugin) getDpdkDevice(id string) (string, error) { if err != nil { return "", errors.WithStack(err) } - s := strings.TrimPrefix(group, sysfsIommuGroupPrefix) + s := path.Base(group) fmt.Printf("The vfio device group detected is %v\n", s) return s, nil } @@ -170,10 +168,9 @@ func (dp *devicePlugin) bindDevice(id string) error { // Unbind from the kernel driver err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0644) if err != nil { - return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s: %v", id) + return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id) } - vfdevID, err := dp.getDeviceID(devicePCIAddr) if err != nil { return err @@ -182,9 +179,8 @@ func (dp *devicePlugin) bindDevice(id string) error { //Bind to the the dpdk driver err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0644) if err != nil { - return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s: %v", id) + return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id) } - return nil } @@ -203,6 +199,13 @@ func isValidDpdkDeviceDriver(dpdkDriver string) bool { } return false } +func isValidVfDeviceID(vfDevID string) bool { + switch vfDevID { + case "0442", "0443", "37c9", "19e3": + return true + } + return false +} func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { devTree := deviceplugin.NewDeviceTree() @@ -210,7 +213,8 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { for _, driver := range append(dp.kernelVfDrivers, dp.dpdkDriver) { files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) if err != nil { - return nil, errors.Wrapf(err, "Can't read sysfs for driver %s", driver) + fmt.Printf("Can't read sysfs for driver as Driver %s is not available: Skipping\n", driver) + continue } n := 0 @@ -218,6 +222,13 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { if !strings.HasPrefix(file.Name(), "0000:") { continue } + vfdevID, err := dp.getDeviceID(file.Name()) + if err != nil { + return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name()) + } + if !isValidVfDeviceID(vfdevID) { + continue + } n = n + 1 // increment after all junk got filtered out if n > dp.maxDevices { @@ -228,7 +239,7 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { // initialize newly found devices which aren't bound to DPDK driver yet if driver != dp.dpdkDriver { - err := dp.bindDevice(vfpciaddr) + err = dp.bindDevice(vfpciaddr) if err != nil { return nil, err } @@ -242,13 +253,14 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { if err != nil { return nil, err } + deviceName := strings.TrimSuffix(namespace, ".intel.com") devinfo := deviceplugin.DeviceInfo{ State: pluginapi.Healthy, Nodes: devNodes, Mounts: devMounts, Envs: map[string]string{ - fmt.Sprintf("%s%d", namespace, n): file.Name(), + fmt.Sprintf("%s%d", strings.ToUpper(deviceName), n): file.Name(), }, } diff --git a/cmd/qat_plugin/qat_plugin_test.go b/cmd/qat_plugin/qat_plugin_test.go index 2529ba90..40e465c1 100644 --- a/cmd/qat_plugin/qat_plugin_test.go +++ b/cmd/qat_plugin/qat_plugin_test.go @@ -31,20 +31,34 @@ func init() { debug.Activate() } -func createTestFiles(prefix string, dirs []string, files map[string][]byte) error { +func createTestFiles(prefix string, dirs []string, files map[string][]byte, symlinks map[string]string) error { for _, dir := range dirs { + err := os.MkdirAll(path.Join(prefix, dir), 0755) if err != nil { return errors.Wrap(err, "Failed to create fake device directory") } + } for filename, body := range files { + err := ioutil.WriteFile(path.Join(prefix, filename), body, 0644) if err != nil { return errors.Wrap(err, "Failed to create fake vendor file") } - } + } + for link, target := range symlinks { + + err := os.MkdirAll(path.Join(prefix, target), 0755) + if err != nil { + return errors.Wrap(err, "Failed to create fake symlink target directory") + } + err = os.Symlink(path.Join(prefix, target), path.Join(prefix, link)) + if err != nil { + return errors.Wrap(err, "Failed to create fake symlink") + } + } return nil } @@ -58,18 +72,19 @@ func TestScanPrivate(t *testing.T) { kernelVfDrivers []string dirs []string files map[string][]byte + symlinks map[string]string expectedErr bool maxDevNum int expectedDevNum int }{ { - name: "Return error for uninitialized device plugin", - expectedErr: true, + name: "No error returned for uninitialized device plugin", }, { - name: "Only DPDK driver is set and no devs allowed", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"}, + name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined", + dpdkDriver: "igb_uio", + dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"}, + expectedErr: true, }, { name: "Only DPDK driver is set and no dev exists", @@ -93,6 +108,16 @@ func TestScanPrivate(t *testing.T) { maxDevNum: 1, expectedErr: true, }, + { + name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined", + dpdkDriver: "igb_uio", + dirs: []string{ + "sys/bus/pci/drivers/igb_uio/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", + }, + maxDevNum: 1, + expectedErr: true, + }, { name: "igb_uio DPDKdriver with one DPDK bound device", dpdkDriver: "igb_uio", @@ -100,8 +125,11 @@ func TestScanPrivate(t *testing.T) { "sys/bus/pci/drivers/igb_uio/0000:02:00.0", "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), + }, maxDevNum: 1, - expectedDevNum: 1, + expectedDevNum: 0, }, { name: "igb_uio DPDKdriver with one kernel bound device, but unbindable", @@ -142,11 +170,11 @@ func TestScanPrivate(t *testing.T) { "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), }, - maxDevNum: 1, - expectedErr: true, + maxDevNum: 1, + expectedDevNum: 0, }, { - name: "igb_uio DPDKdriver with one kernel bound device", + name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ @@ -161,16 +189,152 @@ func TestScanPrivate(t *testing.T) { "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), }, maxDevNum: 1, + expectedDevNum: 0, + }, + { + name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ", + dpdkDriver: "igb_uio", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:00.0/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + }, + maxDevNum: 1, expectedDevNum: 1, }, + { + name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ", + dpdkDriver: "igb_uio", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + }, + maxDevNum: 1, + expectedErr: true, + }, + { + name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but no uio device is found", + dpdkDriver: "igb_uio", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/uio", + "sys/bus/pci/devices/0000:02:00.0/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + }, + maxDevNum: 1, + expectedErr: true, + }, + { + name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where the available devices on the system are 2 but maxNumDevices=1] ", + dpdkDriver: "igb_uio", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.1", + "sys/bus/pci/devices/0000:02:00.1/uio/sometestfile", + "sys/bus/pci/devices/0000:02:00.1/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"), + }, + maxDevNum: 1, + expectedDevNum: 1, + }, + { + name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", + dpdkDriver: "igb_uio", + kernelVfDrivers: []string{"c6xxvf"}, + + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), + "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + }, + maxDevNum: 1, + expectedDevNum: 0, + }, + { + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)", + dpdkDriver: "vfio-pci", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + }, + maxDevNum: 1, + expectedDevNum: 1, + }, + { + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but symlink is broken", + dpdkDriver: "vfio-pci", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + }, + maxDevNum: 1, + expectedErr: true, + }, } - for _, tt := range tcases { - if err := os.MkdirAll(tmpdir, 0755); err != nil { t.Fatal(err) } - if err := createTestFiles(tmpdir, tt.dirs, tt.files); err != nil { + + if err := createTestFiles(tmpdir, tt.dirs, tt.files, tt.symlinks); err != nil { t.Fatalf("%+v", err) } diff --git a/demo/crypto-perf-dpdk-pod-requesting-qat.yaml b/demo/crypto-perf-dpdk-pod-requesting-qat.yaml index 30142ae0..b64fe8ba 100644 --- a/demo/crypto-perf-dpdk-pod-requesting-qat.yaml +++ b/demo/crypto-perf-dpdk-pod-requesting-qat.yaml @@ -16,12 +16,12 @@ spec: requests: cpu: "3" memory: "128Mi" - intel.com/qat: '4' + qat.intel.com/generic: '4' hugepages-2Mi: "1Gi" limits: cpu: "3" memory: "128Mi" - intel.com/qat: '4' + qat.intel.com/generic: '4' hugepages-2Mi: "1Gi" securityContext: capabilities: