mirror of
https://github.com/intel/intel-device-plugins-for-kubernetes.git
synced 2025-06-03 03:59:37 +00:00
Fixing minor bugs with QAT plugin
- To ensure that the device plugin continues to run in case a certain QAT device family is not detected on the node - Updated the extended resource name to qat/generic in demo/crypto-perf-dpdk-pod-requesting-qat.yaml file - Added explicit check for QAT device ID to avoid counting other DPDK bound devices as QAT devices
This commit is contained in:
ssehgal
parent
be07ed777c
commit
8e37adfb42
@ -41,13 +41,11 @@ const (
|
||||
pciDriverDirectory = "/sys/bus/pci/drivers"
|
||||
uioSuffix = "uio"
|
||||
iommuGroupSuffix = "iommu_group"
|
||||
sysfsIommuGroupPrefix = "/sys/kernel/iommu_groups"
|
||||
newIDSuffix = "new_id"
|
||||
driverUnbindSuffix = "driver/unbind"
|
||||
qatDeviceRE = "[0-9|a-f][0-9|a-f]:[0-9|a-f][0-9|a-f]\\.[0-9|a-f].*"
|
||||
vendorPrefix = "8086 "
|
||||
|
||||
namespace = "qat"
|
||||
namespace = "qat.intel.com"
|
||||
)
|
||||
|
||||
type devicePlugin struct {
|
||||
@ -103,7 +101,7 @@ func (dp *devicePlugin) getDpdkDevice(id string) (string, error) {
|
||||
if err != nil {
|
||||
return "", errors.WithStack(err)
|
||||
}
|
||||
s := strings.TrimPrefix(group, sysfsIommuGroupPrefix)
|
||||
s := path.Base(group)
|
||||
fmt.Printf("The vfio device group detected is %v\n", s)
|
||||
return s, nil
|
||||
}
|
||||
@ -170,10 +168,9 @@ func (dp *devicePlugin) bindDevice(id string) error {
|
||||
// Unbind from the kernel driver
|
||||
err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0644)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s: %v", id)
|
||||
return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id)
|
||||
|
||||
}
|
||||
|
||||
vfdevID, err := dp.getDeviceID(devicePCIAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -182,9 +179,8 @@ func (dp *devicePlugin) bindDevice(id string) error {
|
||||
//Bind to the the dpdk driver
|
||||
err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0644)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s: %v", id)
|
||||
return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -203,6 +199,13 @@ func isValidDpdkDeviceDriver(dpdkDriver string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
func isValidVfDeviceID(vfDevID string) bool {
|
||||
switch vfDevID {
|
||||
case "0442", "0443", "37c9", "19e3":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
|
||||
devTree := deviceplugin.NewDeviceTree()
|
||||
@ -210,7 +213,8 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
|
||||
for _, driver := range append(dp.kernelVfDrivers, dp.dpdkDriver) {
|
||||
files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Can't read sysfs for driver %s", driver)
|
||||
fmt.Printf("Can't read sysfs for driver as Driver %s is not available: Skipping\n", driver)
|
||||
continue
|
||||
}
|
||||
|
||||
n := 0
|
||||
@ -218,6 +222,13 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
|
||||
if !strings.HasPrefix(file.Name(), "0000:") {
|
||||
continue
|
||||
}
|
||||
vfdevID, err := dp.getDeviceID(file.Name())
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name())
|
||||
}
|
||||
if !isValidVfDeviceID(vfdevID) {
|
||||
continue
|
||||
}
|
||||
n = n + 1 // increment after all junk got filtered out
|
||||
|
||||
if n > dp.maxDevices {
|
||||
@ -228,7 +239,7 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
|
||||
|
||||
// initialize newly found devices which aren't bound to DPDK driver yet
|
||||
if driver != dp.dpdkDriver {
|
||||
err := dp.bindDevice(vfpciaddr)
|
||||
err = dp.bindDevice(vfpciaddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -242,13 +253,14 @@ func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deviceName := strings.TrimSuffix(namespace, ".intel.com")
|
||||
|
||||
devinfo := deviceplugin.DeviceInfo{
|
||||
State: pluginapi.Healthy,
|
||||
Nodes: devNodes,
|
||||
Mounts: devMounts,
|
||||
Envs: map[string]string{
|
||||
fmt.Sprintf("%s%d", namespace, n): file.Name(),
|
||||
fmt.Sprintf("%s%d", strings.ToUpper(deviceName), n): file.Name(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -31,20 +31,34 @@ func init() {
|
||||
debug.Activate()
|
||||
}
|
||||
|
||||
func createTestFiles(prefix string, dirs []string, files map[string][]byte) error {
|
||||
func createTestFiles(prefix string, dirs []string, files map[string][]byte, symlinks map[string]string) error {
|
||||
for _, dir := range dirs {
|
||||
|
||||
err := os.MkdirAll(path.Join(prefix, dir), 0755)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed to create fake device directory")
|
||||
}
|
||||
|
||||
}
|
||||
for filename, body := range files {
|
||||
|
||||
err := ioutil.WriteFile(path.Join(prefix, filename), body, 0644)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed to create fake vendor file")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
for link, target := range symlinks {
|
||||
|
||||
err := os.MkdirAll(path.Join(prefix, target), 0755)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed to create fake symlink target directory")
|
||||
}
|
||||
err = os.Symlink(path.Join(prefix, target), path.Join(prefix, link))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed to create fake symlink")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -58,18 +72,19 @@ func TestScanPrivate(t *testing.T) {
|
||||
kernelVfDrivers []string
|
||||
dirs []string
|
||||
files map[string][]byte
|
||||
symlinks map[string]string
|
||||
expectedErr bool
|
||||
maxDevNum int
|
||||
expectedDevNum int
|
||||
}{
|
||||
{
|
||||
name: "Return error for uninitialized device plugin",
|
||||
expectedErr: true,
|
||||
name: "No error returned for uninitialized device plugin",
|
||||
},
|
||||
{
|
||||
name: "Only DPDK driver is set and no devs allowed",
|
||||
name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined",
|
||||
dpdkDriver: "igb_uio",
|
||||
dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"},
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "Only DPDK driver is set and no dev exists",
|
||||
@ -94,14 +109,27 @@ func TestScanPrivate(t *testing.T) {
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one DPDK bound device",
|
||||
name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined",
|
||||
dpdkDriver: "igb_uio",
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 1,
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one DPDK bound device",
|
||||
dpdkDriver: "igb_uio",
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 0,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device, but unbindable",
|
||||
@ -143,10 +171,10 @@ func TestScanPrivate(t *testing.T) {
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedErr: true,
|
||||
expectedDevNum: 0,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device",
|
||||
name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
@ -161,16 +189,152 @@ func TestScanPrivate(t *testing.T) {
|
||||
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 0,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 1,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but no uio device is found",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/uio",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where the available devices on the system are 2 but maxNumDevices=1] ",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/igb_uio",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.1",
|
||||
"sys/bus/pci/devices/0000:02:00.1/uio/sometestfile",
|
||||
"sys/bus/pci/devices/0000:02:00.1/driver",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 1,
|
||||
},
|
||||
{
|
||||
name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
|
||||
dpdkDriver: "igb_uio",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/vfio-pci",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
"sys/bus/pci/devices/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
||||
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 0,
|
||||
},
|
||||
{
|
||||
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)",
|
||||
dpdkDriver: "vfio-pci",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/vfio-pci",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
"sys/bus/pci/devices/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
||||
},
|
||||
symlinks: map[string]string{
|
||||
"sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedDevNum: 1,
|
||||
},
|
||||
{
|
||||
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but symlink is broken",
|
||||
dpdkDriver: "vfio-pci",
|
||||
kernelVfDrivers: []string{"c6xxvf"},
|
||||
dirs: []string{
|
||||
"sys/bus/pci/drivers/vfio-pci",
|
||||
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver",
|
||||
"sys/bus/pci/devices/0000:02:00.0",
|
||||
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
||||
},
|
||||
files: map[string][]byte{
|
||||
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
||||
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
||||
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
||||
},
|
||||
maxDevNum: 1,
|
||||
expectedErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tcases {
|
||||
|
||||
if err := os.MkdirAll(tmpdir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := createTestFiles(tmpdir, tt.dirs, tt.files); err != nil {
|
||||
|
||||
if err := createTestFiles(tmpdir, tt.dirs, tt.files, tt.symlinks); err != nil {
|
||||
t.Fatalf("%+v", err)
|
||||
}
|
||||
|
||||
|
||||
@ -16,12 +16,12 @@ spec:
|
||||
requests:
|
||||
cpu: "3"
|
||||
memory: "128Mi"
|
||||
intel.com/qat: '4'
|
||||
qat.intel.com/generic: '4'
|
||||
hugepages-2Mi: "1Gi"
|
||||
limits:
|
||||
cpu: "3"
|
||||
memory: "128Mi"
|
||||
intel.com/qat: '4'
|
||||
qat.intel.com/generic: '4'
|
||||
hugepages-2Mi: "1Gi"
|
||||
securityContext:
|
||||
capabilities:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user