mirror of
https://github.com/intel/intel-device-plugins-for-kubernetes.git
synced 2025-06-03 03:59:37 +00:00
f8dbc896a1
Move the framework, and the qat driver, to use `klog`
for logging and debug.
This has a some noticeable effects:
1) Our default log output gains a bunch of annotation:
From:
QAT device plugin started in 'dpdk' mode
To:
I0312 11:51:02.057728 6053 qat_plugin.go:64] QAT device plugin started in 'dpdk' mode
(there is now a command line option to drop those annotations if
necessary).
2) We gain a bunch of command line parameters from klog for controlling log
levels and output. We go from 5 arguments to 17:
---
Usage of ./cmd/qat_plugin/qat_plugin:
-add_dir_header
If true, adds the file directory to the header
-alsologtostderr
log to standard error as well as files
-debug
enable debug output
-dpdk-driver string
DPDK Device driver for configuring the QAT device (default "vfio-pci")
-kernel-vf-drivers string
Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx (default "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf")
-log_backtrace_at value
when logging hits line file:N, emit a stack trace
-log_dir string
If non-empty, write log files in this directory
-log_file string
If non-empty, use this log file
-log_file_max_size uint
Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
-logtostderr
log to standard error instead of files (default true)
-max-num-devices int
maximum number of QAT devices to be provided to the QuickAssist device plugin (default 32)
-mode string
plugin mode which can be either dpdk (default) or kernel (default "dpdk")
-skip_headers
If true, avoid header prefixes in the log messages
-skip_log_headers
If true, avoid headers when opening log files
-stderrthreshold value
logs at or above this threshold go to stderr (default 2)
-v value
number for the log level verbosity
-vmodule value
comma-separated list of pattern=N settings for file-filtered logging
---
3) Our `-debug` flag is now replaced by the `klog` `-v n` flag.
*NOTE:* This is potentially a minor breaking change. Applying
this debug overlay to any previous (pre-klog edit) images will
cause the container to fail to launch, as it will not recognise
the new `-v` arguments.
We also update the kustomize deployment to move from using
DEBUG env vars to adding a VERBOSITY var that controls both
the log verbosity and now the debug mode enabling.
Signed-off-by: Graham Whaley <graham.whaley@intel.com>
419 lines
14 KiB
Go
419 lines
14 KiB
Go
// Copyright 2018 Intel Corporation. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package dpdkdrv
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
|
|
)
|
|
|
|
func init() {
|
|
flag.Set("v", "4") //Enable debug output
|
|
}
|
|
|
|
func createTestFiles(prefix string, dirs []string, files map[string][]byte, symlinks map[string]string) error {
|
|
for _, dir := range dirs {
|
|
|
|
err := os.MkdirAll(path.Join(prefix, dir), 0755)
|
|
if err != nil {
|
|
return errors.Wrap(err, "Failed to create fake device directory")
|
|
}
|
|
|
|
}
|
|
for filename, body := range files {
|
|
|
|
err := ioutil.WriteFile(path.Join(prefix, filename), body, 0644)
|
|
if err != nil {
|
|
return errors.Wrap(err, "Failed to create fake vendor file")
|
|
}
|
|
|
|
}
|
|
for link, target := range symlinks {
|
|
|
|
err := os.MkdirAll(path.Join(prefix, target), 0755)
|
|
if err != nil {
|
|
return errors.Wrap(err, "Failed to create fake symlink target directory")
|
|
}
|
|
err = os.Symlink(path.Join(prefix, target), path.Join(prefix, link))
|
|
if err != nil {
|
|
return errors.Wrap(err, "Failed to create fake symlink")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestScanPrivate(t *testing.T) {
|
|
tmpdir := fmt.Sprintf("/tmp/qatplugin-TestScanPrivate-%d", time.Now().Unix())
|
|
pciDrvDir := path.Join(tmpdir, "sys/bus/pci/drivers")
|
|
pciDevDir := path.Join(tmpdir, "sys/bus/pci/devices")
|
|
tcases := []struct {
|
|
name string
|
|
dpdkDriver string
|
|
kernelVfDrivers []string
|
|
dirs []string
|
|
files map[string][]byte
|
|
symlinks map[string]string
|
|
expectedErr bool
|
|
maxDevNum int
|
|
expectedDevNum int
|
|
}{
|
|
{
|
|
name: "No error returned for uninitialized device plugin",
|
|
},
|
|
{
|
|
name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"},
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "Only DPDK driver is set and no dev exists",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{"sys/bus/pci/drivers/igb_uio/1111:test"},
|
|
},
|
|
{
|
|
name: "igb_uio DPDK driver with no valid DPDK device under uio directory",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:02:00.0"},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with no DPDK bound devices",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio",
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one DPDK bound device",
|
|
dpdkDriver: "igb_uio",
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 0,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device, but unbindable",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device which gets lost after unbinding",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "Broken igb_uio DPDKdriver with one kernel bound device",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 0,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
|
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 0,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 1,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but no uio device is found",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
{
|
|
name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where the available devices on the system are 2 but maxNumDevices=1] ",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/igb_uio",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.1",
|
|
"sys/bus/pci/devices/0000:02:00.1/uio/sometestfile",
|
|
"sys/bus/pci/devices/0000:02:00.1/driver",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 1,
|
|
},
|
|
{
|
|
name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)",
|
|
dpdkDriver: "igb_uio",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/vfio-pci",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
"sys/bus/pci/devices/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"),
|
|
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 0,
|
|
},
|
|
{
|
|
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)",
|
|
dpdkDriver: "vfio-pci",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/vfio-pci",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
"sys/bus/pci/devices/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
|
},
|
|
symlinks: map[string]string{
|
|
"sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
|
|
},
|
|
maxDevNum: 1,
|
|
expectedDevNum: 1,
|
|
},
|
|
{
|
|
name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) but symlink is broken",
|
|
dpdkDriver: "vfio-pci",
|
|
kernelVfDrivers: []string{"c6xxvf"},
|
|
dirs: []string{
|
|
"sys/bus/pci/drivers/vfio-pci",
|
|
"sys/bus/pci/drivers/c6xxvf/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/driver",
|
|
"sys/bus/pci/devices/0000:02:00.0",
|
|
"sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile",
|
|
},
|
|
files: map[string][]byte{
|
|
"sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"),
|
|
"sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"),
|
|
"sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"),
|
|
},
|
|
maxDevNum: 1,
|
|
expectedErr: true,
|
|
},
|
|
}
|
|
for _, tt := range tcases {
|
|
if err := os.MkdirAll(tmpdir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := createTestFiles(tmpdir, tt.dirs, tt.files, tt.symlinks); err != nil {
|
|
t.Fatalf("%+v", err)
|
|
}
|
|
|
|
dp := &DevicePlugin{
|
|
maxDevices: tt.maxDevNum,
|
|
pciDriverDir: pciDrvDir,
|
|
pciDeviceDir: pciDevDir,
|
|
dpdkDriver: tt.dpdkDriver,
|
|
kernelVfDrivers: tt.kernelVfDrivers,
|
|
}
|
|
|
|
tree, err := dp.scan()
|
|
if tt.expectedErr && err == nil {
|
|
t.Errorf("Test case '%s': expected error, but got success", tt.name)
|
|
}
|
|
if !tt.expectedErr && err != nil {
|
|
t.Errorf("Test case '%s': got unexpected error: %+v", tt.name, err)
|
|
}
|
|
if len(tree) != tt.expectedDevNum {
|
|
t.Errorf("Test case '%s': expected %d, but got %d devices", tt.name, tt.expectedDevNum, len(tree))
|
|
}
|
|
|
|
if err = os.RemoveAll(tmpdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
func eleInSlice(a string, list []string) bool {
|
|
for _, b := range list {
|
|
if b == a {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func TestPostAllocate(t *testing.T) {
|
|
response := new(pluginapi.AllocateResponse)
|
|
cresp := new(pluginapi.ContainerAllocateResponse)
|
|
response.ContainerResponses = append(response.ContainerResponses, cresp)
|
|
testMap := map[string]string{
|
|
"QAT29": "03:04.1",
|
|
"QAT13": "03:04.2",
|
|
"QAT6": "03:04.3",
|
|
"QAT21": "03:04.4",
|
|
}
|
|
response.ContainerResponses[0].Envs = testMap
|
|
resultKey := []string{
|
|
"QAT0",
|
|
"QAT1",
|
|
"QAT2",
|
|
"QAT3",
|
|
}
|
|
expectedValues := map[string]struct{}{
|
|
"03:04.1": {},
|
|
"03:04.2": {},
|
|
"03:04.3": {},
|
|
"03:04.4": {},
|
|
}
|
|
dp := &DevicePlugin{}
|
|
dp.PostAllocate(response)
|
|
if len(response.ContainerResponses[0].Envs) != 4 {
|
|
t.Fatal("Set wrong number of Environment Variables")
|
|
}
|
|
|
|
for k := range response.ContainerResponses[0].Envs {
|
|
if !eleInSlice(k, resultKey) {
|
|
t.Fatalf("Set wrong key: %s. The key should be in the range %v", k, resultKey)
|
|
}
|
|
}
|
|
|
|
for _, key := range resultKey {
|
|
if value, ok := response.ContainerResponses[0].Envs[key]; ok {
|
|
if _, ok := expectedValues[value]; ok {
|
|
delete(expectedValues, value)
|
|
} else {
|
|
t.Errorf("Unexpected value %s", value)
|
|
}
|
|
}
|
|
}
|
|
}
|