Merge pull request #494 from bart0sh/PR0093-DSA-draft

Implement DSA plugin

.github/workflows/ci.yaml

@@ -75,6 +75,7 @@ jobs:
           - intel-deviceplugin-operator
           - intel-sgx-plugin
           - intel-sgx-initcontainer
+          - intel-dsa-plugin
 
           # Demo images
           - crypto-perf

README.md

@@ -19,6 +19,7 @@ Table of Contents
     * [QAT device plugin](#qat-device-plugin)
     * [VPU device plugin](#vpu-device-plugin)
     * [SGX device plugin](#sgx-device-plugin)
+    * [DSA device pugin](#dsa-device-plugin)
 * [Device Plugins Operator](#device-plugins-operator)
 * [Demos](#demos)
 * [Developers](#developers)
@@ -163,6 +164,11 @@ operator deployment and NFD is configured to register the SGX EPC memory extende
 Containers requesting SGX EPC resources in the cluster use `sgx.intel.com/epc` resource which is of
 type [memory](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory).
 
+
+### DSA device plugin
+
+The [DSA device plugin](cmd/dsa_plugin/README.md) supports acceleration using the Intel Data Streaming accelerator(DSA).
+
 ## Device Plugins Operator
 
 Currently the operator has limited support for the QAT, GPU, FPGA and SGX device plugins:

build/docker/intel-dsa-plugin.Dockerfile

@@ -0,0 +1,38 @@
+# CLEAR_LINUX_BASE and CLEAR_LINUX_VERSION can be used to make the build
+# reproducible by choosing an image by its hash and installing an OS version
+# with --version=:
+# CLEAR_LINUX_BASE=clearlinux@sha256:b8e5d3b2576eb6d868f8d52e401f678c873264d349e469637f98ee2adf7b33d4
+# CLEAR_LINUX_VERSION="--version=29970"
+#
+# This is used on release branches before tagging a stable version.
+# The master branch defaults to using the latest Clear Linux.
+ARG CLEAR_LINUX_BASE=clearlinux/golang:latest
+
+FROM ${CLEAR_LINUX_BASE} as builder
+
+ARG CLEAR_LINUX_VERSION=
+
+RUN swupd update --no-boot-update ${CLEAR_LINUX_VERSION}
+
+ARG DIR=/intel-device-plugins-for-kubernetes
+ARG GO111MODULE=on
+WORKDIR $DIR
+COPY . .
+
+RUN mkdir /install_root \
+    && swupd os-install \
+    ${CLEAR_LINUX_VERSION} \
+    --path /install_root \
+    --statedir /swupd-state \
+    --no-boot-update \
+    && rm -rf /install_root/var/lib/swupd/*
+
+RUN cd cmd/dsa_plugin; GO111MODULE=${GO111MODULE} go install; cd -
+RUN chmod a+x /go/bin/dsa_plugin \
+    && install -D /go/bin/dsa_plugin /install_root/usr/local/bin/intel_dsa_device_plugin \
+    && install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE \
+    && scripts/copy-modules-licenses.sh ./cmd/dsa_plugin /install_root/usr/local/share/
+
+FROM scratch as final
+COPY --from=builder /install_root /
+ENTRYPOINT ["/usr/local/bin/intel_dsa_device_plugin"]

cmd/dsa_plugin/README.md

@@ -0,0 +1,130 @@
+# Intel DSA device plugin for Kubernetes
+
+Table of Contents
+
+* [Introduction](#introduction)
+* [Installation](#installation)
+    * [Deploy with pre-built container image](#deploy-with-pre-built-container-image)
+    * [Getting the source code](#getting-the-source-code)
+    * [Verify node kubelet config](#verify-node-kubelet-config)
+    * [Deploying as a DaemonSet](#deploying-as-a-daemonset)
+        * [Build the plugin image](#build-the-plugin-image)
+        * [Deploy plugin DaemonSet](#deploy-plugin-daemonset)
+    * [Deploy by hand](#deploy-by-hand)
+        * [Build the plugin](#build-the-plugin)
+        * [Run the plugin as administrator](#run-the-plugin-as-administrator)
+    * [Verify plugin registration](#verify-plugin-registration)
+
+## Introduction
+
+The DSA device plugin for Kubernetes supports acceleration using the Intel Data Streaming accelerator(DSA).
+
+The DSA plugin discovers DSA work queues and presents them as a node resources.
+
+## Installation
+
+The following sections detail how to obtain, build, deploy and test the DSA device plugin.
+
+Examples are provided showing how to deploy the plugin either using a DaemonSet or by hand on a per-node basis.
+
+### Deploy with pre-built container image
+
+[Pre-built images](https://hub.docker.com/r/intel/intel-dsa-plugin)
+of this component are available on the Docker hub. These images are automatically built and uploaded
+to the hub from the latest master branch of this repository.
+
+Release tagged images of the components are also available on the Docker hub, tagged with their
+release version numbers in the format `x.y.z`, corresponding to the branches and releases in this
+repository. Thus the easiest way to deploy the plugin in your cluster is to run this command
+
+```bash
+$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/dsa_plugin?ref=<REF>
+daemonset.apps/intel-dsa-plugin created
+```
+
+Where `<REF>` needs to be substituted with the desired git ref, e.g. `master`.
+
+Nothing else is needed. But if you want to deploy a customized version of the plugin read further.
+
+### Getting the source code
+
+```bash
+$ export INTEL_DEVICE_PLUGINS_SRC=/path/to/intel-device-plugins-for-kubernetes
+$ git clone https://github.com/intel/intel-device-plugins-for-kubernetes ${INTEL_DEVICE_PLUGINS_SRC}
+```
+
+### Verify node kubelet config
+
+Every node that will be running the dsa plugin must have the
+[kubelet device-plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/)
+configured. For each node, check that the kubelet device plugin socket exists:
+
+```bash
+$ ls /var/lib/kubelet/device-plugins/kubelet.sock
+/var/lib/kubelet/device-plugins/kubelet.sock
+```
+
+### Deploying as a DaemonSet
+
+To deploy the dsa plugin as a daemonset, you first need to build a container image for the
+plugin and ensure that is visible to your nodes.
+
+#### Build the plugin image
+
+The following will use `docker` to build a local container image called
+`intel/intel-dsa-plugin` with the tag `devel`.
+
+The image build tool can be changed from the default `docker` by setting the `BUILDER` argument
+to the [`Makefile`](Makefile).
+
+```bash
+$ cd ${INTEL_DEVICE_PLUGINS_SRC}
+$ make intel-dsa-plugin
+...
+Successfully tagged intel/intel-dsa-plugin:devel
+```
+
+#### Deploy plugin DaemonSet
+
+You can then use the [example DaemonSet YAML](/deployments/dsa_plugin/base/intel-dsa-plugin.yaml)
+file provided to deploy the plugin. The default kustomization that deploys the YAML as is:
+
+```bash
+$ kubectl apply -k deployments/dsa_plugin
+daemonset.apps/intel-dsa-plugin created
+```
+
+### Deploy by hand
+
+For development purposes, it is sometimes convenient to deploy the plugin 'by hand' on a node.
+In this case, you do not need to build the complete container image, and can build just the plugin.
+
+#### Build the plugin
+
+First we build the plugin:
+
+```bash
+$ cd ${INTEL_DEVICE_PLUGINS_SRC}
+$ make dsa_plugin
+```
+
+#### Run the plugin as administrator
+
+Now we can run the plugin directly on the node:
+
+```bash
+$ sudo -E ${INTEL_DEVICE_PLUGINS_SRC}/cmd/dsa_plugin/dsa_plugin
+device-plugin registered
+```
+
+### Verify plugin registration
+
+You can verify the plugin has been registered with the expected nodes by searching for the relevant
+resource allocation status on the nodes:
+
+```bash
+$ kubectl get nodes -o=jsonpath="{range .items[*]}{.metadata.name}{'\n'}{' i915: '}{.status.allocatable.dsa\.intel\.com/*}{'\n'}"
+master
+  dsa.intel.com/wq-user-dedicated:  1
+  dsa.intel.com/wq-user-shared:     1
+```

cmd/dsa_plugin/dsa_plugin.go

@@ -0,0 +1,55 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+
+	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/idxd"
+
+	"k8s.io/klog"
+)
+
+const (
+	// Device plugin settings.
+	namespace = "dsa.intel.com"
+	// SysFS directory.
+	sysfsDir = "/sys/bus/dsa/devices"
+	// Device directories.
+	devDir = "/dev/dsa"
+	// Glob pattern for the state sysfs entry.
+	statePattern = "/sys/bus/dsa/devices/dsa*/wq*/state"
+)
+
+func main() {
+	var sharedDevNum int
+
+	flag.IntVar(&sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same work queue")
+	flag.Parse()
+
+	if sharedDevNum < 1 {
+		klog.Warning("The number of containers sharing the same work queue must be greater than zero")
+		os.Exit(1)
+	}
+
+	plugin := idxd.NewDevicePlugin(sysfsDir, statePattern, devDir, sharedDevNum)
+	if plugin == nil {
+		klog.Fatal("Cannot create device plugin, please check above error messages.")
+	}
+	manager := dpapi.NewManager(namespace, plugin)
+	manager.Run()
+}

deployments/dsa_plugin/base/intel-dsa-plugin.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-dsa-plugin
+  labels:
+    app: intel-dsa-plugin
+spec:
+  selector:
+    matchLabels:
+      app: intel-dsa-plugin
+  template:
+    metadata:
+      labels:
+        app: intel-dsa-plugin
+    spec:
+      containers:
+      - name: intel-dsa-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        image: intel/intel-dsa-plugin:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - name: devfs
+          mountPath: /dev/dsa
+          readOnly: true
+        - name: chardevs
+          mountPath: /dev/char
+          readOnly: true
+        - name: sysfs
+          mountPath: /sys/bus/dsa
+          readOnly: true
+        - name: kubeletsockets
+          mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+      - name: devfs
+        hostPath:
+          path: /dev/dsa
+      - name: chardevs
+        hostPath:
+          path: /dev/char
+      - name: sysfs
+        hostPath:
+          path: /sys/bus/dsa
+      - name: kubeletsockets
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      nodeSelector:
+        kubernetes.io/arch: amd64

deployments/dsa_plugin/base/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - intel-dsa-plugin.yaml

deployments/dsa_plugin/kustomization.yaml

@@ -0,0 +1,2 @@
+bases:
+  - base

deployments/dsa_plugin/overlays/namespace_kube-system/add-namespace-kube-system.yaml

@@ -0,0 +1,5 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-dsa-plugin
+  namespace: kube-system

deployments/dsa_plugin/overlays/namespace_kube-system/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+  - ../../base
+patches:
+  - add-namespace-kube-system.yaml

pkg/idxd/plugin.go

@@ -0,0 +1,202 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package idxd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+	"k8s.io/klog"
+	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+const (
+	// Character devices directory.
+	charDevDir = "/dev/char"
+	// Frequency of device scans.
+	scanFrequency = 5 * time.Second
+)
+
+// getDevNodesFunc type allows overriding filesystem APIs (os.Stat, stat.Sys, etc) in tests.
+type getDevNodesFunc func(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error)
+
+// DevicePlugin defines properties of the idxd device plugin.
+type DevicePlugin struct {
+	sysfsDir     string
+	statePattern string
+	devDir       string
+	charDevDir   string
+	sharedDevNum int
+	scanTicker   *time.Ticker
+	scanDone     chan bool
+	getDevNodes  getDevNodesFunc
+}
+
+// NewDevicePlugin creates DevicePlugin.
+func NewDevicePlugin(sysfsDir, statePattern, devDir string, sharedDevNum int) *DevicePlugin {
+	return &DevicePlugin{
+		sysfsDir:     sysfsDir,
+		statePattern: statePattern,
+		devDir:       devDir,
+		charDevDir:   charDevDir,
+		sharedDevNum: sharedDevNum,
+		scanTicker:   time.NewTicker(scanFrequency),
+		scanDone:     make(chan bool, 1),
+		getDevNodes:  getDevNodes,
+	}
+}
+
+// Scan discovers devices and reports them to the upper level API.
+func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
+	defer dp.scanTicker.Stop()
+	for {
+		devTree, err := dp.scan()
+		if err != nil {
+			return err
+		}
+
+		notifier.Notify(devTree)
+
+		select {
+		case <-dp.scanDone:
+			return nil
+		case <-dp.scanTicker.C:
+		}
+	}
+}
+
+func readFile(fpath string) (string, error) {
+	data, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
+// getDevNodes collects device nodes that belong to working queue.
+func getDevNodes(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error) {
+	// check if /dev/dsa/<work queue> device node exists
+	devPath := path.Join(devDir, wqName)
+	stat, err := os.Stat(devPath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	// Check if it's a character device
+	if stat.Mode()&os.ModeCharDevice == 0 {
+		return nil, errors.Errorf("%s is not a character device", devPath)
+	}
+
+	// get /dev/char/<major>:<minor> symlink for the device node
+	// as libaccel-config requires it
+	rdev := stat.Sys().(*syscall.Stat_t).Rdev
+	charDevPath := path.Join(charDevDir, fmt.Sprintf("%d:%d", unix.Major(rdev), unix.Minor(rdev)))
+	stat, err = os.Lstat(charDevPath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if stat.Mode()&os.ModeSymlink == 0 {
+		return nil, errors.Errorf("%s is not a symlink", charDevPath)
+	}
+	// Check if symlink points to the correct device node
+	destPath, err := filepath.EvalSymlinks(charDevPath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if destPath != devPath {
+		return nil, errors.Errorf("%s points to %s instead of device node %s", charDevPath, destPath, devPath)
+	}
+
+	// report device node and /dev/char/<major>:<minor> symlink
+	// as libaccel-config works with a symlink
+	return []pluginapi.DeviceSpec{
+		{
+			HostPath:      devPath,
+			ContainerPath: devPath,
+			Permissions:   "rw",
+		},
+		{
+			HostPath:      charDevPath,
+			ContainerPath: charDevPath,
+			Permissions:   "rw",
+		}}, nil
+}
+
+// scan collects devices by scanning sysfs and devfs entries.
+func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
+	// scan sysfs tree
+	matches, err := filepath.Glob(dp.statePattern)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	devTree := dpapi.NewDeviceTree()
+
+	for _, fpath := range matches {
+		// Read queue state entry
+		wqState, err := readFile(fpath)
+		if err != nil {
+			return nil, err
+		}
+
+		if wqState != "enabled" {
+			continue
+		}
+
+		// Read queue mode
+		queueDir := filepath.Dir(fpath)
+		wqMode, err := readFile(path.Join(queueDir, "mode"))
+		if err != nil {
+			return nil, err
+		}
+
+		// Read queue type
+		wqType, err := readFile(path.Join(queueDir, "type"))
+		if err != nil {
+			return nil, err
+		}
+
+		wqName := filepath.Base(queueDir)
+		devNodes := []pluginapi.DeviceSpec{}
+
+		if wqType == "user" {
+			devNodes, err = dp.getDevNodes(dp.devDir, dp.charDevDir, wqName)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		amount := dp.sharedDevNum
+		if wqMode != "shared" {
+			amount = 1
+		}
+		klog.V(4).Infof("%s: amount: %d, type: %s, mode: %s, nodes: %+v", wqName, amount, wqType, wqMode, devNodes)
+		for i := 0; i < amount; i++ {
+			deviceType := fmt.Sprintf("wq-%s-%s", wqType, wqMode)
+			deviceID := fmt.Sprintf("%s-%s-%d", deviceType, wqName, i)
+			devTree.AddDevice(deviceType, deviceID, dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, nil, nil))
+		}
+	}
+
+	return devTree, nil
+}

pkg/idxd/plugin_test.go

@@ -0,0 +1,256 @@
+// Copyright 2017 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package idxd
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+const (
+	dsaMajor int = 375
+)
+
+func getFakeDevNodes(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error) {
+	devPath := path.Join(devDir, wqName)
+	var devNum int
+	var queueNum int
+	fmt.Sscanf(wqName, "wq%d.%d", &devNum, &queueNum)
+	charDevPath := path.Join(charDevDir, fmt.Sprintf("%d:%d", dsaMajor, devNum*10+queueNum))
+
+	return []pluginapi.DeviceSpec{
+		{
+			HostPath:      devPath,
+			ContainerPath: devPath,
+			Permissions:   "rw",
+		},
+		{
+			HostPath:      charDevPath,
+			ContainerPath: charDevPath,
+			Permissions:   "rw",
+		}}, nil
+}
+
+func init() {
+	_ = flag.Set("v", "4") //Enable debug output
+}
+
+// fakeNotifier implements Notifier interface.
+type fakeNotifier struct {
+	scanDone   chan bool
+	deviceTree dpapi.DeviceTree
+}
+
+// Notify stops plugin Scan.
+func (n *fakeNotifier) Notify(deviceTree dpapi.DeviceTree) {
+	n.deviceTree = deviceTree
+	n.scanDone <- true
+}
+
+type testCase struct {
+	name           string
+	sysfsDirs      []string
+	sysfsFiles     map[string][]byte
+	sharedDevNum   int
+	expectedResult map[string]int
+	expectedError  bool
+}
+
+func TestScan(t *testing.T) {
+	root, err := ioutil.TempDir("", "test_idxd_device_plugin")
+	if err != nil {
+		t.Fatalf("can't create temporary directory: %+v", err)
+	}
+
+	defer os.RemoveAll(root)
+
+	sysfs := path.Join(root, "sys/bus/dsa/devices")
+	statePattern := path.Join(sysfs, "dsa*/wq*/state")
+
+	tcases := []testCase{
+		{
+			name: "no sysfs mounted",
+		},
+		{
+			name:      "all queues are disabled",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"},
+			sysfsFiles: map[string][]byte{
+				"dsa0/wq0.0/state": []byte(""),
+				"dsa0/wq0.1/state": []byte(""),
+				"dsa0/wq0.2/state": []byte(""),
+				"dsa0/wq0.3/state": []byte(""),
+			},
+		},
+		{
+			name:      "invalid: mode entry doesn't exist",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"},
+			sysfsFiles: map[string][]byte{
+				"dsa0/wq0.0/state": []byte("enabled"),
+				"dsa0/wq0.1/state": []byte("enabled"),
+				"dsa0/wq0.2/state": []byte(""),
+				"dsa0/wq0.3/state": []byte(""),
+			},
+			expectedError: true,
+		},
+		{
+			name:      "invalid: type entry doesn't exist",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"},
+			sysfsFiles: map[string][]byte{
+				"dsa0/wq0.0/state": []byte("enabled"),
+				"dsa0/wq0.1/state": []byte("enabled"),
+				"dsa0/wq0.2/state": []byte(""),
+				"dsa0/wq0.3/state": []byte(""),
+				"dsa0/wq0.0/mode":  []byte("dedicated"),
+				"dsa0/wq0.1/mode":  []byte("dedicated"),
+			},
+			expectedError: true,
+		},
+		{
+			name:      "valid: two dedicated user queues",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"},
+			sysfsFiles: map[string][]byte{
+				"dsa0/wq0.0/state": []byte("enabled"),
+				"dsa0/wq0.1/state": []byte("enabled"),
+				"dsa0/wq0.2/state": []byte(""),
+				"dsa0/wq0.3/state": []byte(""),
+				"dsa0/wq0.0/mode":  []byte("dedicated"),
+				"dsa0/wq0.1/mode":  []byte("dedicated"),
+				"dsa0/wq0.0/type":  []byte("user"),
+				"dsa0/wq0.1/type":  []byte("user"),
+			},
+			expectedResult: map[string]int{
+				"wq-user-dedicated": 2,
+			},
+		},
+		{
+			name:      "valid: two shared user queues x 10",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"},
+			sysfsFiles: map[string][]byte{
+				"dsa0/wq0.0/state": []byte("enabled"),
+				"dsa0/wq0.1/state": []byte("enabled"),
+				"dsa0/wq0.2/state": []byte(""),
+				"dsa0/wq0.3/state": []byte(""),
+				"dsa0/wq0.0/mode":  []byte("shared"),
+				"dsa0/wq0.1/mode":  []byte("shared"),
+				"dsa0/wq0.0/type":  []byte("user"),
+				"dsa0/wq0.1/type":  []byte("user"),
+			},
+			sharedDevNum: 10,
+			expectedResult: map[string]int{
+				"wq-user-shared": 20,
+			},
+		},
+		{
+			name:      "valid: all types of queues",
+			sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa1/wq1.0", "dsa1/wq1.1", "dsa1/wq1.2", "dsa1/wq1.3"},
+			sysfsFiles: map[string][]byte{
+				//device 0
+				"dsa0/wq0.0/state": []byte("enabled"),
+				"dsa0/wq0.1/state": []byte("enabled"),
+				"dsa0/wq0.0/mode":  []byte("shared"),
+				"dsa0/wq0.1/mode":  []byte("dedicated"),
+				"dsa0/wq0.0/type":  []byte("user"),
+				"dsa0/wq0.1/type":  []byte("kernel"),
+				//device 1
+				"dsa1/wq1.0/state": []byte("enabled"),
+				"dsa1/wq1.1/state": []byte("enabled"),
+				"dsa1/wq1.2/state": []byte("enabled"),
+				"dsa1/wq1.3/state": []byte("enabled"),
+				"dsa1/wq1.0/mode":  []byte("shared"),
+				"dsa1/wq1.1/mode":  []byte("dedicated"),
+				"dsa1/wq1.2/mode":  []byte("shared"),
+				"dsa1/wq1.3/mode":  []byte("dedicated"),
+				"dsa1/wq1.0/type":  []byte("mdev"),
+				"dsa1/wq1.1/type":  []byte("mdev"),
+				"dsa1/wq1.2/type":  []byte("mdev"),
+				"dsa1/wq1.3/type":  []byte("user"),
+			},
+			sharedDevNum: 10,
+			expectedResult: map[string]int{
+				"wq-user-shared":      10,
+				"wq-kernel-dedicated": 1,
+				"wq-mdev-shared":      20,
+				"wq-user-dedicated":   1,
+				"wq-mdev-dedicated":   1,
+			},
+		},
+	}
+
+	for _, tc := range tcases {
+		t.Run(tc.name, genTest(sysfs, statePattern, tc))
+	}
+}
+
+// generate test to decrease cyclomatic complexity.
+func genTest(sysfs, statePattern string, tc testCase) func(t *testing.T) {
+	return func(t *testing.T) {
+		for _, sysfsdir := range tc.sysfsDirs {
+			if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil {
+				t.Fatalf("Failed to create fake sysfs directory: %+v", err)
+			}
+		}
+		for filename, body := range tc.sysfsFiles {
+			if err := ioutil.WriteFile(path.Join(sysfs, filename), body, 0600); err != nil {
+				t.Fatalf("Failed to create fake sysfs entry: %+v", err)
+			}
+		}
+
+		plugin := NewDevicePlugin(sysfs, statePattern, "", tc.sharedDevNum)
+		plugin.getDevNodes = getFakeDevNodes
+
+		notifier := &fakeNotifier{
+			scanDone: plugin.scanDone,
+		}
+
+		err := plugin.Scan(notifier)
+		if !tc.expectedError && err != nil {
+			t.Errorf("unexpected error: %+v", err)
+		}
+		if tc.expectedError && err == nil {
+			t.Errorf("unexpected success")
+		}
+		if err := checkDeviceTree(notifier.deviceTree, tc.expectedResult, tc.expectedError); err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+// checkDeviceTree checks discovered device types and number of discovered devices.
+func checkDeviceTree(deviceTree dpapi.DeviceTree, expectedResult map[string]int, expectedError bool) error {
+	if !expectedError && deviceTree != nil {
+		for key := range deviceTree {
+			val, ok := expectedResult[key]
+			if !ok {
+				return fmt.Errorf("unexpected device type: %s", key)
+			}
+			numberDev := len(deviceTree[key])
+			if numberDev != val {
+				return fmt.Errorf("%s: unexpected number of devices: %d, expected: %d", key, numberDev, val)
+			}
+			delete(expectedResult, key)
+		}
+		if len(expectedResult) > 0 {
+			return fmt.Errorf("missing expected result(s): %+v", expectedResult)
+		}
+	}
+	return nil
+}