dlb: Add new device plugin

Signed-off-by: Hyeongju Johannes Lee <hyeongju.lee@intel.com>

.github/workflows/ci.yaml

@@ -75,6 +75,7 @@ jobs:
           - intel-sgx-initcontainer
           - intel-dsa-plugin
           - intel-idxd-initcontainer
+          - intel-dlb-plugin
 
           # Demo images
           - crypto-perf

.gitignore

@@ -8,6 +8,7 @@ coverage.txt
 cmd/dsa_plugin/dsa_plugin
 cmd/fpga_admissionwebhook/fpga_admissionwebhook
 cmd/fpga_crihook/fpga_crihook
+cmd/dlb_plugin/dlb_plugin
 cmd/fpga_plugin/fpga_plugin
 cmd/fpga_tool/fpga_tool
 cmd/gpu_nfdhook/gpu_nfdhook

build/docker/intel-dlb-plugin.Dockerfile

@@ -0,0 +1,39 @@
+# Copyright 2021 Intel Corporation. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# GOLANG_BASE can be used to make the build reproducible by choosing an
+# image by its hash:
+# GOLANG_BASE=golang@sha256:9d64369fd3c633df71d7465d67d43f63bb31192193e671742fa1c26ebc3a6210
+#
+# This is used on release branches before tagging a stable version.
+# The main branch defaults to using the latest Golang base image.
+ARG GOLANG_BASE=golang:1.17-bullseye
+
+FROM ${GOLANG_BASE} as builder
+
+ARG DIR=/intel-device-plugins-for-kubernetes
+ARG GO111MODULE=on
+ARG BUILDFLAGS="-ldflags=-w -s"
+WORKDIR $DIR
+COPY . .
+
+RUN cd cmd/dlb_plugin; GO111MODULE=${GO111MODULE} CGO_ENABLED=0 go install "${BUILDFLAGS}"; cd -
+RUN install -D /go/bin/dlb_plugin /install_root/usr/local/bin/intel_dlb_device_plugin \
+    && install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE \
+    && scripts/copy-modules-licenses.sh ./cmd/dlb_plugin /install_root/usr/local/share/
+
+FROM gcr.io/distroless/static
+COPY --from=builder /install_root /
+ENTRYPOINT ["/usr/local/bin/intel_dlb_device_plugin"]
+

cmd/dlb_plugin/dlb_plugin.go

@@ -0,0 +1,112 @@
+// Copyright 2021 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"path/filepath"
+	"reflect"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
+	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+const (
+	dlbDeviceFilePathRE = "/dev/dlb*"
+	namespace           = "dlb.intel.com"
+	deviceTypePF        = "pf"
+	deviceTypeVF        = "vf"
+	sysfsDir            = "/sys/class/dlb2"
+	// Period of device scans.
+	scanPeriod = 5 * time.Second
+)
+
+type DevicePlugin struct {
+	scanTicker *time.Ticker
+	scanDone   chan bool
+
+	dlbDeviceFilePathReg string
+	sysfsDir             string
+}
+
+func NewDevicePlugin(dlbDeviceFilePathReg string, sysfsDir string) *DevicePlugin {
+	return &DevicePlugin{
+		dlbDeviceFilePathReg: dlbDeviceFilePathReg,
+		sysfsDir:             sysfsDir,
+		scanTicker:           time.NewTicker(scanPeriod),
+		scanDone:             make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
+	}
+}
+
+func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
+	defer dp.scanTicker.Stop()
+
+	var prevDevTree dpapi.DeviceTree
+	for {
+		devTree := dp.scan()
+
+		if !reflect.DeepEqual(prevDevTree, devTree) {
+			klog.V(1).Info("DLB scan update: pf: ", len(devTree[deviceTypePF]), " / vf: ", len(devTree[deviceTypeVF]))
+			prevDevTree = devTree
+		}
+		notifier.Notify(devTree)
+
+		select {
+		case <-dp.scanDone:
+			return nil
+		case <-dp.scanTicker.C:
+		}
+	}
+}
+
+func (dp *DevicePlugin) scan() dpapi.DeviceTree {
+	files, _ := filepath.Glob(dp.dlbDeviceFilePathReg)
+
+	devTree := dpapi.NewDeviceTree()
+
+	for _, file := range files {
+		devs := []pluginapi.DeviceSpec{{
+			HostPath:      file,
+			ContainerPath: file,
+			Permissions:   "rw",
+		}}
+		deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, nil)
+
+		sysfsDev := filepath.Join(dp.sysfsDir, filepath.Base(file))
+		sriovNumVFs := pluginutils.GetSriovNumVFs(sysfsDev)
+
+		switch sriovNumVFs {
+		case "0":
+			devTree.AddDevice(deviceTypePF, file, deviceInfo)
+		case "-1":
+			devTree.AddDevice(deviceTypeVF, file, deviceInfo)
+		default:
+			continue
+		}
+	}
+
+	return devTree
+}
+
+func main() {
+	klog.V(1).Infof("DLB device plugin started")
+
+	plugin := NewDevicePlugin(dlbDeviceFilePathRE, sysfsDir)
+	manager := dpapi.NewManager(namespace, plugin)
+	manager.Run()
+}

cmd/dlb_plugin/dlb_plugin_test.go

@@ -0,0 +1,148 @@
+// Copyright 2021 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"path"
+	"testing"
+
+	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	_ = flag.Set("v", "4") //Enable debug output
+}
+
+// mockNotifier implements Notifier interface.
+type mockNotifier struct {
+	scanDone chan bool
+	devCount int
+}
+
+// Notify stops plugin Scan.
+func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
+	n.devCount = len(newDeviceTree[deviceTypePF]) + len(newDeviceTree[deviceTypeVF])
+	n.scanDone <- true
+}
+
+func createTestFiles(devfs string, devfsdirs []string, sysfs string, pfDevs []string, sriovnumvfs []string) error {
+	for _, devfsdir := range devfsdirs {
+		if err := os.MkdirAll(path.Join(devfs, devfsdir), 0750); err != nil {
+			return errors.Wrap(err, "Failed to create fake device directory")
+		}
+
+		if err := os.MkdirAll(path.Join(sysfs, devfsdir), 0750); err != nil {
+			return errors.Wrap(err, "Failed to create fake device directory")
+		}
+	}
+
+	for index, pfDev := range pfDevs {
+		if err := os.MkdirAll(path.Join(sysfs, pfDev, "device"), 0750); err != nil {
+			return errors.Wrap(err, "Failed to create fake device directory")
+		}
+		if err := os.WriteFile(path.Join(sysfs, pfDev, "device", "sriov_numvfs"), []byte(sriovnumvfs[index]), 0600); err != nil {
+			return errors.Wrap(err, "Failed to create fake device directory")
+		}
+	}
+
+	return nil
+}
+
+func TestNewDevicePlugin(t *testing.T) {
+	if NewDevicePlugin("", "") == nil {
+		t.Error("Failed to create plugin")
+	}
+}
+
+func TestScan(t *testing.T) {
+	tcases := []struct {
+		name string
+		// test-case environment
+		devfsdirs   []string
+		pfDevs      []string
+		sriovnumvfs []string
+		// what the result should be
+		expectedDevs int
+	}{
+		{
+			name:         "no device",
+			expectedDevs: 0,
+		},
+		{
+			name:         "pf devices",
+			devfsdirs:    []string{"dlb1", "dlb2", "dlb4"},
+			pfDevs:       []string{"dlb1", "dlb2", "dlb4"},
+			sriovnumvfs:  []string{"0", "0", "0"},
+			expectedDevs: 3,
+		},
+		{
+			name:         "vf devices",
+			devfsdirs:    []string{"dlb1", "dlb2", "dlb4"},
+			expectedDevs: 3,
+		},
+		{
+			name:         "pf & vf devices (derived from another pf)",
+			devfsdirs:    []string{"dlb1", "dlb2", "dlb3"},
+			pfDevs:       []string{"dlb1"},
+			sriovnumvfs:  []string{"0"},
+			expectedDevs: 3,
+		},
+		{
+			name:         "pf & vf devices (derived from the pf)",
+			devfsdirs:    []string{"dlb0", "dlb2", "dlb3"},
+			pfDevs:       []string{"dlb0"},
+			sriovnumvfs:  []string{"1"},
+			expectedDevs: 2,
+		},
+	}
+
+	for _, tc := range tcases {
+		t.Run(tc.name, func(t *testing.T) {
+			root, err := os.MkdirTemp("", "test_new_device_plugin")
+			if err != nil {
+				t.Fatalf("can't create temporary directory: %+v", err)
+			}
+			// dirs/files need to be removed for the next test
+			defer os.RemoveAll(root)
+
+			devfs := path.Join(root, "dev")
+			sysfs := path.Join(root, sysfsDir)
+			err = createTestFiles(devfs, tc.devfsdirs, sysfs, tc.pfDevs, tc.sriovnumvfs)
+			if err != nil {
+				t.Errorf("unexpected error: %+v", err)
+			}
+
+			devfs = path.Join(devfs, "dlb*")
+			plugin := NewDevicePlugin(devfs, sysfs)
+
+			notifier := &mockNotifier{
+				scanDone: plugin.scanDone,
+			}
+
+			err = plugin.Scan(notifier)
+			// Scans in DLB plugin never fail
+			if err != nil {
+				t.Errorf("unexpected error: %+v", err)
+			}
+			if tc.expectedDevs != notifier.devCount {
+				t.Errorf("Expected %d, discovered %d devices",
+					tc.expectedDevs, notifier.devCount)
+			}
+		})
+	}
+}

cmd/internal/pluginutils/sriov.go

@@ -22,11 +22,14 @@ import (
 
 // IsSriovPFwithVFs returns true if the device with given pfpath has virtual functions, false otherwise.
 func IsSriovPFwithVFs(pfpath string) bool {
-	dat, err := os.ReadFile(path.Join(pfpath, "device/sriov_numvfs"))
-
-	if err == nil && strings.TrimSpace(string(dat)) != "0" {
-		return true
-	}
-
-	return false
+	dat := GetSriovNumVFs(pfpath)
+	return dat != "-1" && dat != "0"
+}
+
+func GetSriovNumVFs(sysFSPath string) string {
+	dat, err := os.ReadFile(path.Join(sysFSPath, "device/sriov_numvfs"))
+	if err != nil {
+		return "-1"
+	}
+	return strings.TrimSpace(string(dat))
 }

demo/dlb-libdlb-demo-pod.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dlb-libdlb-demo
+spec:
+  restartPolicy: Never
+  containers:
+  - name: pf
+    image: intel/dlb-libdlb-demo:devel
+    imagePullPolicy: IfNotPresent
+    resources:
+      limits:
+        dlb.intel.com/pf: 1
+        cpu: 1
+  - name: vf
+    image: intel/dlb-libdlb-demo:devel
+    imagePullPolicy: IfNotPresent
+    resources:
+      limits:
+        dlb.intel.com/vf: 1
+        cpu: 1

deployments/dlb_plugin/base/intel-dlb-plugin.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-dlb-plugin
+  labels:
+    app: intel-dlb-plugin
+spec:
+  selector:
+    matchLabels:
+      app: intel-dlb-plugin
+  template:
+    metadata:
+      labels:
+        app: intel-dlb-plugin
+    spec:
+      containers:
+      - name: intel-dlb-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        image: intel/intel-dlb-plugin:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - name: devfs
+          mountPath: /dev
+          readOnly: true
+        - name: sysfs
+          mountPath: /sys/class/dlb2
+          readOnly: true
+        - name: kubeletsockets
+          mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+      - name: devfs
+        hostPath:
+          path: /dev
+      - name: sysfs
+        hostPath:
+          path: /sys/class/dlb2
+      - name: kubeletsockets
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      nodeSelector:
+        kubernetes.io/arch: amd64

deployments/dlb_plugin/base/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - intel-dlb-plugin.yaml

deployments/dlb_plugin/kustomization.yaml

@@ -0,0 +1,2 @@
+bases:
+  - base