gpu-plugin nfd-hook

This adds an nfd-hook for the gpu-plugin, which will create labels
for the GPUs that can then be used for POD deployment purposes or
creation of GPU extended resources which allow then finer grained
GPU resource management.

The nfd-hook will install to the host system when the
intel-gpu-initcontainer is run. It is added into the plugin deployment
yaml.

Signed-off-by: Ukri Niemimuukko <ukri.niemimuukko@intel.com>

.github/workflows/ci.yaml

@@ -67,6 +67,7 @@ jobs:
         image:
           - intel-fpga-admissionwebhook
           - intel-fpga-initcontainer
+          - intel-gpu-initcontainer
           - intel-gpu-plugin
           - intel-fpga-plugin
           - intel-qat-plugin

build/docker/intel-gpu-initcontainer.Dockerfile

@@ -0,0 +1,53 @@
+# CLEAR_LINUX_BASE and CLEAR_LINUX_VERSION can be used to make the build
+# reproducible by choosing an image by its hash and installing an OS version
+# with --version=:
+# CLEAR_LINUX_BASE=clearlinux@sha256:b8e5d3b2576eb6d868f8d52e401f678c873264d349e469637f98ee2adf7b33d4
+# CLEAR_LINUX_VERSION="--version=29970"
+#
+# This is used on release branches before tagging a stable version.
+# The master branch defaults to using the latest Clear Linux.
+ARG CLEAR_LINUX_BASE=clearlinux/golang:latest
+
+FROM ${CLEAR_LINUX_BASE} as builder
+
+ARG CLEAR_LINUX_VERSION=
+
+RUN swupd update --no-boot-update ${CLEAR_LINUX_VERSION}
+
+ARG DIR=/intel-device-plugins-for-kubernetes
+ARG GO111MODULE=on
+WORKDIR $DIR
+COPY . .
+
+RUN mkdir /install_root \
+    && swupd os-install \
+    ${CLEAR_LINUX_VERSION} \
+    --path /install_root \
+    --statedir /swupd-state \
+    --bundles=rsync \
+    --no-boot-update \
+    && rm -rf /install_root/var/lib/swupd/*
+
+# Build NFD Feature Detector Hook
+RUN cd $DIR/cmd/gpu_nfdhook && \
+    GO111MODULE=${GO111MODULE} go install -ldflags="-s -w" && \
+    chmod a+x /go/bin/gpu_nfdhook && \
+    cd $DIR && \
+    install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE && \
+    scripts/copy-modules-licenses.sh ./cmd/gpu_nfdhook /install_root/usr/local/share/package-licenses/
+
+FROM scratch as final
+COPY --from=builder /install_root /
+
+ARG NFD_HOOK=intel-gpu-nfdhook
+ARG SRC_DIR=/usr/local/bin/gpu-sw
+ARG DST_DIR=/etc/kubernetes/node-feature-discovery/source.d/
+
+COPY --from=builder /go/bin/gpu_nfdhook $SRC_DIR/$NFD_HOOK
+
+RUN echo -e "#!/bin/sh\n\
+rsync -a $SRC_DIR/ $DST_DIR\n\
+rm $DST_DIR/deploy.sh\
+">> $SRC_DIR/deploy.sh && chmod +x $SRC_DIR/deploy.sh
+
+ENTRYPOINT [ "/usr/local/bin/gpu-sw/deploy.sh" ]

cmd/gpu_nfdhook/README.md

@@ -0,0 +1,20 @@
+# Intel GPU NFD hook
+
+This is the Node Feature Discovery binary hook implementation for the Intel
+GPUs. The intel-gpu-initcontainer which is built among other images can be
+placed as part of the gpu-plugin deployment, so that it copies this hook to the
+host system only in those hosts, in which also gpu-plugin is deployed.
+
+When NFD worker runs this hook, it will add a number of labels to the nodes,
+which can be used for example to deploy services to nodes with specific GPU
+types. Selected numeric labels can be turned into kubernetes extended resources
+by the NFD, allowing for finer grained resource management for GPU-using PODs.
+
+In the NFD deployment, the hook requires /host-sys -folder to have the host /sys
+-folder content mounted, and /host-dev to have the host /dev/ -folder content
+mounted. Write access is not necessary.
+
+There is one supported environment variable named GPU_MEMORY_OVERRIDE, which is
+supposed to hold a numeric value. For systems with GPUs which do not support
+reading the GPU memory amount, the environment variable memory value is turned
+into a GPU memory amount label instead of a read value.

cmd/gpu_nfdhook/labeler.go

@@ -0,0 +1,211 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog"
+)
+
+const (
+	labelNamespace     = "gpu.intel.com/"
+	gpuListLabelName   = "cards"
+	millicoreLabelName = "millicores"
+	millicoresPerGPU   = 1000
+	memoryOverrideEnv  = "GPU_MEMORY_OVERRIDE"
+	gpuDeviceRE        = `^card[0-9]+$`
+	controlDeviceRE    = `^controlD[0-9]+$`
+	vendorString       = "0x8086"
+)
+
+type labelMap map[string]string
+
+type labeler struct {
+	sysfsDir      string
+	devfsDir      string
+	debugfsDRIDir string
+
+	gpuDeviceReg     *regexp.Regexp
+	controlDeviceReg *regexp.Regexp
+	labels           labelMap
+}
+
+func newLabeler(sysfsDir, devfsDir, debugfsDRIDir string) *labeler {
+	return &labeler{
+		sysfsDir:         sysfsDir,
+		devfsDir:         devfsDir,
+		debugfsDRIDir:    debugfsDRIDir,
+		gpuDeviceReg:     regexp.MustCompile(gpuDeviceRE),
+		controlDeviceReg: regexp.MustCompile(controlDeviceRE),
+		labels:           labelMap{},
+	}
+}
+
+func (l *labeler) scan() ([]string, error) {
+	files, err := ioutil.ReadDir(l.sysfsDir)
+	gpuNameList := []string{}
+
+	if err != nil {
+		return gpuNameList, errors.Wrap(err, "Can't read sysfs folder")
+	}
+
+	for _, f := range files {
+		if !l.gpuDeviceReg.MatchString(f.Name()) {
+			klog.V(4).Info("Not compatible device", f.Name())
+			continue
+		}
+
+		dat, err := ioutil.ReadFile(path.Join(l.sysfsDir, f.Name(), "device/vendor"))
+		if err != nil {
+			klog.Warning("Skipping. Can't read vendor file: ", err)
+			continue
+		}
+
+		if strings.TrimSpace(string(dat)) != vendorString {
+			klog.V(4).Info("Non-Intel GPU", f.Name())
+			continue
+		}
+
+		drmFiles, err := ioutil.ReadDir(path.Join(l.sysfsDir, f.Name(), "device/drm"))
+		if err != nil {
+			return gpuNameList, errors.Wrap(err, "Can't read device folder")
+		}
+
+		for _, drmFile := range drmFiles {
+			if l.controlDeviceReg.MatchString(drmFile.Name()) {
+				//Skipping possible drm control node
+				continue
+			}
+			devPath := path.Join(l.devfsDir, drmFile.Name())
+			if _, err := os.Stat(devPath); err != nil {
+				continue
+			}
+
+			gpuNameList = append(gpuNameList, f.Name())
+			break
+		}
+	}
+
+	return gpuNameList, nil
+}
+
+// getMemoryValues reads the GPU memory amount from the system.
+func (l *labeler) getMemoryAmount( /*cardNum*/ string) uint64 {
+	// reading GPU local memory amount is not yet available in the driver,
+	// so just return the environment variable value
+	envValue := os.Getenv(memoryOverrideEnv)
+	if envValue != "" {
+		val, err := strconv.ParseUint(envValue, 10, 64)
+		if err == nil {
+			return val
+		}
+	}
+	return 0
+}
+
+// addNumericLabel creates a new label if one doesn't exist. Else the new value is added to the previous value.
+func (lm labelMap) addNumericLabel(labelName string, valueToAdd int64) {
+	value := int64(0)
+	if numstr, ok := lm[labelName]; ok {
+		_, _ = fmt.Sscanf(numstr, "%d", &value)
+	}
+	value += valueToAdd
+	lm[labelName] = strconv.FormatInt(value, 10)
+}
+
+// createCapabilityLabels creates labels from the gpu capability file under debugfs.
+func (l *labeler) createCapabilityLabels(cardNum string) {
+	// try to read the capabilities from the i915_capabilities file
+	file, err := os.Open(filepath.Join(l.debugfsDRIDir, cardNum, "i915_capabilities"))
+	if err != nil {
+		klog.V(3).Infof("Couldn't open file:%s", err.Error()) // debugfs is not stable, there is no need to spam with error level prints
+		return
+	}
+	defer file.Close()
+
+	// define strings to search from the file, and the actions to take in order to create labels from those strings (as funcs)
+	searchStringActionMap := map[string]func(string){
+		"platform: ": func(platformName string) {
+			l.labels.addNumericLabel(labelNamespace+"platform_"+platformName+".count", 1)
+			l.labels[labelNamespace+"platform_"+platformName+".present"] = "true"
+		},
+		"gen: ": func(genName string) {
+			l.labels[labelNamespace+"platform_gen"] = genName
+		},
+	}
+
+	// Finally, read the file, and try to find the matches. Perform actions and reduce the search map size as we proceed. Return at 0 size.
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		for searchString, action := range searchStringActionMap {
+			var stringValue string
+			n, _ := fmt.Sscanf(scanner.Text(), searchString+"%s", &stringValue)
+			if n > 0 {
+				action(stringValue)
+				delete(searchStringActionMap, searchString)
+				if len(searchStringActionMap) == 0 {
+					return
+				}
+				break
+			}
+		}
+	}
+}
+
+// createLabels is the main function of plugin labeler, it creates label-value pairs for the gpus.
+func (l *labeler) createLabels() error {
+	gpuNameList, err := l.scan()
+	if err != nil {
+		return err
+	}
+
+	for _, gpuName := range gpuNameList {
+		gpuNum := ""
+		// extract card number as a string. scan() has already checked name syntax
+		_, err = fmt.Sscanf(gpuName, "card%s", &gpuNum)
+		if err != nil {
+			return errors.Wrap(err, "gpu name parsing error")
+		}
+
+		// try to add capability labels
+		l.createCapabilityLabels(gpuNum)
+
+		// read the memory amount to find a proper max allocation value
+		l.labels.addNumericLabel(labelNamespace+"memory.max", int64(l.getMemoryAmount(gpuNum)))
+	}
+	gpuCount := len(gpuNameList)
+	// add gpu list label (example: "card0.card1.card2")
+	l.labels[labelNamespace+gpuListLabelName] = strings.Join(gpuNameList, ".")
+	// all GPUs get default number of millicores (1000)
+	l.labels.addNumericLabel(labelNamespace+millicoreLabelName, int64(millicoresPerGPU*gpuCount))
+
+	return nil
+}
+
+func (l *labeler) printLabels() {
+	for key, val := range l.labels {
+		fmt.Println(key + "=" + val)
+	}
+}

cmd/gpu_nfdhook/labeler_test.go

@@ -0,0 +1,177 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strconv"
+	"testing"
+)
+
+type testcase struct {
+	sysfsdirs      []string
+	sysfsfiles     map[string][]byte
+	devfsdirs      []string
+	name           string
+	memoryOverride uint64
+	capabilityFile map[string][]byte
+	expectedRetval error
+	expectedLabels labelMap
+}
+
+//nolint:funlen
+func getTestCases() []testcase {
+	return []testcase{
+		{
+			sysfsdirs: []string{
+				"card0/device/drm/card0",
+			},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+			},
+			devfsdirs:      []string{"card0"},
+			name:           "successful labeling",
+			memoryOverride: 16000000000,
+			capabilityFile: map[string][]byte{
+				"0/i915_capabilities": []byte(
+					"platform: new\n" +
+						"gen: 9"),
+			},
+			expectedRetval: nil,
+			expectedLabels: labelMap{
+				"gpu.intel.com/millicores":           "1000",
+				"gpu.intel.com/memory.max":           "16000000000",
+				"gpu.intel.com/platform_new.count":   "1",
+				"gpu.intel.com/platform_new.present": "true",
+				"gpu.intel.com/platform_gen":         "9",
+				"gpu.intel.com/cards":                "card0",
+			},
+		},
+		{
+			sysfsdirs: []string{
+				"card0/device/drm/card0",
+			},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+			},
+			devfsdirs:      []string{"card0"},
+			name:           "when gen:capability info is missing",
+			memoryOverride: 16000000000,
+			capabilityFile: map[string][]byte{
+				"0/i915_capabilities": []byte(
+					"platform: new"),
+			},
+			expectedRetval: nil,
+			expectedLabels: labelMap{
+				"gpu.intel.com/millicores":           "1000",
+				"gpu.intel.com/memory.max":           "16000000000",
+				"gpu.intel.com/platform_new.count":   "1",
+				"gpu.intel.com/platform_new.present": "true",
+				"gpu.intel.com/cards":                "card0",
+			},
+		},
+		{
+			sysfsdirs: []string{
+				"card0/device/drm/card0",
+				"card1/device/drm/card1",
+			},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+				"card1/device/vendor": []byte("0x8086"),
+			},
+			devfsdirs:      []string{"card0", "card1"},
+			name:           "when capability file is missing (foobar), related labels don't appear",
+			memoryOverride: 16000000000,
+			capabilityFile: map[string][]byte{
+				"foobar": []byte(
+					"platform: new\n" +
+						"gen: 9"),
+			},
+			expectedRetval: nil,
+			expectedLabels: labelMap{
+				"gpu.intel.com/millicores": "2000",
+				"gpu.intel.com/memory.max": "32000000000",
+				"gpu.intel.com/cards":      "card0.card1",
+			},
+		},
+	}
+}
+
+func (tc *testcase) createFiles(t *testing.T, sysfs, devfs, root string) {
+	var err error
+	for filename, body := range tc.capabilityFile {
+		if err = ioutil.WriteFile(path.Join(root, filename), body, 0600); err != nil {
+			t.Fatalf("Failed to create fake capability file: %+v", err)
+		}
+	}
+	for _, devfsdir := range tc.devfsdirs {
+		if err := os.MkdirAll(path.Join(devfs, devfsdir), 0750); err != nil {
+			t.Fatalf("Failed to create fake device directory: %+v", err)
+		}
+	}
+	for _, sysfsdir := range tc.sysfsdirs {
+		if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil {
+			t.Fatalf("Failed to create fake sysfs directory: %+v", err)
+		}
+	}
+	for filename, body := range tc.sysfsfiles {
+		if err := ioutil.WriteFile(path.Join(sysfs, filename), body, 0600); err != nil {
+			t.Fatalf("Failed to create fake vendor file: %+v", err)
+		}
+	}
+}
+
+func TestLabeling(t *testing.T) {
+	root, err := ioutil.TempDir("", "test_new_device_plugin")
+	if err != nil {
+		t.Fatalf("can't create temporary directory: %+v", err)
+	}
+
+	defer os.RemoveAll(root)
+
+	testcases := getTestCases()
+
+	for _, tc := range testcases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := os.MkdirAll(path.Join(root, "0"), 0750)
+			if err != nil {
+				t.Fatalf("couldn't create dir: %s", err.Error())
+			}
+			sysfs := path.Join(root, sysfsDirectory)
+			devfs := path.Join(root, devfsDirectory)
+
+			tc.createFiles(t, sysfs, devfs, root)
+
+			os.Setenv(memoryOverrideEnv, strconv.FormatUint(tc.memoryOverride, 10))
+
+			labeler := newLabeler(sysfs, devfs, root)
+			err = labeler.createLabels()
+			if err != nil && tc.expectedRetval == nil ||
+				err == nil && tc.expectedRetval != nil {
+				t.Errorf("unexpected return value")
+			}
+			if tc.expectedRetval == nil && !reflect.DeepEqual(labeler.labels, tc.expectedLabels) {
+				t.Errorf("label mismatch with expectation:\n%v\n%v\n", labeler.labels, tc.expectedLabels)
+			}
+			for filename := range tc.capabilityFile {
+				os.Remove(path.Join(root, filename))
+			}
+		})
+	}
+}

cmd/gpu_nfdhook/main.go

@@ -0,0 +1,40 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os"
+
+	"k8s.io/klog"
+)
+
+const (
+	sysfsDirectory      = "/host-sys"
+	devfsDirectory      = "/host-dev"
+	sysfsDrmDirectory   = sysfsDirectory + "/class/drm"
+	devfsDriDirectory   = devfsDirectory + "/dri"
+	debugfsDRIDirectory = sysfsDirectory + "/kernel/debug/dri"
+)
+
+func main() {
+	l := newLabeler(sysfsDrmDirectory, devfsDriDirectory, debugfsDRIDirectory)
+	err := l.createLabels()
+	if err != nil {
+		klog.Errorf("%+v", err)
+		os.Exit(1)
+	}
+
+	l.printLabels()
+}

deployments/gpu_plugin/base/intel-gpu-plugin.yaml

@@ -13,6 +13,14 @@ spec:
       labels:
         app: intel-gpu-plugin
     spec:
+      initContainers:
+      - name: intel-gpu-initcontainer
+        image: intel/intel-gpu-initcontainer:devel
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/kubernetes/node-feature-discovery/source.d/
+          name: nfd-source-hooks
       containers:
       - name: intel-gpu-plugin
         env:
@@ -43,5 +51,9 @@ spec:
       - name: kubeletsockets
         hostPath:
           path: /var/lib/kubelet/device-plugins
+      - name: nfd-source-hooks
+        hostPath:
+          path: /etc/kubernetes/node-feature-discovery/source.d/
+          type: DirectoryOrCreate
       nodeSelector:
         kubernetes.io/arch: amd64