From 3f83e2b2eb5daa291ff843b38dd7a54173dd6873 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 11 Feb 2020 11:47:19 +0200 Subject: [PATCH] Copy toplogogy module from cri-resource-manager Copied topology module to avoid external dependency. Added GetTopologyInfo function and its test case. --- go.mod | 1 - go.sum | 2 - pkg/deviceplugin/api.go | 49 +-- pkg/topology/testdata/go.mod | 3 + .../sys/devices/pci0000:00/0000:00:02.0/class | 1 + .../devices/pci0000:00/0000:00:02.0/device | 1 + .../pci0000:00/0000:00:02.0/drm/card1/dev | 1 + .../0000:00:02.0/drm/renderD129/dev | 1 + .../pci0000:00/0000:00:02.0/local_cpulist | 1 + .../pci0000:00/0000:00:02.0/local_cpus | 1 + .../devices/pci0000:00/0000:00:02.0/numa_node | 1 + .../devices/pci0000:00/0000:00:02.0/vendor | 1 + .../devices/virtual/mem/null/local_cpulist | 1 + .../sys/devices/virtual/mem/null/numa_node | 1 + .../devices/virtual/mem/random/local_cpulist | 1 + .../sys/devices/virtual/mem/random/numa_node | 1 + .../sys/devices/virtual/tty/tty/local_cpulist | 1 + .../sys/devices/virtual/tty/tty/numa_node | 1 + .../iommu_groups/42/devices/0000:00:02.0 | 1 + pkg/topology/topology.go | 282 +++++++++++++ pkg/topology/topology_test.go | 395 ++++++++++++++++++ 21 files changed, 697 insertions(+), 50 deletions(-) create mode 100644 pkg/topology/testdata/go.mod create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/class create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/device create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/card1/dev create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/renderD129/dev create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpulist create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpus create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/numa_node create mode 100644 pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/vendor create mode 100644 pkg/topology/testdata/sys/devices/virtual/mem/null/local_cpulist create mode 100644 pkg/topology/testdata/sys/devices/virtual/mem/null/numa_node create mode 100644 pkg/topology/testdata/sys/devices/virtual/mem/random/local_cpulist create mode 100644 pkg/topology/testdata/sys/devices/virtual/mem/random/numa_node create mode 100644 pkg/topology/testdata/sys/devices/virtual/tty/tty/local_cpulist create mode 100644 pkg/topology/testdata/sys/devices/virtual/tty/tty/numa_node create mode 120000 pkg/topology/testdata/sys/kernel/iommu_groups/42/devices/0000:00:02.0 create mode 100644 pkg/topology/topology.go create mode 100644 pkg/topology/topology_test.go diff --git a/go.mod b/go.mod index f8e561fc..d24acb13 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/go-ini/ini v1.46.0 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750 - github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e github.com/pkg/errors v0.8.1 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 diff --git a/go.sum b/go.sum index d7575ef2..098bdc7a 100644 --- a/go.sum +++ b/go.sum @@ -281,8 +281,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e h1:+pD7vYnhjGklatQ/Zs98yB/l6ETL+79WiI3IGM9Bmqk= -github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e/go.mod h1:98ghzNsdMHeewnXWSFGx+Sf1AbZEVcL+mOLvE6DI4OQ= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/pkg/deviceplugin/api.go b/pkg/deviceplugin/api.go index 180e6015..99594eba 100644 --- a/pkg/deviceplugin/api.go +++ b/pkg/deviceplugin/api.go @@ -15,13 +15,7 @@ package deviceplugin import ( - "sort" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/intel/cri-resource-manager/pkg/topology" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/topology" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) @@ -34,45 +28,6 @@ type DeviceInfo struct { topology *pluginapi.TopologyInfo } -// getTopologyInfo returns topology information for the list of device nodes -func getTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) { - var result pluginapi.TopologyInfo - nodeIDs := map[int64]struct{}{} - for _, dev := range devs { - sysfsDevice, err := topology.FindSysFsDevice(dev) - if err != nil { - return nil, err - } - - if sysfsDevice == "" { - return nil, errors.Errorf("device %s doesn't exist", dev) - } - - hints, err := topology.NewTopologyHints(sysfsDevice) - if err != nil { - return nil, err - } - - for _, hint := range hints { - for _, nNode := range strings.Split(hint.NUMAs, ",") { - nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode) - } - if nNodeID < 0 { - return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID) - } - if _, ok := nodeIDs[nNodeID]; !ok { - result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID}) - nodeIDs[nNodeID] = struct{}{} - } - } - } - } - sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID }) - return &result, nil -} - // NewDeviceInfo makes DeviceInfo struct and adds topology information to it func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string) DeviceInfo { deviceInfo := DeviceInfo{ @@ -86,7 +41,7 @@ func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginap devPaths = append(devPaths, node.HostPath) } - topologyInfo, err := getTopologyInfo(devPaths) + topologyInfo, err := topology.GetTopologyInfo(devPaths) if err == nil { deviceInfo.topology = topologyInfo } diff --git a/pkg/topology/testdata/go.mod b/pkg/topology/testdata/go.mod new file mode 100644 index 00000000..0fc0d9c1 --- /dev/null +++ b/pkg/topology/testdata/go.mod @@ -0,0 +1,3 @@ +// This is a kludge to work around the golang toolchain not being able +// to handle packages with any files with a colon (:) in their name, +// even if those are not golang sources. diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/class b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/class new file mode 100644 index 00000000..c5f02e33 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/class @@ -0,0 +1 @@ +0x030000 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/device b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/device new file mode 100644 index 00000000..70d6a1ce --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/device @@ -0,0 +1 @@ +0x5912 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/card1/dev b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/card1/dev new file mode 100644 index 00000000..fdd256a9 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/card1/dev @@ -0,0 +1 @@ +226:1 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/renderD129/dev b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/renderD129/dev new file mode 100644 index 00000000..c8d81347 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/drm/renderD129/dev @@ -0,0 +1 @@ +226:129 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpulist b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpulist new file mode 100644 index 00000000..74fc2fb6 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpulist @@ -0,0 +1 @@ +0-7 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpus b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpus new file mode 100644 index 00000000..fcd15acf --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/local_cpus @@ -0,0 +1 @@ +ff diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/numa_node b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/numa_node new file mode 100644 index 00000000..3a2e3f49 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/numa_node @@ -0,0 +1 @@ +-1 diff --git a/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/vendor b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/vendor new file mode 100644 index 00000000..ce6dc4da --- /dev/null +++ b/pkg/topology/testdata/sys/devices/pci0000:00/0000:00:02.0/vendor @@ -0,0 +1 @@ +0x8086 diff --git a/pkg/topology/testdata/sys/devices/virtual/mem/null/local_cpulist b/pkg/topology/testdata/sys/devices/virtual/mem/null/local_cpulist new file mode 100644 index 00000000..74fc2fb6 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/mem/null/local_cpulist @@ -0,0 +1 @@ +0-7 diff --git a/pkg/topology/testdata/sys/devices/virtual/mem/null/numa_node b/pkg/topology/testdata/sys/devices/virtual/mem/null/numa_node new file mode 100644 index 00000000..b0246d59 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/mem/null/numa_node @@ -0,0 +1 @@ +1,2,3 diff --git a/pkg/topology/testdata/sys/devices/virtual/mem/random/local_cpulist b/pkg/topology/testdata/sys/devices/virtual/mem/random/local_cpulist new file mode 100644 index 00000000..74fc2fb6 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/mem/random/local_cpulist @@ -0,0 +1 @@ +0-7 diff --git a/pkg/topology/testdata/sys/devices/virtual/mem/random/numa_node b/pkg/topology/testdata/sys/devices/virtual/mem/random/numa_node new file mode 100644 index 00000000..3a2e3f49 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/mem/random/numa_node @@ -0,0 +1 @@ +-1 diff --git a/pkg/topology/testdata/sys/devices/virtual/tty/tty/local_cpulist b/pkg/topology/testdata/sys/devices/virtual/tty/tty/local_cpulist new file mode 100644 index 00000000..74fc2fb6 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/tty/tty/local_cpulist @@ -0,0 +1 @@ +0-7 diff --git a/pkg/topology/testdata/sys/devices/virtual/tty/tty/numa_node b/pkg/topology/testdata/sys/devices/virtual/tty/tty/numa_node new file mode 100644 index 00000000..6400ac84 --- /dev/null +++ b/pkg/topology/testdata/sys/devices/virtual/tty/tty/numa_node @@ -0,0 +1 @@ +4,5,6 diff --git a/pkg/topology/testdata/sys/kernel/iommu_groups/42/devices/0000:00:02.0 b/pkg/topology/testdata/sys/kernel/iommu_groups/42/devices/0000:00:02.0 new file mode 120000 index 00000000..485b1d27 --- /dev/null +++ b/pkg/topology/testdata/sys/kernel/iommu_groups/42/devices/0000:00:02.0 @@ -0,0 +1 @@ +../../../../devices/pci0000:00/0000:00:02.0/ \ No newline at end of file diff --git a/pkg/topology/topology.go b/pkg/topology/topology.go new file mode 100644 index 00000000..56471c3b --- /dev/null +++ b/pkg/topology/topology.go @@ -0,0 +1,282 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +// to mock in tests +var ( + mockRoot = "" +) + +const ( + // ProviderKubelet is a constant to distinguish that topology hint comes + // from parameters passed to CRI create/update requests from Kubelet + ProviderKubelet = "kubelet" +) + +// TopologyHint represents various hints that can be detected from sysfs for the device +type TopologyHint struct { + Provider string + CPUs string + NUMAs string + Sockets string +} + +// TopologyHints represents set of hints collected from multiple providers +type TopologyHints map[string]TopologyHint + +func getDevicesFromVirtual(realDevPath string) (devs []string, err error) { + if !filepath.HasPrefix(realDevPath, "/sys/devices/virtual") { + return nil, fmt.Errorf("%s is not a virtual device", realDevPath) + } + + relPath, _ := filepath.Rel("/sys/devices/virtual", realDevPath) + + dir, file := filepath.Split(relPath) + switch dir { + case "vfio/": + iommuGroup := filepath.Join(mockRoot, "/sys/kernel/iommu_groups", file, "devices") + files, err := ioutil.ReadDir(iommuGroup) + if err != nil { + return nil, errors.Wrapf(err, "failed to read IOMMU group %s", iommuGroup) + } + for _, file := range files { + realDev, err := filepath.EvalSymlinks(filepath.Join(iommuGroup, file.Name())) + if err != nil { + return nil, errors.Wrapf(err, "failed to get real path for %s", file.Name()) + } + devs = append(devs, realDev) + } + return devs, nil + default: + return nil, nil + } +} + +// NewTopologyHints return array of hints for the device and its slaves (e.g. RAID). +func NewTopologyHints(devPath string) (hints TopologyHints, err error) { + hints = make(TopologyHints) + realDevPath, err := filepath.EvalSymlinks(devPath) + if err != nil { + return nil, errors.Wrapf(err, "failed get realpath for %s", devPath) + } + for p := realDevPath; strings.HasPrefix(p, mockRoot+"/sys/devices/"); p = filepath.Dir(p) { + hint := TopologyHint{Provider: p} + fileMap := map[string]*string{ + "local_cpulist": &hint.CPUs, + "numa_node": &hint.NUMAs, + } + if err = readFilesInDirectory(fileMap, p); err != nil { + return nil, err + } + // Workarounds for broken information provided by kernel + if hint.NUMAs == "-1" { + // non-NUMA aware device or system, ignore it + hint.NUMAs = "" + } + if hint.NUMAs != "" && hint.CPUs == "" { + // broken topology hint. BIOS reports socket id as NUMA node + // First, try to get hints from parent device or bus. + parentHints, er := NewTopologyHints(filepath.Dir(p)) + if er == nil { + cpulist := map[string]bool{} + numalist := map[string]bool{} + for _, h := range parentHints { + if h.CPUs != "" { + cpulist[h.CPUs] = true + } + if h.NUMAs != "" { + numalist[h.NUMAs] = true + } + } + if cpus := strings.Join(mapKeys(cpulist), ","); cpus != "" { + hint.CPUs = cpus + } + if numas := strings.Join(mapKeys(numalist), ","); numas != "" { + hint.NUMAs = numas + } + } + // if after parent hints we still don't have CPUs hints, use numa hint as sockets. + if hint.CPUs == "" && hint.NUMAs != "" { + hint.Sockets = hint.NUMAs + hint.NUMAs = "" + } + } + if hint.CPUs != "" || hint.NUMAs != "" || hint.Sockets != "" { + hints[hint.Provider] = hint + break + } + } + fromVirtual, _ := getDevicesFromVirtual(realDevPath) + slaves, _ := filepath.Glob(filepath.Join(realDevPath, "slaves/*")) + for _, device := range append(slaves, fromVirtual...) { + deviceHints, er := NewTopologyHints(device) + if er != nil { + return nil, er + } + hints = MergeTopologyHints(hints, deviceHints) + } + return +} + +// MergeTopologyHints combines org and hints. +func MergeTopologyHints(org, hints TopologyHints) (res TopologyHints) { + if org != nil { + res = org + } else { + res = make(TopologyHints) + } + for k, v := range hints { + if _, ok := res[k]; ok { + continue + } + res[k] = v + } + return +} + +// String returns the hints as a string. +func (h *TopologyHint) String() string { + cpus, nodes, sockets, sep := "", "", "", "" + + if h.CPUs != "" { + cpus = "CPUs:" + h.CPUs + sep = ", " + } + if h.NUMAs != "" { + nodes = sep + "NUMAs:" + h.NUMAs + sep = ", " + } + if h.Sockets != "" { + sockets = sep + "sockets:" + h.Sockets + } + + return "" +} + +// FindSysFsDevice for given argument returns physical device where it is linked to. +// For device nodes it will return path for device itself. For regular files or directories +// this function returns physical device where this inode resides (storage device). +// If result device is a virtual one (e.g. tmpfs), error will be returned. +// For non-existing path, no error returned and path is empty. +func FindSysFsDevice(dev string) (string, error) { + fi, err := os.Stat(dev) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", errors.Wrapf(err, "unable to get stat for %s", dev) + } + + devType := "block" + rdev := fi.Sys().(*syscall.Stat_t).Dev + if mode := fi.Mode(); mode&os.ModeDevice != 0 { + rdev = fi.Sys().(*syscall.Stat_t).Rdev + if mode&os.ModeCharDevice != 0 { + devType = "char" + } + } + + major := unix.Major(rdev) + minor := unix.Minor(rdev) + if major == 0 { + return "", errors.Errorf("%s is a virtual device node", dev) + } + devPath := fmt.Sprintf("/sys/dev/%s/%d:%d", devType, major, minor) + realDevPath, err := filepath.EvalSymlinks(devPath) + if err != nil { + return "", errors.Wrapf(err, "failed get realpath for %s", devPath) + } + return filepath.Join(mockRoot, realDevPath), nil +} + +// readFilesInDirectory small helper to fill struct with content from sysfs entry +func readFilesInDirectory(fileMap map[string]*string, dir string) error { + for k, v := range fileMap { + b, err := ioutil.ReadFile(filepath.Join(dir, k)) + if err != nil { + if os.IsNotExist(err) { + continue + } + return errors.Wrapf(err, "%s: unable to read file %q", dir, k) + } + *v = strings.TrimSpace(string(b)) + } + return nil +} + +// mapKeys is a small helper that returns slice of keys for a given map +func mapKeys(m map[string]bool) []string { + ret := make([]string, len(m)) + i := 0 + for k := range m { + ret[i] = k + i++ + } + return ret +} + +// GetTopologyInfo returns topology information for the list of device nodes +func GetTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) { + var result pluginapi.TopologyInfo + nodeIDs := map[int64]struct{}{} + for _, dev := range devs { + sysfsDevice, err := FindSysFsDevice(dev) + if err != nil { + return nil, err + } + + if sysfsDevice == "" { + return nil, errors.Errorf("device %s doesn't exist", dev) + } + + hints, err := NewTopologyHints(sysfsDevice) + if err != nil { + return nil, err + } + + for _, hint := range hints { + for _, nNode := range strings.Split(hint.NUMAs, ",") { + nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode) + } + if nNodeID < 0 { + return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID) + } + if _, ok := nodeIDs[nNodeID]; !ok { + result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID}) + nodeIDs[nNodeID] = struct{}{} + } + } + } + } + sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID }) + return &result, nil +} diff --git a/pkg/topology/topology_test.go b/pkg/topology/topology_test.go new file mode 100644 index 00000000..5e34db2e --- /dev/null +++ b/pkg/topology/topology_test.go @@ -0,0 +1,395 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +func setupTestEnv(t *testing.T) func() { + pwd, err := os.Getwd() + if err != nil { + t.Fatal("unable to get current directory") + } + if path, err := filepath.EvalSymlinks(pwd); err == nil { + pwd = path + } + mockRoot = pwd + "/testdata" + teardown := func() { + mockRoot = "" + } + return teardown +} + +func TestMapKeys(t *testing.T) { + cases := []struct { + name string + input map[string]bool + output []string + }{ + { + name: "empty", + input: map[string]bool{}, + output: []string{}, + }, + { + name: "one", + input: map[string]bool{"a": false}, + output: []string{"a"}, + }, + { + name: "multiple", + input: map[string]bool{"a": false, "b": true, "c": false}, + output: []string{"a", "b", "c"}, + }, + } + for _, tc := range cases { + test := tc + t.Run(test.name, func(t *testing.T) { + t.Parallel() + output := mapKeys(test.input) + sort.Strings(output) + if !reflect.DeepEqual(output, test.output) { + t.Fatalf("expected output: %+v got: %+v", test.output, output) + } + }) + } +} + +func TestFindSysFsDevice(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + teardown := setupTestEnv(t) + defer teardown() + cases := []struct { + name string + input string + output string + expectedErr bool + }{ + { + name: "empty", + input: "", + output: "", + expectedErr: false, + }, + { + name: "null", + input: "/dev/null", + output: "/sys/devices/virtual/mem/null", + expectedErr: false, + }, + { + name: "proc", + input: "/proc/self", + output: "", + expectedErr: true, + }, + } + for _, tc := range cases { + test := tc + t.Run(test.name, func(t *testing.T) { + t.Parallel() + output, err := FindSysFsDevice(test.input) + switch { + case err != nil && !test.expectedErr: + t.Fatalf("unexpected error returned: %+v", err) + case err == nil && test.expectedErr: + t.Fatalf("unexpected success: %+v", output) + case output != test.output: + t.Fatalf("expected: %q got: %q", test.output, output) + } + }) + } +} + +func TestReadFilesInDirectory(t *testing.T) { + var file, empty string + fname := "test-a" + content := []byte(" something\n") + expectedContent := "something" + + fileMap := map[string]*string{ + fname: &file, + "non_existing": &empty, + } + + dir, err := ioutil.TempDir("", "readFilesInDirectory") + if err != nil { + t.Fatalf("unable to create test directory: %+v", err) + } + defer os.RemoveAll(dir) + ioutil.WriteFile(filepath.Join(dir, fname), content, 0644) + + if err = readFilesInDirectory(fileMap, dir); err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if empty != "" { + t.Fatalf("unexpected content: %q", empty) + } + if file != expectedContent { + t.Fatalf("unexpected content: %q expected: %q", file, expectedContent) + } +} + +func TestGetDevicesFromVirtual(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + cases := []struct { + name string + input string + output []string + expectedErr bool + }{ + { + name: "vfio", + input: "/sys/devices/virtual/vfio/42", + output: []string{mockRoot + "/sys/devices/pci0000:00/0000:00:02.0"}, + expectedErr: false, + }, + { + name: "misc", + input: "/sys/devices/virtual/misc/vfio", + output: nil, + expectedErr: false, + }, + { + name: "missing-iommu-group", + input: "/sys/devices/virtual/vfio/84", + output: nil, + expectedErr: true, + }, + { + name: "non-virtual", + input: "/sys/devices/pci0000:00/0000:00:02.0", + output: nil, + expectedErr: true, + }, + } + + for _, tc := range cases { + test := tc + t.Run(test.name, func(t *testing.T) { + output, err := getDevicesFromVirtual(test.input) + switch { + case err != nil && !test.expectedErr: + t.Fatalf("unexpected error returned: %+v", err) + case err == nil && test.expectedErr: + t.Fatalf("unexpected success: %+v", output) + case len(output) != len(test.output): + t.Fatalf("expected: %q got: %q", len(test.output), len(output)) + } + for i, p := range test.output { + if test.output[i] != p { + t.Fatalf("expected: %q got: %q", test.output[i], p) + } + } + }) + } +} + +func TestMergeTopologyHints(t *testing.T) { + cases := []struct { + name string + inputA TopologyHints + inputB TopologyHints + expectedOutput TopologyHints + expectedErr bool + }{ + { + name: "empty", + inputA: nil, + inputB: nil, + expectedOutput: TopologyHints{}, + }, + { + name: "one,nil", + inputA: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + inputB: nil, + expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + }, + { + name: "nil, one", + inputA: nil, + inputB: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + }, + { + name: "duplicate", + inputA: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + inputB: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}}, + }, + { + name: "two", + inputA: TopologyHints{"test1": TopologyHint{Provider: "test1", CPUs: "0"}}, + inputB: TopologyHints{"test2": TopologyHint{Provider: "test2", CPUs: "1"}}, + expectedOutput: TopologyHints{ + "test1": TopologyHint{Provider: "test1", CPUs: "0"}, + "test2": TopologyHint{Provider: "test2", CPUs: "1"}, + }, + }, + } + for _, tc := range cases { + test := tc + t.Run(test.name, func(t *testing.T) { + t.Parallel() + output := MergeTopologyHints(test.inputA, test.inputB) + if !reflect.DeepEqual(output, test.expectedOutput) { + t.Fatalf("expected output: %+v got: %+v", test.expectedOutput, output) + } + }) + } +} + +func TestNewTopologyHints(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + teardown := setupTestEnv(t) + defer teardown() + cases := []struct { + name string + input string + output TopologyHints + expectedErr bool + }{ + { + name: "empty", + input: "non-existing", + output: nil, + expectedErr: true, + }, + { + name: "pci card1", + input: mockRoot + "/sys/devices/pci0000:00/0000:00:02.0/drm/card1", + output: TopologyHints{ + mockRoot + "/sys/devices/pci0000:00/0000:00:02.0": TopologyHint{ + Provider: mockRoot + "/sys/devices/pci0000:00/0000:00:02.0", + CPUs: "0-7", + NUMAs: "", + Sockets: ""}, + }, + expectedErr: false, + }, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + output, err := NewTopologyHints(test.input) + switch { + case err != nil && !test.expectedErr: + t.Fatalf("unexpected error returned: %+v", err) + case err == nil && test.expectedErr: + t.Fatalf("unexpected success: %+v", output) + case !reflect.DeepEqual(output, test.output): + t.Fatalf("expected: %q got: %q", test.output, output) + } + }) + } +} + +func TestGetTopologyInfo(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + teardown := setupTestEnv(t) + defer teardown() + cases := []struct { + name string + input []string + output *pluginapi.TopologyInfo + expectedErr bool + }{ + { + name: "valid: device with 3 numa nodes", + input: []string{"/dev/null"}, + output: &pluginapi.TopologyInfo{ + Nodes: []*pluginapi.NUMANode{ + {ID: 1}, + {ID: 2}, + {ID: 3}, + }, + }, + expectedErr: false, + }, + { + name: "valid: 2 identical devices with 3 numa nodes", + input: []string{"/dev/null", "/dev/null"}, + output: &pluginapi.TopologyInfo{ + Nodes: []*pluginapi.NUMANode{ + {ID: 1}, + {ID: 2}, + {ID: 3}, + }, + }, + expectedErr: false, + }, + { + name: "valid: 2 different devicees with 3 numa nodes each", + input: []string{"/dev/tty", "/dev/null"}, + output: &pluginapi.TopologyInfo{ + Nodes: []*pluginapi.NUMANode{ + {ID: 1}, + {ID: 2}, + {ID: 3}, + {ID: 4}, + {ID: 5}, + {ID: 6}, + }, + }, + expectedErr: false, + }, + { + name: "invalid: device doesn't exist", + input: []string{"/dev/non-existing-device"}, + output: nil, + expectedErr: true, + }, + { + name: "invalid: empty device path", + input: []string{""}, + output: nil, + expectedErr: true, + }, + { + name: "incorrect numa node ID", + input: []string{"/dev/random"}, + output: nil, + expectedErr: true, + }, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + output, err := GetTopologyInfo(test.input) + switch { + case err != nil && !test.expectedErr: + t.Fatalf("unexpected error returned: %+v", err) + case err == nil && test.expectedErr: + t.Fatalf("unexpected success: %+v", output) + case !reflect.DeepEqual(output, test.output): + t.Fatalf("expected: %q got: %q", test.output, output) + } + }) + } +}