Copy toplogogy module from cri-resource-manager

Copied topology module to avoid external dependency.
Added GetTopologyInfo function and its test case.
This commit is contained in:
Ed Bartosh 2020-02-11 11:47:19 +02:00
parent 2eae086f6f
commit 3f83e2b2eb
21 changed files with 697 additions and 50 deletions

1
go.mod
View File

@ -7,7 +7,6 @@ require (
github.com/go-ini/ini v1.46.0
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e
github.com/pkg/errors v0.8.1
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456

2
go.sum
View File

@ -281,8 +281,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e h1:+pD7vYnhjGklatQ/Zs98yB/l6ETL+79WiI3IGM9Bmqk=
github.com/intel/cri-resource-manager/pkg/topology v0.0.0-20200207111533-82d10bdaca4e/go.mod h1:98ghzNsdMHeewnXWSFGx+Sf1AbZEVcL+mOLvE6DI4OQ=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

View File

@ -15,13 +15,7 @@
package deviceplugin
import (
"sort"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/intel/cri-resource-manager/pkg/topology"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/topology"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)
@ -34,45 +28,6 @@ type DeviceInfo struct {
topology *pluginapi.TopologyInfo
}
// getTopologyInfo returns topology information for the list of device nodes
func getTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) {
var result pluginapi.TopologyInfo
nodeIDs := map[int64]struct{}{}
for _, dev := range devs {
sysfsDevice, err := topology.FindSysFsDevice(dev)
if err != nil {
return nil, err
}
if sysfsDevice == "" {
return nil, errors.Errorf("device %s doesn't exist", dev)
}
hints, err := topology.NewTopologyHints(sysfsDevice)
if err != nil {
return nil, err
}
for _, hint := range hints {
for _, nNode := range strings.Split(hint.NUMAs, ",") {
nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode)
}
if nNodeID < 0 {
return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID)
}
if _, ok := nodeIDs[nNodeID]; !ok {
result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID})
nodeIDs[nNodeID] = struct{}{}
}
}
}
}
sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID })
return &result, nil
}
// NewDeviceInfo makes DeviceInfo struct and adds topology information to it
func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string) DeviceInfo {
deviceInfo := DeviceInfo{
@ -86,7 +41,7 @@ func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginap
devPaths = append(devPaths, node.HostPath)
}
topologyInfo, err := getTopologyInfo(devPaths)
topologyInfo, err := topology.GetTopologyInfo(devPaths)
if err == nil {
deviceInfo.topology = topologyInfo
}

3
pkg/topology/testdata/go.mod vendored Normal file
View File

@ -0,0 +1,3 @@
// This is a kludge to work around the golang toolchain not being able
// to handle packages with any files with a colon (:) in their name,
// even if those are not golang sources.

View File

@ -0,0 +1 @@
0x030000

View File

@ -0,0 +1 @@
0x5912

View File

@ -0,0 +1 @@
226:1

View File

@ -0,0 +1 @@
226:129

View File

@ -0,0 +1 @@
0-7

View File

@ -0,0 +1 @@
ff

View File

@ -0,0 +1 @@
-1

View File

@ -0,0 +1 @@
0x8086

View File

@ -0,0 +1 @@
0-7

View File

@ -0,0 +1 @@
1,2,3

View File

@ -0,0 +1 @@
0-7

View File

@ -0,0 +1 @@
-1

View File

@ -0,0 +1 @@
0-7

View File

@ -0,0 +1 @@
4,5,6

View File

@ -0,0 +1 @@
../../../../devices/pci0000:00/0000:00:02.0/

282
pkg/topology/topology.go Normal file
View File

@ -0,0 +1,282 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package topology
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)
// to mock in tests
var (
mockRoot = ""
)
const (
// ProviderKubelet is a constant to distinguish that topology hint comes
// from parameters passed to CRI create/update requests from Kubelet
ProviderKubelet = "kubelet"
)
// TopologyHint represents various hints that can be detected from sysfs for the device
type TopologyHint struct {
Provider string
CPUs string
NUMAs string
Sockets string
}
// TopologyHints represents set of hints collected from multiple providers
type TopologyHints map[string]TopologyHint
func getDevicesFromVirtual(realDevPath string) (devs []string, err error) {
if !filepath.HasPrefix(realDevPath, "/sys/devices/virtual") {
return nil, fmt.Errorf("%s is not a virtual device", realDevPath)
}
relPath, _ := filepath.Rel("/sys/devices/virtual", realDevPath)
dir, file := filepath.Split(relPath)
switch dir {
case "vfio/":
iommuGroup := filepath.Join(mockRoot, "/sys/kernel/iommu_groups", file, "devices")
files, err := ioutil.ReadDir(iommuGroup)
if err != nil {
return nil, errors.Wrapf(err, "failed to read IOMMU group %s", iommuGroup)
}
for _, file := range files {
realDev, err := filepath.EvalSymlinks(filepath.Join(iommuGroup, file.Name()))
if err != nil {
return nil, errors.Wrapf(err, "failed to get real path for %s", file.Name())
}
devs = append(devs, realDev)
}
return devs, nil
default:
return nil, nil
}
}
// NewTopologyHints return array of hints for the device and its slaves (e.g. RAID).
func NewTopologyHints(devPath string) (hints TopologyHints, err error) {
hints = make(TopologyHints)
realDevPath, err := filepath.EvalSymlinks(devPath)
if err != nil {
return nil, errors.Wrapf(err, "failed get realpath for %s", devPath)
}
for p := realDevPath; strings.HasPrefix(p, mockRoot+"/sys/devices/"); p = filepath.Dir(p) {
hint := TopologyHint{Provider: p}
fileMap := map[string]*string{
"local_cpulist": &hint.CPUs,
"numa_node": &hint.NUMAs,
}
if err = readFilesInDirectory(fileMap, p); err != nil {
return nil, err
}
// Workarounds for broken information provided by kernel
if hint.NUMAs == "-1" {
// non-NUMA aware device or system, ignore it
hint.NUMAs = ""
}
if hint.NUMAs != "" && hint.CPUs == "" {
// broken topology hint. BIOS reports socket id as NUMA node
// First, try to get hints from parent device or bus.
parentHints, er := NewTopologyHints(filepath.Dir(p))
if er == nil {
cpulist := map[string]bool{}
numalist := map[string]bool{}
for _, h := range parentHints {
if h.CPUs != "" {
cpulist[h.CPUs] = true
}
if h.NUMAs != "" {
numalist[h.NUMAs] = true
}
}
if cpus := strings.Join(mapKeys(cpulist), ","); cpus != "" {
hint.CPUs = cpus
}
if numas := strings.Join(mapKeys(numalist), ","); numas != "" {
hint.NUMAs = numas
}
}
// if after parent hints we still don't have CPUs hints, use numa hint as sockets.
if hint.CPUs == "" && hint.NUMAs != "" {
hint.Sockets = hint.NUMAs
hint.NUMAs = ""
}
}
if hint.CPUs != "" || hint.NUMAs != "" || hint.Sockets != "" {
hints[hint.Provider] = hint
break
}
}
fromVirtual, _ := getDevicesFromVirtual(realDevPath)
slaves, _ := filepath.Glob(filepath.Join(realDevPath, "slaves/*"))
for _, device := range append(slaves, fromVirtual...) {
deviceHints, er := NewTopologyHints(device)
if er != nil {
return nil, er
}
hints = MergeTopologyHints(hints, deviceHints)
}
return
}
// MergeTopologyHints combines org and hints.
func MergeTopologyHints(org, hints TopologyHints) (res TopologyHints) {
if org != nil {
res = org
} else {
res = make(TopologyHints)
}
for k, v := range hints {
if _, ok := res[k]; ok {
continue
}
res[k] = v
}
return
}
// String returns the hints as a string.
func (h *TopologyHint) String() string {
cpus, nodes, sockets, sep := "", "", "", ""
if h.CPUs != "" {
cpus = "CPUs:" + h.CPUs
sep = ", "
}
if h.NUMAs != "" {
nodes = sep + "NUMAs:" + h.NUMAs
sep = ", "
}
if h.Sockets != "" {
sockets = sep + "sockets:" + h.Sockets
}
return "<hints " + cpus + nodes + sockets + " (from " + h.Provider + ")>"
}
// FindSysFsDevice for given argument returns physical device where it is linked to.
// For device nodes it will return path for device itself. For regular files or directories
// this function returns physical device where this inode resides (storage device).
// If result device is a virtual one (e.g. tmpfs), error will be returned.
// For non-existing path, no error returned and path is empty.
func FindSysFsDevice(dev string) (string, error) {
fi, err := os.Stat(dev)
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", errors.Wrapf(err, "unable to get stat for %s", dev)
}
devType := "block"
rdev := fi.Sys().(*syscall.Stat_t).Dev
if mode := fi.Mode(); mode&os.ModeDevice != 0 {
rdev = fi.Sys().(*syscall.Stat_t).Rdev
if mode&os.ModeCharDevice != 0 {
devType = "char"
}
}
major := unix.Major(rdev)
minor := unix.Minor(rdev)
if major == 0 {
return "", errors.Errorf("%s is a virtual device node", dev)
}
devPath := fmt.Sprintf("/sys/dev/%s/%d:%d", devType, major, minor)
realDevPath, err := filepath.EvalSymlinks(devPath)
if err != nil {
return "", errors.Wrapf(err, "failed get realpath for %s", devPath)
}
return filepath.Join(mockRoot, realDevPath), nil
}
// readFilesInDirectory small helper to fill struct with content from sysfs entry
func readFilesInDirectory(fileMap map[string]*string, dir string) error {
for k, v := range fileMap {
b, err := ioutil.ReadFile(filepath.Join(dir, k))
if err != nil {
if os.IsNotExist(err) {
continue
}
return errors.Wrapf(err, "%s: unable to read file %q", dir, k)
}
*v = strings.TrimSpace(string(b))
}
return nil
}
// mapKeys is a small helper that returns slice of keys for a given map
func mapKeys(m map[string]bool) []string {
ret := make([]string, len(m))
i := 0
for k := range m {
ret[i] = k
i++
}
return ret
}
// GetTopologyInfo returns topology information for the list of device nodes
func GetTopologyInfo(devs []string) (*pluginapi.TopologyInfo, error) {
var result pluginapi.TopologyInfo
nodeIDs := map[int64]struct{}{}
for _, dev := range devs {
sysfsDevice, err := FindSysFsDevice(dev)
if err != nil {
return nil, err
}
if sysfsDevice == "" {
return nil, errors.Errorf("device %s doesn't exist", dev)
}
hints, err := NewTopologyHints(sysfsDevice)
if err != nil {
return nil, err
}
for _, hint := range hints {
for _, nNode := range strings.Split(hint.NUMAs, ",") {
nNodeID, err := strconv.ParseInt(strings.TrimSpace(nNode), 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "unable to convert numa node %s into int64", nNode)
}
if nNodeID < 0 {
return nil, errors.Wrapf(err, "numa node is negative: %d", nNodeID)
}
if _, ok := nodeIDs[nNodeID]; !ok {
result.Nodes = append(result.Nodes, &pluginapi.NUMANode{ID: nNodeID})
nodeIDs[nNodeID] = struct{}{}
}
}
}
}
sort.Slice(result.Nodes, func(i, j int) bool { return result.Nodes[i].ID < result.Nodes[j].ID })
return &result, nil
}

View File

@ -0,0 +1,395 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package topology
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)
func setupTestEnv(t *testing.T) func() {
pwd, err := os.Getwd()
if err != nil {
t.Fatal("unable to get current directory")
}
if path, err := filepath.EvalSymlinks(pwd); err == nil {
pwd = path
}
mockRoot = pwd + "/testdata"
teardown := func() {
mockRoot = ""
}
return teardown
}
func TestMapKeys(t *testing.T) {
cases := []struct {
name string
input map[string]bool
output []string
}{
{
name: "empty",
input: map[string]bool{},
output: []string{},
},
{
name: "one",
input: map[string]bool{"a": false},
output: []string{"a"},
},
{
name: "multiple",
input: map[string]bool{"a": false, "b": true, "c": false},
output: []string{"a", "b", "c"},
},
}
for _, tc := range cases {
test := tc
t.Run(test.name, func(t *testing.T) {
t.Parallel()
output := mapKeys(test.input)
sort.Strings(output)
if !reflect.DeepEqual(output, test.output) {
t.Fatalf("expected output: %+v got: %+v", test.output, output)
}
})
}
}
func TestFindSysFsDevice(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
teardown := setupTestEnv(t)
defer teardown()
cases := []struct {
name string
input string
output string
expectedErr bool
}{
{
name: "empty",
input: "",
output: "",
expectedErr: false,
},
{
name: "null",
input: "/dev/null",
output: "/sys/devices/virtual/mem/null",
expectedErr: false,
},
{
name: "proc",
input: "/proc/self",
output: "",
expectedErr: true,
},
}
for _, tc := range cases {
test := tc
t.Run(test.name, func(t *testing.T) {
t.Parallel()
output, err := FindSysFsDevice(test.input)
switch {
case err != nil && !test.expectedErr:
t.Fatalf("unexpected error returned: %+v", err)
case err == nil && test.expectedErr:
t.Fatalf("unexpected success: %+v", output)
case output != test.output:
t.Fatalf("expected: %q got: %q", test.output, output)
}
})
}
}
func TestReadFilesInDirectory(t *testing.T) {
var file, empty string
fname := "test-a"
content := []byte(" something\n")
expectedContent := "something"
fileMap := map[string]*string{
fname: &file,
"non_existing": &empty,
}
dir, err := ioutil.TempDir("", "readFilesInDirectory")
if err != nil {
t.Fatalf("unable to create test directory: %+v", err)
}
defer os.RemoveAll(dir)
ioutil.WriteFile(filepath.Join(dir, fname), content, 0644)
if err = readFilesInDirectory(fileMap, dir); err != nil {
t.Fatalf("unexpected failure: %v", err)
}
if empty != "" {
t.Fatalf("unexpected content: %q", empty)
}
if file != expectedContent {
t.Fatalf("unexpected content: %q expected: %q", file, expectedContent)
}
}
func TestGetDevicesFromVirtual(t *testing.T) {
teardown := setupTestEnv(t)
defer teardown()
cases := []struct {
name string
input string
output []string
expectedErr bool
}{
{
name: "vfio",
input: "/sys/devices/virtual/vfio/42",
output: []string{mockRoot + "/sys/devices/pci0000:00/0000:00:02.0"},
expectedErr: false,
},
{
name: "misc",
input: "/sys/devices/virtual/misc/vfio",
output: nil,
expectedErr: false,
},
{
name: "missing-iommu-group",
input: "/sys/devices/virtual/vfio/84",
output: nil,
expectedErr: true,
},
{
name: "non-virtual",
input: "/sys/devices/pci0000:00/0000:00:02.0",
output: nil,
expectedErr: true,
},
}
for _, tc := range cases {
test := tc
t.Run(test.name, func(t *testing.T) {
output, err := getDevicesFromVirtual(test.input)
switch {
case err != nil && !test.expectedErr:
t.Fatalf("unexpected error returned: %+v", err)
case err == nil && test.expectedErr:
t.Fatalf("unexpected success: %+v", output)
case len(output) != len(test.output):
t.Fatalf("expected: %q got: %q", len(test.output), len(output))
}
for i, p := range test.output {
if test.output[i] != p {
t.Fatalf("expected: %q got: %q", test.output[i], p)
}
}
})
}
}
func TestMergeTopologyHints(t *testing.T) {
cases := []struct {
name string
inputA TopologyHints
inputB TopologyHints
expectedOutput TopologyHints
expectedErr bool
}{
{
name: "empty",
inputA: nil,
inputB: nil,
expectedOutput: TopologyHints{},
},
{
name: "one,nil",
inputA: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
inputB: nil,
expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
},
{
name: "nil, one",
inputA: nil,
inputB: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
},
{
name: "duplicate",
inputA: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
inputB: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
expectedOutput: TopologyHints{"test": TopologyHint{Provider: "test", CPUs: "0"}},
},
{
name: "two",
inputA: TopologyHints{"test1": TopologyHint{Provider: "test1", CPUs: "0"}},
inputB: TopologyHints{"test2": TopologyHint{Provider: "test2", CPUs: "1"}},
expectedOutput: TopologyHints{
"test1": TopologyHint{Provider: "test1", CPUs: "0"},
"test2": TopologyHint{Provider: "test2", CPUs: "1"},
},
},
}
for _, tc := range cases {
test := tc
t.Run(test.name, func(t *testing.T) {
t.Parallel()
output := MergeTopologyHints(test.inputA, test.inputB)
if !reflect.DeepEqual(output, test.expectedOutput) {
t.Fatalf("expected output: %+v got: %+v", test.expectedOutput, output)
}
})
}
}
func TestNewTopologyHints(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
teardown := setupTestEnv(t)
defer teardown()
cases := []struct {
name string
input string
output TopologyHints
expectedErr bool
}{
{
name: "empty",
input: "non-existing",
output: nil,
expectedErr: true,
},
{
name: "pci card1",
input: mockRoot + "/sys/devices/pci0000:00/0000:00:02.0/drm/card1",
output: TopologyHints{
mockRoot + "/sys/devices/pci0000:00/0000:00:02.0": TopologyHint{
Provider: mockRoot + "/sys/devices/pci0000:00/0000:00:02.0",
CPUs: "0-7",
NUMAs: "",
Sockets: ""},
},
expectedErr: false,
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
output, err := NewTopologyHints(test.input)
switch {
case err != nil && !test.expectedErr:
t.Fatalf("unexpected error returned: %+v", err)
case err == nil && test.expectedErr:
t.Fatalf("unexpected success: %+v", output)
case !reflect.DeepEqual(output, test.output):
t.Fatalf("expected: %q got: %q", test.output, output)
}
})
}
}
func TestGetTopologyInfo(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
teardown := setupTestEnv(t)
defer teardown()
cases := []struct {
name string
input []string
output *pluginapi.TopologyInfo
expectedErr bool
}{
{
name: "valid: device with 3 numa nodes",
input: []string{"/dev/null"},
output: &pluginapi.TopologyInfo{
Nodes: []*pluginapi.NUMANode{
{ID: 1},
{ID: 2},
{ID: 3},
},
},
expectedErr: false,
},
{
name: "valid: 2 identical devices with 3 numa nodes",
input: []string{"/dev/null", "/dev/null"},
output: &pluginapi.TopologyInfo{
Nodes: []*pluginapi.NUMANode{
{ID: 1},
{ID: 2},
{ID: 3},
},
},
expectedErr: false,
},
{
name: "valid: 2 different devicees with 3 numa nodes each",
input: []string{"/dev/tty", "/dev/null"},
output: &pluginapi.TopologyInfo{
Nodes: []*pluginapi.NUMANode{
{ID: 1},
{ID: 2},
{ID: 3},
{ID: 4},
{ID: 5},
{ID: 6},
},
},
expectedErr: false,
},
{
name: "invalid: device doesn't exist",
input: []string{"/dev/non-existing-device"},
output: nil,
expectedErr: true,
},
{
name: "invalid: empty device path",
input: []string{""},
output: nil,
expectedErr: true,
},
{
name: "incorrect numa node ID",
input: []string{"/dev/random"},
output: nil,
expectedErr: true,
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
output, err := GetTopologyInfo(test.input)
switch {
case err != nil && !test.expectedErr:
t.Fatalf("unexpected error returned: %+v", err)
case err == nil && test.expectedErr:
t.Fatalf("unexpected success: %+v", output)
case !reflect.DeepEqual(output, test.output):
t.Fatalf("expected: %q got: %q", test.output, output)
}
})
}
}