diff --git a/Gopkg.lock b/Gopkg.lock index 1080b742..67e3dbf8 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -25,6 +25,14 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:d74ba1aea6244ead12e4f16d5f61a15ced9a2f2d1cae2021fbb76088b27e7afa" + name = "github.com/go-ini/ini" + packages = ["."] + pruneopts = "NUT" + revision = "c85607071cf08ca1adaf48319cd1aa322e81d8c1" + version = "v1.42.0" + [[projects]] digest = "1:cd4f86461732066e277335465962660cbf02999e18d5bbb5e9285eac4608b970" name = "github.com/gogo/protobuf" @@ -619,6 +627,7 @@ analyzer-version = 1 input-imports = [ "github.com/fsnotify/fsnotify", + "github.com/go-ini/ini", "github.com/pkg/errors", "google.golang.org/grpc", "google.golang.org/grpc/metadata", diff --git a/build/docker/intel-qat-plugin.Dockerfile b/build/docker/intel-qat-plugin.Dockerfile index f6d908cc..eaefa389 100644 --- a/build/docker/intel-qat-plugin.Dockerfile +++ b/build/docker/intel-qat-plugin.Dockerfile @@ -1,10 +1,22 @@ -FROM golang:1.11-alpine as builder -ARG DIR=/go/src/github.com/intel/intel-device-plugins-for-kubernetes +FROM fedora:28 as builder +RUN dnf update -y && \ + dnf install -y wget make gcc-c++ findutils golang-bin && \ + mkdir -p /usr/src/qat && \ + cd /usr/src/qat && \ + wget https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.l.4.3.0-00033.tar.gz && \ + tar xf *.tar.gz +RUN cd /usr/src/qat/quickassist/utilities/adf_ctl && \ + make KERNEL_SOURCE_DIR=/usr/src/qat/quickassist/qat && \ + cp -a adf_ctl /usr/bin/ +ARG DIR=/root/go/src/github.com/intel/intel-device-plugins-for-kubernetes WORKDIR $DIR COPY . . RUN cd cmd/qat_plugin; go install -RUN chmod a+x /go/bin/qat_plugin +RUN chmod a+x /root/go/bin/qat_plugin -FROM alpine -COPY --from=builder /go/bin/qat_plugin /usr/bin/intel_qat_device_plugin +FROM fedora:28 +RUN dnf update -y && \ + dnf install -y libstdc++ +COPY --from=builder /root/go/bin/qat_plugin /usr/bin/intel_qat_device_plugin +COPY --from=builder /usr/bin/adf_ctl /usr/bin/adf_ctl CMD ["/usr/bin/intel_qat_device_plugin"] diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go new file mode 100644 index 00000000..8700a1ef --- /dev/null +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -0,0 +1,321 @@ +// Copyright 2017 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dpdkdrv + +import ( + "bytes" + "fmt" + "io/ioutil" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" +) + +const ( + uioDevicePath = "/dev" + vfioDevicePath = "/dev/vfio" + uioMountPath = "/sys/class/uio" + pciDeviceDirectory = "/sys/bus/pci/devices" + pciDriverDirectory = "/sys/bus/pci/drivers" + uioSuffix = "uio" + iommuGroupSuffix = "iommu_group" + newIDSuffix = "new_id" + driverUnbindSuffix = "driver/unbind" + vendorPrefix = "8086 " + envVarPrefix = "QAT" +) + +// DevicePlugin represents vfio based QAT plugin. +type DevicePlugin struct { + maxDevices int + pciDriverDir string + pciDeviceDir string + kernelVfDrivers []string + dpdkDriver string +} + +// NewDevicePlugin returns new instance of vfio based QAT plugin. +func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string) (*DevicePlugin, error) { + if !isValidDpdkDeviceDriver(dpdkDriver) { + return nil, errors.Errorf("wrong DPDK device driver: %s", dpdkDriver) + } + + kernelDrivers := strings.Split(kernelVfDrivers, ",") + for _, driver := range kernelDrivers { + if !isValidKerneDriver(driver) { + return nil, errors.Errorf("wrong kernel VF driver: %s", driver) + } + } + + return newDevicePlugin(pciDriverDirectory, pciDeviceDirectory, maxDevices, kernelDrivers, dpdkDriver), nil +} + +func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVfDrivers []string, dpdkDriver string) *DevicePlugin { + return &DevicePlugin{ + maxDevices: maxDevices, + pciDriverDir: pciDriverDir, + pciDeviceDir: pciDeviceDir, + kernelVfDrivers: kernelVfDrivers, + dpdkDriver: dpdkDriver, + } +} + +// Scan implements Scanner interface for vfio based QAT plugin. +func (dp *DevicePlugin) Scan(notifier deviceplugin.Notifier) error { + for { + devTree, err := dp.scan() + if err != nil { + return err + } + + notifier.Notify(devTree) + + time.Sleep(5 * time.Second) + } +} + +func (dp *DevicePlugin) getDpdkDevice(id string) (string, error) { + + devicePCIAdd := "0000:" + id + switch dp.dpdkDriver { + // TODO: case "pci-generic" and "kernel": + case "igb_uio": + uioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, uioSuffix) + files, err := ioutil.ReadDir(uioDirPath) + if err != nil { + return "", err + } + if len(files) == 0 { + return "", errors.New("No devices found") + } + return files[0].Name(), nil + + case "vfio-pci": + vfioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, iommuGroupSuffix) + group, err := filepath.EvalSymlinks(vfioDirPath) + if err != nil { + return "", errors.WithStack(err) + } + s := path.Base(group) + fmt.Printf("The vfio device group detected is %v\n", s) + return s, nil + } + + return "", errors.New("Unknown DPDK driver") +} + +func (dp *DevicePlugin) getDpdkDeviceSpecs(id string) ([]pluginapi.DeviceSpec, error) { + dpdkDeviceName, err := dp.getDpdkDevice(id) + if err != nil { + return nil, err + } + fmt.Printf("%s device: corresponding DPDK device detected is %s\n", id, dpdkDeviceName) + + switch dp.dpdkDriver { + // TODO: case "pci-generic" and "kernel": + case "igb_uio": + //Setting up with uio + uioDev := path.Join(uioDevicePath, dpdkDeviceName) + return []pluginapi.DeviceSpec{ + { + HostPath: uioDev, + ContainerPath: uioDev, + Permissions: "rw", + }, + }, nil + case "vfio-pci": + //Setting up with vfio + vfioDev1 := path.Join(vfioDevicePath, dpdkDeviceName) + vfioDev2 := path.Join(vfioDevicePath, "/vfio") + return []pluginapi.DeviceSpec{ + { + HostPath: vfioDev1, + ContainerPath: vfioDev1, + Permissions: "rw", + }, + { + HostPath: vfioDev2, + ContainerPath: vfioDev2, + Permissions: "rw", + }, + }, nil + } + + return nil, errors.New("Unknown DPDK driver") +} + +func (dp *DevicePlugin) getDpdkMounts(id string) ([]pluginapi.Mount, error) { + dpdkDeviceName, err := dp.getDpdkDevice(id) + if err != nil { + return nil, err + } + + switch dp.dpdkDriver { + case "igb_uio": + //Setting up with uio mountpoints + uioMountPoint := path.Join(uioMountPath, dpdkDeviceName, "/device") + return []pluginapi.Mount{ + { + HostPath: uioMountPoint, + ContainerPath: uioMountPath, + }, + }, nil + case "vfio-pci": + //No mountpoint for vfio needs to be populated + return nil, nil + } + + return nil, errors.New("Unknown DPDK driver") +} + +func (dp *DevicePlugin) getDeviceID(pciAddr string) (string, error) { + devID, err := ioutil.ReadFile(path.Join(dp.pciDeviceDir, pciAddr, "device")) + if err != nil { + return "", errors.Wrapf(err, "Cannot obtain ID for the device %s", pciAddr) + } + + return strings.TrimPrefix(string(bytes.TrimSpace(devID)), "0x"), nil +} + +// bindDevice unbinds given device from kernel driver and binds to DPDK driver +func (dp *DevicePlugin) bindDevice(id string) error { + devicePCIAddr := "0000:" + id + unbindDevicePath := path.Join(dp.pciDeviceDir, devicePCIAddr, driverUnbindSuffix) + + // Unbind from the kernel driver + err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0644) + if err != nil { + return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id) + + } + vfdevID, err := dp.getDeviceID(devicePCIAddr) + if err != nil { + return err + } + bindDevicePath := path.Join(dp.pciDriverDir, dp.dpdkDriver, newIDSuffix) + //Bind to the the dpdk driver + err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0644) + if err != nil { + return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id) + } + return nil +} + +func isValidKerneDriver(kernelvfDriver string) bool { + switch kernelvfDriver { + case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf": + return true + } + return false +} + +func isValidDpdkDeviceDriver(dpdkDriver string) bool { + switch dpdkDriver { + case "igb_uio", "vfio-pci": + return true + } + return false +} +func isValidVfDeviceID(vfDevID string) bool { + switch vfDevID { + case "0442", "0443", "37c9", "19e3": + return true + } + return false +} + +// PostAllocate implements PostAllocator interface for vfio based QAT plugin. +func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error { + tempMap := make(map[string]string) + for _, cresp := range response.ContainerResponses { + counter := 0 + for k := range cresp.Envs { + tempMap[strings.Join([]string{"QAT", strconv.Itoa(counter)}, "")] = cresp.Envs[k] + counter++ + } + cresp.Envs = tempMap + } + return nil +} + +func (dp *DevicePlugin) scan() (deviceplugin.DeviceTree, error) { + devTree := deviceplugin.NewDeviceTree() + n := 0 + for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { + files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) + if err != nil { + fmt.Printf("Can't read sysfs for driver as Driver %s is not available: Skipping\n", driver) + continue + } + + for _, file := range files { + if !strings.HasPrefix(file.Name(), "0000:") { + continue + } + vfdevID, err := dp.getDeviceID(file.Name()) + if err != nil { + return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name()) + } + if !isValidVfDeviceID(vfdevID) { + continue + } + n = n + 1 // increment after all junk got filtered out + + if n > dp.maxDevices { + break + } + + vfpciaddr := strings.TrimPrefix(file.Name(), "0000:") + + // initialize newly found devices which aren't bound to DPDK driver yet + if driver != dp.dpdkDriver { + err = dp.bindDevice(vfpciaddr) + if err != nil { + return nil, err + } + } + + devNodes, err := dp.getDpdkDeviceSpecs(vfpciaddr) + if err != nil { + return nil, err + } + devMounts, err := dp.getDpdkMounts(vfpciaddr) + if err != nil { + return nil, err + } + + devinfo := deviceplugin.DeviceInfo{ + State: pluginapi.Healthy, + Nodes: devNodes, + Mounts: devMounts, + Envs: map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), + }, + } + + devTree.AddDevice("generic", vfpciaddr, devinfo) + } + } + + return devTree, nil +} diff --git a/cmd/qat_plugin/qat_plugin_test.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go similarity index 99% rename from cmd/qat_plugin/qat_plugin_test.go rename to cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go index d98fd82a..28edc977 100644 --- a/cmd/qat_plugin/qat_plugin_test.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package dpdkdrv import ( "fmt" @@ -339,7 +339,7 @@ func TestScanPrivate(t *testing.T) { t.Fatalf("%+v", err) } - dp := &devicePlugin{ + dp := &DevicePlugin{ maxDevices: tt.maxDevNum, pciDriverDir: pciDrvDir, pciDeviceDir: pciDevDir, @@ -394,7 +394,7 @@ func TestPostAllocate(t *testing.T) { "03:04.3": {}, "03:04.4": {}, } - dp := &devicePlugin{} + dp := &DevicePlugin{} dp.PostAllocate(response) if len(response.ContainerResponses[0].Envs) != 4 { t.Fatal("Set wrong number of Environment Variables") diff --git a/cmd/qat_plugin/kerneldrv/kerneldrv.go b/cmd/qat_plugin/kerneldrv/kerneldrv.go new file mode 100644 index 00000000..5c2e09a9 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/kerneldrv.go @@ -0,0 +1,337 @@ +// Copyright 2018 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kerneldrv + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/go-ini/ini" + "github.com/pkg/errors" + + pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + utilsexec "k8s.io/utils/exec" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" +) + +const ( + namespace = "qat.intel.com" +) + +var ( + adfCtlRegex = regexp.MustCompile(`qat_(?P[[:alnum:]]+) - type: (?P[[:alnum:]]+), .* bsf: ([0-9a-f]{4}:)?(?P[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]), .* state: (?P[[:alpha:]]+)$`) +) + +type endpoint struct { + id string + processes int +} + +type section struct { + endpoints []endpoint + cryptoEngines int + compressionEngines int + pinned bool +} + +type device struct { + id string + devtype string + bsf string +} + +type driverConfig map[string]section + +func newDeviceSpec(devPath string) pluginapi.DeviceSpec { + return pluginapi.DeviceSpec{ + HostPath: devPath, + ContainerPath: devPath, + Permissions: "rw", + } +} + +func getDevTree(sysfs string, qatDevs []device, config map[string]section) (dpapi.DeviceTree, error) { + devTree := dpapi.NewDeviceTree() + + devs := []pluginapi.DeviceSpec{ + newDeviceSpec("/dev/qat_adf_ctl"), + newDeviceSpec("/dev/qat_dev_processes"), + newDeviceSpec("/dev/usdm_drv"), + } + for _, qatDev := range qatDevs { + uiodevs, err := getUIODevices(sysfs, qatDev.devtype, qatDev.bsf) + if err != nil { + return nil, err + } + for _, uiodev := range uiodevs { + devs = append(devs, newDeviceSpec(filepath.Join("/dev/", uiodev))) + } + } + + uniqID := 0 + for sname, svalue := range config { + var devType string + + devType = fmt.Sprintf("cy%d_dc%d", svalue.cryptoEngines, svalue.compressionEngines) + for _, ep := range svalue.endpoints { + for i := 0; i < ep.processes; i++ { + devTree.AddDevice(devType, fmt.Sprintf("%s_%s_%d", sname, ep.id, i), dpapi.DeviceInfo{ + State: pluginapi.Healthy, + Nodes: devs, + Envs: map[string]string{ + fmt.Sprintf("QAT_SECTION_NAME_%s_%d", devType, uniqID): sname, + // This env variable may get overriden if a container requests more than one QAT process. + // But we keep this code since the majority of pod workloads run only one QAT process. + // The rest should use QAT_SECTION_NAME_XXX variables. + "QAT_SECTION_NAME": sname, + }, + }) + uniqID++ + } + + if !svalue.pinned { + break + } + } + } + + return devTree, nil +} + +// DevicePlugin represents QAT plugin exploiting kernel driver. +type DevicePlugin struct { + execer utilsexec.Interface + configDir string +} + +// NewDevicePlugin returns new instance of kernel based QAT plugin. +func NewDevicePlugin() *DevicePlugin { + return newDevicePlugin("/etc", utilsexec.New()) +} + +func newDevicePlugin(configDir string, execer utilsexec.Interface) *DevicePlugin { + return &DevicePlugin{ + execer: execer, + configDir: configDir, + } +} + +func (dp *DevicePlugin) getOnlineDevices() ([]device, error) { + outputBytes, err := dp.execer.Command("adf_ctl", "status").CombinedOutput() + if err != nil { + return nil, errors.Wrapf(err, "Can't get driver status") + } + + devices := []device{} + for _, line := range strings.Split(string(outputBytes[:]), "\n") { + matches := adfCtlRegex.FindStringSubmatch(line) + if len(matches) == 0 { + continue + } + + // Ignore devices which are down. + if matches[5] != "up" { + continue + } + + devices = append(devices, device{ + id: matches[1], + devtype: matches[2], + bsf: matches[4], + }) + } + + return devices, nil +} + +func getUIODeviceListPath(sysfs, devtype, bsf string) string { + return filepath.Join(sysfs, "bus", "pci", "drivers", devtype, "0000:"+bsf, "uio") +} + +func getUIODevices(sysfs, devtype, bsf string) ([]string, error) { + sysfsDir := getUIODeviceListPath(sysfs, devtype, bsf) + debug.Print("Path to uio devices:", sysfsDir) + + devFiles, err := ioutil.ReadDir(sysfsDir) + if err != nil { + return nil, errors.Wrapf(err, "Can't read %s", sysfsDir) + } + + if len(devFiles) == 0 { + fmt.Println("WARNING: no uio devices listed in", sysfsDir) + } + + devices := []string{} + for _, devFile := range devFiles { + devices = append(devices, devFile.Name()) + } + + return devices, nil +} + +func (dp *DevicePlugin) parseConfigs(devices []device) (map[string]section, error) { + devNum := 0 + drvConfig := make(driverConfig) + for _, dev := range devices { + // Parse the configuration. + config, err := ini.Load(filepath.Join(dp.configDir, fmt.Sprintf("%s_%s.conf", dev.devtype, dev.id))) + if err != nil { + return nil, errors.Wrap(err, "failed to parse device config") + } + devNum++ + + for _, section := range config.Sections() { + if section.Name() == "GENERAL" || section.Name() == "KERNEL" || section.Name() == "KERNEL_QAT" || section.Name() == ini.DEFAULT_SECTION { + continue + } + debug.Print(section.Name()) + if err := drvConfig.update(dev.id, section); err != nil { + return nil, err + } + } + + } + + // check if the number of sections with LimitDevAccess=1 is equal to the number of endpoints + for sname, svalue := range drvConfig { + if svalue.pinned && len(svalue.endpoints) != devNum { + return nil, errors.Errorf("Section [%s] must be defined for all QAT devices since it contains LimitDevAccess=1", sname) + } + } + + return drvConfig, nil +} + +func (drvConfig driverConfig) update(devID string, iniSection *ini.Section) error { + numProcesses, err := iniSection.Key("NumProcesses").Int() + if err != nil { + return errors.Wrapf(err, "Can't parse NumProcesses in %s", iniSection.Name()) + } + cryptoEngines, err := iniSection.Key("NumberCyInstances").Int() + if err != nil { + return errors.Wrapf(err, "Can't parse NumberCyInstances in %s", iniSection.Name()) + } + compressionEngines, err := iniSection.Key("NumberDcInstances").Int() + if err != nil { + return errors.Wrapf(err, "Can't parse NumberDcInstances in %s", iniSection.Name()) + } + pinned := false + if limitDevAccessKey, err := iniSection.GetKey("LimitDevAccess"); err == nil { + limitDevAccess, err := limitDevAccessKey.Bool() + if err != nil { + return errors.Wrapf(err, "Can't parse LimitDevAccess in %s", iniSection.Name()) + } + + if limitDevAccess { + pinned = true + } + } + + if old, ok := drvConfig[iniSection.Name()]; ok { + // first check the sections are consistent across endpoints + if old.pinned != pinned { + return errors.Errorf("Value of LimitDevAccess must be consistent across all devices in %s", iniSection.Name()) + } + if !pinned && old.endpoints[0].processes != numProcesses { + return errors.Errorf("For not pinned section \"%s\" NumProcesses must be equal for all devices", iniSection.Name()) + } + if old.cryptoEngines != cryptoEngines || old.compressionEngines != compressionEngines { + return errors.Errorf("NumberCyInstances and NumberDcInstances must be consistent across all devices in %s", iniSection.Name()) + } + + // then add a new endpoint to the section + old.endpoints = append(old.endpoints, endpoint{ + id: devID, + processes: numProcesses, + }) + drvConfig[iniSection.Name()] = old + } else { + drvConfig[iniSection.Name()] = section{ + endpoints: []endpoint{ + { + id: devID, + processes: numProcesses, + }, + }, + cryptoEngines: cryptoEngines, + compressionEngines: compressionEngines, + pinned: pinned, + } + } + + return nil +} + +// Scan implements Scanner interface for kernel based QAT plugin. +func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error { + for { + devices, err := dp.getOnlineDevices() + if err != nil { + return err + } + + driverConfig, err := dp.parseConfigs(devices) + if err != nil { + return err + } + + devTree, err := getDevTree("/sys", devices, driverConfig) + if err != nil { + return err + } + + notifier.Notify(devTree) + + time.Sleep(5 * time.Second) + } +} + +// PostAllocate implements PostAllocator interface for kernel based QAT plugin. +func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error { + for _, containerResponse := range response.GetContainerResponses() { + envsToDelete := []string{} + envsToAdd := make(map[string]string) + counter := 0 + for key, value := range containerResponse.Envs { + if !strings.HasPrefix(key, "QAT_SECTION_NAME_") { + continue + } + parts := strings.Split(key, "_") + if len(parts) != 6 { + return errors.Errorf("Wrong format of env variable name %s", key) + } + prefix := strings.Join(parts[0:5], "_") + envsToDelete = append(envsToDelete, key) + envsToAdd[fmt.Sprintf("%s_%d", prefix, counter)] = value + counter++ + } + + for _, key := range envsToDelete { + delete(containerResponse.Envs, key) + } + + for key, value := range envsToAdd { + containerResponse.Envs[key] = value + } + } + + return nil +} diff --git a/cmd/qat_plugin/kerneldrv/kerneldrv_test.go b/cmd/qat_plugin/kerneldrv/kerneldrv_test.go new file mode 100644 index 00000000..92e69430 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/kerneldrv_test.go @@ -0,0 +1,370 @@ +// Copyright 2018 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kerneldrv + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "testing" + "time" + + pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" +) + +const ( + adfCtlOutput = `Checking status of all devices. +There is 3 QAT acceleration device(s) in the system: + qat_dev0 - type: c6xx, inst_id: 0, node_id: 0, bsf: 0000:3b:00.0, #accel: 5 #engines: 10 state: up + qat_dev1 - type: c6xx, inst_id: 1, node_id: 0, bsf: 0000:3d:00.0, #accel: 5 #engines: 10 state: up + qat_dev2 - type: c6xx, inst_id: 2, node_id: 3, bsf: 0000:d8:00.0, #accel: 5 #engines: 10 state: up +` + adfCtlOutputOneDown = `Checking status of all devices. +There is 3 QAT acceleration device(s) in the system: + qat_dev0 - type: c6xx, inst_id: 0, node_id: 0, bsf: 3b:00.0, #accel: 5 #engines: 10 state: up + qat_dev1 - type: c6xx, inst_id: 1, node_id: 0, bsf: 3d:00.0, #accel: 5 #engines: 10 state: down + qat_dev2 - type: c6xx, inst_id: 2, node_id: 3, bsf: d8:00.0, #accel: 5 #engines: 10 state: up +` +) + +func init() { + debug.Activate() +} + +func TestGetOnlineDevices(t *testing.T) { + tcases := []struct { + name string + adfCtlOutput string + adfCtlError error + expectedDevNum int + expectedErr bool + }{ + { + name: "all is good", + adfCtlOutput: adfCtlOutput, + expectedDevNum: 3, + }, + { + name: "one device is down", + adfCtlOutput: adfCtlOutputOneDown, + expectedDevNum: 2, + }, + { + name: "adf_ctl fails to run", + adfCtlError: errors.New("fake error"), + expectedErr: true, + }, + } + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ + func() ([]byte, error) { + return []byte(tt.adfCtlOutput), tt.adfCtlError + }, + }, + } + execer := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { + return fakeexec.InitFakeCmd(&fcmd, cmd, args...) + }, + }, + } + dp := &DevicePlugin{ + execer: &execer, + } + devices, err := dp.getOnlineDevices() + if tt.expectedErr && err == nil { + t.Error("Expected error hasn't been triggered") + } + if !tt.expectedErr && err != nil { + t.Errorf("Unexpected error: %+v", err) + } + if len(devices) != tt.expectedDevNum { + t.Errorf("Wrong number of device detected: %d instead of %d", len(devices), tt.expectedDevNum) + } + }) + } +} + +func TestGetUIODevices(t *testing.T) { + tcases := []struct { + name string + devType string + bsf string + expectedErr bool + uiodevs []string + }{ + { + name: "can't read sysfs", + devType: "faketype", + expectedErr: true, + }, + { + name: "all is good", + devType: "c6xx", + uiodevs: []string{"uio0", "uio1"}, + bsf: "da:00.0", + }, + } + for tnum, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + var err error + tmpdir := fmt.Sprintf("/tmp/qatplugin-getUIODevices-%d-%d", time.Now().Unix(), tnum) + sysfs := filepath.Join(tmpdir, "sys") + + for _, uiodev := range tt.uiodevs { + err = os.MkdirAll(filepath.Join(getUIODeviceListPath(sysfs, tt.devType, tt.bsf), uiodev), 0755) + if err != nil { + t.Fatal(err) + } + } + devs, err := getUIODevices(sysfs, tt.devType, tt.bsf) + if tt.expectedErr && err == nil { + t.Error("Expected error hasn't been triggered") + } + if !tt.expectedErr && err != nil { + t.Errorf("Unexpected error: %+v", err) + } + sort.Strings(tt.uiodevs) + sort.Strings(devs) + if tt.uiodevs != nil && !reflect.DeepEqual(devs, tt.uiodevs) { + t.Error("Unexpected devices: ", devs) + } + + err = os.RemoveAll(tmpdir) + if err != nil { + t.Fatal(err) + } + }) + } +} + +func TestParseConfigs(t *testing.T) { + qatdevs := []device{ + { + id: "dev0", + devtype: "c6xx", + }, + { + id: "dev1", + devtype: "c6xx", + }, + { + id: "dev2", + devtype: "c6xx", + }, + } + tcases := []struct { + name string + testData string + expectedErr bool + }{ + { + name: "All is good", + testData: "all_is_good", + }, + { + name: "Missing section with LinitDevAccess=1", + testData: "missing_pinned_section", + expectedErr: true, + }, + { + name: "Can't parse NumProcesses", + testData: "cant_parse_num_processes", + expectedErr: true, + }, + { + name: "Inconsistent LimitDevAccess", + testData: "inconsistent_limitdev", + expectedErr: true, + }, + } + for _, tt := range tcases { + dp := &DevicePlugin{ + configDir: "./test_data/" + tt.testData, + } + _, err := dp.parseConfigs(qatdevs) + if tt.expectedErr && err == nil { + t.Errorf("Test case '%s': expected error hasn't been triggered", tt.name) + } + if !tt.expectedErr && err != nil { + t.Errorf("Test case '%s': Unexpected error: %+v", tt.name, err) + } + } +} + +func TestGetDevTree(t *testing.T) { + tmpdir := fmt.Sprintf("/tmp/qatplugin-getDevTree-%d", time.Now().Unix()) + tcases := []struct { + name string + sysfs string + uiodevs map[string][]string + qatdevs []device + config map[string]section + expectedErr bool + }{ + { + name: "All is good", + sysfs: "sys", + uiodevs: map[string][]string{ + "da:00.0": {"uio4", "uio5"}, + }, + qatdevs: []device{ + { + id: "dev0", + devtype: "c6xx", + bsf: "da:00.0", + }, + }, + config: map[string]section{ + "TESTSHIM": { + endpoints: []endpoint{ + { + id: "dev0", + processes: 2, + }, + }, + }, + "TESTSHIM2": { + endpoints: []endpoint{ + { + id: "dev0", + processes: 2, + }, + }, + }, + "TESTPINNED": { + endpoints: []endpoint{ + { + id: "dev0", + processes: 2, + }, + }, + pinned: true, + }, + }, + }, + { + name: "Wrong devfs", + sysfs: "wrongdev", + qatdevs: []device{ + { + id: "dev0", + devtype: "c6xx", + bsf: "da:00.0", + }, + }, + expectedErr: true, + }, + } + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + var err error + + sysfs := filepath.Join(tmpdir, "sys") + err = os.MkdirAll(sysfs, 0755) + if err != nil { + t.Fatal(err) + } + + for _, qatdev := range tt.qatdevs { + for _, uiodev := range tt.uiodevs[qatdev.bsf] { + err = os.MkdirAll(filepath.Join(getUIODeviceListPath(sysfs, qatdev.devtype, qatdev.bsf), uiodev), 0755) + if err != nil { + t.Fatal(err) + } + + } + } + + _, err = getDevTree(path.Join(tmpdir, tt.sysfs), tt.qatdevs, tt.config) + if tt.expectedErr && err == nil { + t.Errorf("Test case '%s': expected error hasn't been triggered", tt.name) + } + if !tt.expectedErr && err != nil { + t.Errorf("Test case '%s': Unexpected error: %+v", tt.name, err) + } + + err = os.RemoveAll(tmpdir) + if err != nil { + t.Fatal(err) + } + }) + } +} + +func TestPostAllocate(t *testing.T) { + tcases := []struct { + name string + envs map[string]string + expectedEnvs []string + expectedErr bool + }{ + { + name: "All is good", + envs: map[string]string{ + "SOMEVAR": "some value", + "QAT_SECTION_NAME_cy1_dc0_15": "TESTSHIM", + "QAT_SECTION_NAME_cy1_dc0_32": "TESTSHIM2", + }, + expectedEnvs: []string{ + "SOMEVAR", + "QAT_SECTION_NAME_cy1_dc0_0", + "QAT_SECTION_NAME_cy1_dc0_1", + }, + }, + { + name: "Wrong env variable name format", + envs: map[string]string{ + "QAT_SECTION_NAME_JUSTWRONG": "some value", + }, + expectedErr: true, + }, + } + for _, tc := range tcases { + response := new(pluginapi.AllocateResponse) + cresp := new(pluginapi.ContainerAllocateResponse) + cresp.Envs = tc.envs + response.ContainerResponses = append(response.ContainerResponses, cresp) + + dp := &DevicePlugin{} + + err := dp.PostAllocate(response) + + for _, key := range tc.expectedEnvs { + if _, ok := cresp.Envs[key]; !ok { + t.Errorf("Test case '%s': expcted env variable '%s' is missing", tc.name, key) + } + } + + if tc.expectedErr && err == nil { + t.Errorf("Test case '%s': expected error hasn't been triggered", tc.name) + } + if !tc.expectedErr && err != nil { + t.Errorf("Test case '%s': Unexpected error: %+v", tc.name, err) + } + debug.Print(response) + } +} diff --git a/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev0.conf b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev0.conf new file mode 100644 index 00000000..e63b5950 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev0.conf @@ -0,0 +1,205 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 1 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 2 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 3 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 4 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 5 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 6 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 1 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 2 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[SHIM2] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev1.conf b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev1.conf new file mode 100644 index 00000000..df92e1d2 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev1.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 9 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 10 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 11 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 12 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 13 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 14 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 9 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 10 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev2.conf b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev2.conf new file mode 100644 index 00000000..d0779e5c --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/all_is_good/c6xx_dev2.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 17 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 18 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 19 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 20 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 21 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 22 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 17 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 18 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/cant_parse_num_processes/c6xx_dev0.conf b/cmd/qat_plugin/kerneldrv/test_data/cant_parse_num_processes/c6xx_dev0.conf new file mode 100644 index 00000000..a33d7a6b --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/cant_parse_num_processes/c6xx_dev0.conf @@ -0,0 +1,5 @@ +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = this is error +LimitDevAccess = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev0.conf b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev0.conf new file mode 100644 index 00000000..39de5816 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev0.conf @@ -0,0 +1,205 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 1 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 1 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 2 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 3 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 4 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 5 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 6 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 1 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 2 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[SHIM2] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev1.conf b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev1.conf new file mode 100644 index 00000000..df92e1d2 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev1.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 9 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 10 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 11 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 12 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 13 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 14 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 9 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 10 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev2.conf b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev2.conf new file mode 100644 index 00000000..d0779e5c --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/inconsistent_limitdev/c6xx_dev2.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 17 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 18 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 19 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 20 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 21 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 22 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 17 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 18 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev0.conf b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev0.conf new file mode 100644 index 00000000..3047702b --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev0.conf @@ -0,0 +1,195 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 1 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 2 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 3 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 4 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 5 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 6 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 1 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 2 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[SHIM2] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + diff --git a/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev1.conf b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev1.conf new file mode 100644 index 00000000..df92e1d2 --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev1.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 9 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 10 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 11 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 12 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 13 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 14 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 9 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 10 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev2.conf b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev2.conf new file mode 100644 index 00000000..d0779e5c --- /dev/null +++ b/cmd/qat_plugin/kerneldrv/test_data/missing_pinned_section/c6xx_dev2.conf @@ -0,0 +1,193 @@ +################################################################ +# This file is provided under a dual BSD/GPLv2 license. When using or +# redistributing this file, you may do so under either license. +# +# GPL LICENSE SUMMARY +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +# The full GNU General Public License is included in this distribution +# in the file called LICENSE.GPL. +# +# Contact Information: +# Intel Corporation +# +# BSD LICENSE +# +# Copyright(c) 2007-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# version: QAT1.7.L.4.2.0-00012 +################################################################ +[GENERAL] +ServicesEnabled = cy;dc + +ConfigVersion = 2 + +#Default values for number of concurrent requests*/ +CyNumConcurrentSymRequests = 512 +CyNumConcurrentAsymRequests = 64 + +#Statistics, valid values: 1,0 +statsGeneral = 1 +statsDh = 1 +statsDrbg = 1 +statsDsa = 1 +statsEcc = 1 +statsKeyGen = 1 +statsDc = 1 +statsLn = 1 +statsPrime = 1 +statsRsa = 1 +statsSym = 1 +KptEnabled = 0 + +# Disable public key crypto and prime number +# services by specifying a value of 1 (default is 0) +PkeServiceDisabled = 0 + +# Specify size of intermediate buffers for which to +# allocate on-chip buffers. Legal values are 32 and +# 64 (default is 64). Specify 32 to optimize for +# compressing buffers <=32KB in size. +DcIntermediateBufferSizeInKB = 64 + +# This flag is to enable device auto reset on heartbeat error +AutoResetOnError = 0 + +############################################## +# Kernel Instances Section +############################################## +[KERNEL] +NumberCyInstances = 1 +NumberDcInstances = 1 + +# Crypto - Kernel instance #0 +Cy0Name = "IPSec0" +Cy0IsPolled = 0 +Cy0CoreAffinity = 0 + +# Data Compression - Kernel instance #0 +Dc0Name = "IPComp0" +Dc0IsPolled = 0 +Dc0CoreAffinity = 0 + +############################################## +# User Process Instance Section +############################################## +[SSL] +NumberCyInstances = 6 +NumberDcInstances = 2 +NumProcesses = 1 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "SSL0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 17 + +# Crypto - User instance #1 +Cy1Name = "SSL1" +Cy1IsPolled = 1 +# List of core affinities +Cy1CoreAffinity = 18 + +# Crypto - User instance #2 +Cy2Name = "SSL2" +Cy2IsPolled = 1 +# List of core affinities +Cy2CoreAffinity = 19 + +# Crypto - User instance #3 +Cy3Name = "SSL3" +Cy3IsPolled = 1 +# List of core affinities +Cy3CoreAffinity = 20 + +# Crypto - User instance #4 +Cy4Name = "SSL4" +Cy4IsPolled = 1 +# List of core affinities +Cy4CoreAffinity = 21 + +# Crypto - User instance #5 +Cy5Name = "SSL5" +Cy5IsPolled = 1 +# List of core affinities +Cy5CoreAffinity = 22 + + +# Data Compression - User instance #0 +Dc0Name = "Dc0" +Dc0IsPolled = 1 +# List of core affinities +Dc0CoreAffinity = 17 + +# Data Compression - User instance #1 +Dc1Name = "Dc1" +Dc1IsPolled = 1 +# List of core affinities +Dc1CoreAffinity = 18 + +[SHIM] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 2 +LimitDevAccess = 0 + +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 + +[PINNED] +NumberCyInstances = 1 +NumberDcInstances = 0 +NumProcesses = 4 +LimitDevAccess = 1 +# Crypto - User instance #0 +Cy0Name = "UserCY0" +Cy0IsPolled = 1 +# List of core affinities +Cy0CoreAffinity = 0 diff --git a/cmd/qat_plugin/qat_plugin.go b/cmd/qat_plugin/qat_plugin.go index 5a03616b..bcdd2a2f 100644 --- a/cmd/qat_plugin/qat_plugin.go +++ b/cmd/qat_plugin/qat_plugin.go @@ -15,323 +15,52 @@ package main import ( - "bytes" "flag" "fmt" - "io/ioutil" "os" - "path" - "path/filepath" - "strconv" - "strings" - "time" "github.com/pkg/errors" - pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" - + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/qat_plugin/dpdkdrv" + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/qat_plugin/kerneldrv" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/debug" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ) const ( - uioDevicePath = "/dev" - vfioDevicePath = "/dev/vfio" - uioMountPath = "/sys/class/uio" - pciDeviceDirectory = "/sys/bus/pci/devices" - pciDriverDirectory = "/sys/bus/pci/drivers" - uioSuffix = "uio" - iommuGroupSuffix = "iommu_group" - newIDSuffix = "new_id" - driverUnbindSuffix = "driver/unbind" - vendorPrefix = "8086 " - namespace = "qat.intel.com" ) -type devicePlugin struct { - maxDevices int - pciDriverDir string - pciDeviceDir string - kernelVfDrivers []string - dpdkDriver string -} - -func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVfDrivers []string, dpdkDriver string) *devicePlugin { - return &devicePlugin{ - maxDevices: maxDevices, - pciDriverDir: pciDriverDir, - pciDeviceDir: pciDeviceDir, - kernelVfDrivers: kernelVfDrivers, - dpdkDriver: dpdkDriver, - } -} - -func (dp *devicePlugin) Scan(notifier deviceplugin.Notifier) error { - for { - devTree, err := dp.scan() - if err != nil { - return err - } - - notifier.Notify(devTree) - - time.Sleep(5 * time.Second) - } -} - -func (dp *devicePlugin) getDpdkDevice(id string) (string, error) { - - devicePCIAdd := "0000:" + id - switch dp.dpdkDriver { - // TODO: case "pci-generic" and "kernel": - case "igb_uio": - uioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, uioSuffix) - files, err := ioutil.ReadDir(uioDirPath) - if err != nil { - return "", err - } - if len(files) == 0 { - return "", errors.New("No devices found") - } - return files[0].Name(), nil - - case "vfio-pci": - vfioDirPath := path.Join(dp.pciDeviceDir, devicePCIAdd, iommuGroupSuffix) - group, err := filepath.EvalSymlinks(vfioDirPath) - if err != nil { - return "", errors.WithStack(err) - } - s := path.Base(group) - fmt.Printf("The vfio device group detected is %v\n", s) - return s, nil - } - - return "", errors.New("Unknown DPDK driver") -} - -func (dp *devicePlugin) getDpdkDeviceSpecs(id string) ([]pluginapi.DeviceSpec, error) { - dpdkDeviceName, err := dp.getDpdkDevice(id) - if err != nil { - return nil, err - } - fmt.Printf("%s device: corresponding DPDK device detected is %s\n", id, dpdkDeviceName) - - switch dp.dpdkDriver { - // TODO: case "pci-generic" and "kernel": - case "igb_uio": - //Setting up with uio - uioDev := path.Join(uioDevicePath, dpdkDeviceName) - return []pluginapi.DeviceSpec{ - { - HostPath: uioDev, - ContainerPath: uioDev, - Permissions: "rw", - }, - }, nil - case "vfio-pci": - //Setting up with vfio - vfioDev1 := path.Join(vfioDevicePath, dpdkDeviceName) - vfioDev2 := path.Join(vfioDevicePath, "/vfio") - return []pluginapi.DeviceSpec{ - { - HostPath: vfioDev1, - ContainerPath: vfioDev1, - Permissions: "rw", - }, - { - HostPath: vfioDev2, - ContainerPath: vfioDev2, - Permissions: "rw", - }, - }, nil - } - - return nil, errors.New("Unknown DPDK driver") -} - -func (dp *devicePlugin) getDpdkMounts(id string) ([]pluginapi.Mount, error) { - dpdkDeviceName, err := dp.getDpdkDevice(id) - if err != nil { - return nil, err - } - - switch dp.dpdkDriver { - case "igb_uio": - //Setting up with uio mountpoints - uioMountPoint := path.Join(uioMountPath, dpdkDeviceName, "/device") - return []pluginapi.Mount{ - { - HostPath: uioMountPoint, - ContainerPath: uioMountPath, - }, - }, nil - case "vfio-pci": - //No mountpoint for vfio needs to be populated - return nil, nil - } - - return nil, errors.New("Unknown DPDK driver") -} - -func (dp *devicePlugin) getDeviceID(pciAddr string) (string, error) { - devID, err := ioutil.ReadFile(path.Join(dp.pciDeviceDir, pciAddr, "device")) - if err != nil { - return "", errors.Wrapf(err, "Cannot obtain ID for the device %s", pciAddr) - } - - return strings.TrimPrefix(string(bytes.TrimSpace(devID)), "0x"), nil -} - -// bindDevice unbinds given device from kernel driver and binds to DPDK driver -func (dp *devicePlugin) bindDevice(id string) error { - devicePCIAddr := "0000:" + id - unbindDevicePath := path.Join(dp.pciDeviceDir, devicePCIAddr, driverUnbindSuffix) - - // Unbind from the kernel driver - err := ioutil.WriteFile(unbindDevicePath, []byte(devicePCIAddr), 0644) - if err != nil { - return errors.Wrapf(err, "Unbinding from kernel driver failed for the device %s", id) - - } - vfdevID, err := dp.getDeviceID(devicePCIAddr) - if err != nil { - return err - } - bindDevicePath := path.Join(dp.pciDriverDir, dp.dpdkDriver, newIDSuffix) - //Bind to the the dpdk driver - err = ioutil.WriteFile(bindDevicePath, []byte(vendorPrefix+vfdevID), 0644) - if err != nil { - return errors.Wrapf(err, "Binding to the DPDK driver failed for the device %s", id) - } - return nil -} - -func isValidKerneDriver(kernelvfDriver string) bool { - switch kernelvfDriver { - case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf": - return true - } - return false -} - -func isValidDpdkDeviceDriver(dpdkDriver string) bool { - switch dpdkDriver { - case "igb_uio", "vfio-pci": - return true - } - return false -} -func isValidVfDeviceID(vfDevID string) bool { - switch vfDevID { - case "0442", "0443", "37c9", "19e3": - return true - } - return false -} - -func (dp *devicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error { - tempMap := make(map[string]string) - for _, cresp := range response.ContainerResponses { - counter := 0 - for k := range cresp.Envs { - tempMap[strings.Join([]string{"QAT", strconv.Itoa(counter)}, "")] = cresp.Envs[k] - counter++ - } - cresp.Envs = tempMap - } - return nil -} - -func (dp *devicePlugin) scan() (deviceplugin.DeviceTree, error) { - devTree := deviceplugin.NewDeviceTree() - n := 0 - for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { - files, err := ioutil.ReadDir(path.Join(dp.pciDriverDir, driver)) - if err != nil { - fmt.Printf("Can't read sysfs for driver as Driver %s is not available: Skipping\n", driver) - continue - } - - for _, file := range files { - if !strings.HasPrefix(file.Name(), "0000:") { - continue - } - vfdevID, err := dp.getDeviceID(file.Name()) - if err != nil { - return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name()) - } - if !isValidVfDeviceID(vfdevID) { - continue - } - n = n + 1 // increment after all junk got filtered out - - if n > dp.maxDevices { - break - } - - vfpciaddr := strings.TrimPrefix(file.Name(), "0000:") - - // initialize newly found devices which aren't bound to DPDK driver yet - if driver != dp.dpdkDriver { - err = dp.bindDevice(vfpciaddr) - if err != nil { - return nil, err - } - } - - devNodes, err := dp.getDpdkDeviceSpecs(vfpciaddr) - if err != nil { - return nil, err - } - devMounts, err := dp.getDpdkMounts(vfpciaddr) - if err != nil { - return nil, err - } - deviceName := strings.TrimSuffix(namespace, ".intel.com") - - devinfo := deviceplugin.DeviceInfo{ - State: pluginapi.Healthy, - Nodes: devNodes, - Mounts: devMounts, - Envs: map[string]string{ - fmt.Sprintf("%s%d", strings.ToUpper(deviceName), n): file.Name(), - }, - } - - devTree.AddDevice("generic", vfpciaddr, devinfo) - } - } - - return devTree, nil -} - func main() { + var plugin deviceplugin.Scanner + var err error + + debugEnabled := flag.Bool("debug", false, "enable debug output") + mode := flag.String("mode", "dpdk", "plugin mode which can be either dpdk (default) or kernel") + dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device") kernelVfDrivers := flag.String("kernel-vf-drivers", "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx") maxNumDevices := flag.Int("max-num-devices", 32, "maximum number of QAT devices to be provided to the QuickAssist device plugin") - debugEnabled := flag.Bool("debug", false, "enable debug output") flag.Parse() - fmt.Println("QAT device plugin started") if *debugEnabled { debug.Activate() } - if !isValidDpdkDeviceDriver(*dpdkDriver) { - fmt.Println("Wrong DPDK device driver:", *dpdkDriver) + switch *mode { + case "dpdk": + plugin, err = dpdkdrv.NewDevicePlugin(*maxNumDevices, *kernelVfDrivers, *dpdkDriver) + case "kernel": + plugin = kerneldrv.NewDevicePlugin() + default: + err = errors.Errorf("Uknown mode: %s", *mode) + } + if err != nil { + fmt.Println(err.Error()) os.Exit(1) } - kernelDrivers := strings.Split(*kernelVfDrivers, ",") - for _, driver := range kernelDrivers { - if !isValidKerneDriver(driver) { - fmt.Println("Wrong kernel VF driver:", driver) - os.Exit(1) - } - } - - plugin := newDevicePlugin(pciDriverDirectory, pciDeviceDirectory, *maxNumDevices, kernelDrivers, *dpdkDriver) + fmt.Printf("QAT device plugin started in '%s' mode\n", *mode) manager := deviceplugin.NewManager(namespace, plugin) manager.Run() } diff --git a/deployments/qat_plugin/qat_plugin_kernel_mode.yaml b/deployments/qat_plugin/qat_plugin_kernel_mode.yaml new file mode 100644 index 00000000..177a6eac --- /dev/null +++ b/deployments/qat_plugin/qat_plugin_kernel_mode.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-qat-kernel-plugin + labels: + app: intel-qat-kernel-plugin +spec: + selector: + matchLabels: + app: intel-qat-kernel-plugin + template: + metadata: + labels: + app: intel-qat-kernel-plugin + spec: + containers: + - name: intel-qat-kernel-plugin + securityContext: + privileged: true + image: intel-qat-plugin:devel + imagePullPolicy: IfNotPresent + command: ["/usr/bin/intel_qat_device_plugin", "-mode", "kernel"] + volumeMounts: + - name: devfs + mountPath: /dev + - name: etcdir + mountPath: /etc + readOnly: true + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: etcdir + hostPath: + path: /etc + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins + - name: devfs + hostPath: + path: /dev diff --git a/vendor/github.com/go-ini/ini/LICENSE b/vendor/github.com/go-ini/ini/LICENSE new file mode 100644 index 00000000..d361bbcd --- /dev/null +++ b/vendor/github.com/go-ini/ini/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2014 Unknwon + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/go-ini/ini/error.go b/vendor/github.com/go-ini/ini/error.go new file mode 100644 index 00000000..80afe743 --- /dev/null +++ b/vendor/github.com/go-ini/ini/error.go @@ -0,0 +1,32 @@ +// Copyright 2016 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "fmt" +) + +type ErrDelimiterNotFound struct { + Line string +} + +func IsErrDelimiterNotFound(err error) bool { + _, ok := err.(ErrDelimiterNotFound) + return ok +} + +func (err ErrDelimiterNotFound) Error() string { + return fmt.Sprintf("key-value delimiter not found: %s", err.Line) +} diff --git a/vendor/github.com/go-ini/ini/file.go b/vendor/github.com/go-ini/ini/file.go new file mode 100644 index 00000000..0ed0eafd --- /dev/null +++ b/vendor/github.com/go-ini/ini/file.go @@ -0,0 +1,418 @@ +// Copyright 2017 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "sync" +) + +// File represents a combination of a or more INI file(s) in memory. +type File struct { + options LoadOptions + dataSources []dataSource + + // Should make things safe, but sometimes doesn't matter. + BlockMode bool + lock sync.RWMutex + + // To keep data in order. + sectionList []string + // Actual data is stored here. + sections map[string]*Section + + NameMapper + ValueMapper +} + +// newFile initializes File object with given data sources. +func newFile(dataSources []dataSource, opts LoadOptions) *File { + if len(opts.KeyValueDelimiters) == 0 { + opts.KeyValueDelimiters = "=:" + } + return &File{ + BlockMode: true, + dataSources: dataSources, + sections: make(map[string]*Section), + sectionList: make([]string, 0, 10), + options: opts, + } +} + +// Empty returns an empty file object. +func Empty() *File { + // Ignore error here, we sure our data is good. + f, _ := Load([]byte("")) + return f +} + +// NewSection creates a new section. +func (f *File) NewSection(name string) (*Section, error) { + if len(name) == 0 { + return nil, errors.New("error creating new section: empty section name") + } else if f.options.Insensitive && name != DEFAULT_SECTION { + name = strings.ToLower(name) + } + + if f.BlockMode { + f.lock.Lock() + defer f.lock.Unlock() + } + + if inSlice(name, f.sectionList) { + return f.sections[name], nil + } + + f.sectionList = append(f.sectionList, name) + f.sections[name] = newSection(f, name) + return f.sections[name], nil +} + +// NewRawSection creates a new section with an unparseable body. +func (f *File) NewRawSection(name, body string) (*Section, error) { + section, err := f.NewSection(name) + if err != nil { + return nil, err + } + + section.isRawSection = true + section.rawBody = body + return section, nil +} + +// NewSections creates a list of sections. +func (f *File) NewSections(names ...string) (err error) { + for _, name := range names { + if _, err = f.NewSection(name); err != nil { + return err + } + } + return nil +} + +// GetSection returns section by given name. +func (f *File) GetSection(name string) (*Section, error) { + if len(name) == 0 { + name = DEFAULT_SECTION + } + if f.options.Insensitive { + name = strings.ToLower(name) + } + + if f.BlockMode { + f.lock.RLock() + defer f.lock.RUnlock() + } + + sec := f.sections[name] + if sec == nil { + return nil, fmt.Errorf("section '%s' does not exist", name) + } + return sec, nil +} + +// Section assumes named section exists and returns a zero-value when not. +func (f *File) Section(name string) *Section { + sec, err := f.GetSection(name) + if err != nil { + // Note: It's OK here because the only possible error is empty section name, + // but if it's empty, this piece of code won't be executed. + sec, _ = f.NewSection(name) + return sec + } + return sec +} + +// Section returns list of Section. +func (f *File) Sections() []*Section { + if f.BlockMode { + f.lock.RLock() + defer f.lock.RUnlock() + } + + sections := make([]*Section, len(f.sectionList)) + for i, name := range f.sectionList { + sections[i] = f.sections[name] + } + return sections +} + +// ChildSections returns a list of child sections of given section name. +func (f *File) ChildSections(name string) []*Section { + return f.Section(name).ChildSections() +} + +// SectionStrings returns list of section names. +func (f *File) SectionStrings() []string { + list := make([]string, len(f.sectionList)) + copy(list, f.sectionList) + return list +} + +// DeleteSection deletes a section. +func (f *File) DeleteSection(name string) { + if f.BlockMode { + f.lock.Lock() + defer f.lock.Unlock() + } + + if len(name) == 0 { + name = DEFAULT_SECTION + } + + for i, s := range f.sectionList { + if s == name { + f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...) + delete(f.sections, name) + return + } + } +} + +func (f *File) reload(s dataSource) error { + r, err := s.ReadCloser() + if err != nil { + return err + } + defer r.Close() + + return f.parse(r) +} + +// Reload reloads and parses all data sources. +func (f *File) Reload() (err error) { + for _, s := range f.dataSources { + if err = f.reload(s); err != nil { + // In loose mode, we create an empty default section for nonexistent files. + if os.IsNotExist(err) && f.options.Loose { + f.parse(bytes.NewBuffer(nil)) + continue + } + return err + } + } + return nil +} + +// Append appends one or more data sources and reloads automatically. +func (f *File) Append(source interface{}, others ...interface{}) error { + ds, err := parseDataSource(source) + if err != nil { + return err + } + f.dataSources = append(f.dataSources, ds) + for _, s := range others { + ds, err = parseDataSource(s) + if err != nil { + return err + } + f.dataSources = append(f.dataSources, ds) + } + return f.Reload() +} + +func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) { + equalSign := DefaultFormatLeft + "=" + DefaultFormatRight + + if PrettyFormat || PrettyEqual { + equalSign = " = " + } + + // Use buffer to make sure target is safe until finish encoding. + buf := bytes.NewBuffer(nil) + for i, sname := range f.sectionList { + sec := f.Section(sname) + if len(sec.Comment) > 0 { + // Support multiline comments + lines := strings.Split(sec.Comment, LineBreak) + for i := range lines { + if lines[i][0] != '#' && lines[i][0] != ';' { + lines[i] = "; " + lines[i] + } else { + lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:]) + } + + if _, err := buf.WriteString(lines[i] + LineBreak); err != nil { + return nil, err + } + } + } + + if i > 0 || DefaultHeader { + if _, err := buf.WriteString("[" + sname + "]" + LineBreak); err != nil { + return nil, err + } + } else { + // Write nothing if default section is empty + if len(sec.keyList) == 0 { + continue + } + } + + if sec.isRawSection { + if _, err := buf.WriteString(sec.rawBody); err != nil { + return nil, err + } + + if PrettySection { + // Put a line between sections + if _, err := buf.WriteString(LineBreak); err != nil { + return nil, err + } + } + continue + } + + // Count and generate alignment length and buffer spaces using the + // longest key. Keys may be modifed if they contain certain characters so + // we need to take that into account in our calculation. + alignLength := 0 + if PrettyFormat { + for _, kname := range sec.keyList { + keyLength := len(kname) + // First case will surround key by ` and second by """ + if strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters) { + keyLength += 2 + } else if strings.Contains(kname, "`") { + keyLength += 6 + } + + if keyLength > alignLength { + alignLength = keyLength + } + } + } + alignSpaces := bytes.Repeat([]byte(" "), alignLength) + + KEY_LIST: + for _, kname := range sec.keyList { + key := sec.Key(kname) + if len(key.Comment) > 0 { + if len(indent) > 0 && sname != DEFAULT_SECTION { + buf.WriteString(indent) + } + + // Support multiline comments + lines := strings.Split(key.Comment, LineBreak) + for i := range lines { + if lines[i][0] != '#' && lines[i][0] != ';' { + lines[i] = "; " + strings.TrimSpace(lines[i]) + } else { + lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:]) + } + + if _, err := buf.WriteString(lines[i] + LineBreak); err != nil { + return nil, err + } + } + } + + if len(indent) > 0 && sname != DEFAULT_SECTION { + buf.WriteString(indent) + } + + switch { + case key.isAutoIncrement: + kname = "-" + case strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters): + kname = "`" + kname + "`" + case strings.Contains(kname, "`"): + kname = `"""` + kname + `"""` + } + + for _, val := range key.ValueWithShadows() { + if _, err := buf.WriteString(kname); err != nil { + return nil, err + } + + if key.isBooleanType { + if kname != sec.keyList[len(sec.keyList)-1] { + buf.WriteString(LineBreak) + } + continue KEY_LIST + } + + // Write out alignment spaces before "=" sign + if PrettyFormat { + buf.Write(alignSpaces[:alignLength-len(kname)]) + } + + // In case key value contains "\n", "`", "\"", "#" or ";" + if strings.ContainsAny(val, "\n`") { + val = `"""` + val + `"""` + } else if !f.options.IgnoreInlineComment && strings.ContainsAny(val, "#;") { + val = "`" + val + "`" + } + if _, err := buf.WriteString(equalSign + val + LineBreak); err != nil { + return nil, err + } + } + + for _, val := range key.nestedValues { + if _, err := buf.WriteString(indent + " " + val + LineBreak); err != nil { + return nil, err + } + } + } + + if PrettySection { + // Put a line between sections + if _, err := buf.WriteString(LineBreak); err != nil { + return nil, err + } + } + } + + return buf, nil +} + +// WriteToIndent writes content into io.Writer with given indention. +// If PrettyFormat has been set to be true, +// it will align "=" sign with spaces under each section. +func (f *File) WriteToIndent(w io.Writer, indent string) (int64, error) { + buf, err := f.writeToBuffer(indent) + if err != nil { + return 0, err + } + return buf.WriteTo(w) +} + +// WriteTo writes file content into io.Writer. +func (f *File) WriteTo(w io.Writer) (int64, error) { + return f.WriteToIndent(w, "") +} + +// SaveToIndent writes content to file system with given value indention. +func (f *File) SaveToIndent(filename, indent string) error { + // Note: Because we are truncating with os.Create, + // so it's safer to save to a temporary file location and rename afte done. + buf, err := f.writeToBuffer(indent) + if err != nil { + return err + } + + return ioutil.WriteFile(filename, buf.Bytes(), 0666) +} + +// SaveTo writes content to file system. +func (f *File) SaveTo(filename string) error { + return f.SaveToIndent(filename, "") +} diff --git a/vendor/github.com/go-ini/ini/ini.go b/vendor/github.com/go-ini/ini/ini.go new file mode 100644 index 00000000..f827a1ef --- /dev/null +++ b/vendor/github.com/go-ini/ini/ini.go @@ -0,0 +1,219 @@ +// +build go1.6 + +// Copyright 2014 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Package ini provides INI file read and write functionality in Go. +package ini + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "regexp" + "runtime" +) + +const ( + // Name for default section. You can use this constant or the string literal. + // In most of cases, an empty string is all you need to access the section. + DEFAULT_SECTION = "DEFAULT" + + // Maximum allowed depth when recursively substituing variable names. + _DEPTH_VALUES = 99 + _VERSION = "1.42.0" +) + +// Version returns current package version literal. +func Version() string { + return _VERSION +} + +var ( + // Delimiter to determine or compose a new line. + // This variable will be changed to "\r\n" automatically on Windows + // at package init time. + LineBreak = "\n" + + // Place custom spaces when PrettyFormat and PrettyEqual are both disabled + DefaultFormatLeft = "" + DefaultFormatRight = "" + + // Variable regexp pattern: %(variable)s + varPattern = regexp.MustCompile(`%\(([^\)]+)\)s`) + + // Indicate whether to align "=" sign with spaces to produce pretty output + // or reduce all possible spaces for compact format. + PrettyFormat = true + + // Place spaces around "=" sign even when PrettyFormat is false + PrettyEqual = false + + // Explicitly write DEFAULT section header + DefaultHeader = false + + // Indicate whether to put a line between sections + PrettySection = true +) + +func init() { + if runtime.GOOS == "windows" { + LineBreak = "\r\n" + } +} + +func inSlice(str string, s []string) bool { + for _, v := range s { + if str == v { + return true + } + } + return false +} + +// dataSource is an interface that returns object which can be read and closed. +type dataSource interface { + ReadCloser() (io.ReadCloser, error) +} + +// sourceFile represents an object that contains content on the local file system. +type sourceFile struct { + name string +} + +func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) { + return os.Open(s.name) +} + +// sourceData represents an object that contains content in memory. +type sourceData struct { + data []byte +} + +func (s *sourceData) ReadCloser() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(s.data)), nil +} + +// sourceReadCloser represents an input stream with Close method. +type sourceReadCloser struct { + reader io.ReadCloser +} + +func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) { + return s.reader, nil +} + +func parseDataSource(source interface{}) (dataSource, error) { + switch s := source.(type) { + case string: + return sourceFile{s}, nil + case []byte: + return &sourceData{s}, nil + case io.ReadCloser: + return &sourceReadCloser{s}, nil + default: + return nil, fmt.Errorf("error parsing data source: unknown type '%s'", s) + } +} + +type LoadOptions struct { + // Loose indicates whether the parser should ignore nonexistent files or return error. + Loose bool + // Insensitive indicates whether the parser forces all section and key names to lowercase. + Insensitive bool + // IgnoreContinuation indicates whether to ignore continuation lines while parsing. + IgnoreContinuation bool + // IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value. + IgnoreInlineComment bool + // SkipUnrecognizableLines indicates whether to skip unrecognizable lines that do not conform to key/value pairs. + SkipUnrecognizableLines bool + // AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing. + // This type of keys are mostly used in my.cnf. + AllowBooleanKeys bool + // AllowShadows indicates whether to keep track of keys with same name under same section. + AllowShadows bool + // AllowNestedValues indicates whether to allow AWS-like nested values. + // Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values + AllowNestedValues bool + // AllowPythonMultilineValues indicates whether to allow Python-like multi-line values. + // Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure + // Relevant quote: Values can also span multiple lines, as long as they are indented deeper + // than the first line of the value. + AllowPythonMultilineValues bool + // SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value. + // Docs: https://docs.python.org/2/library/configparser.html + // Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names. + // In the latter case, they need to be preceded by a whitespace character to be recognized as a comment. + SpaceBeforeInlineComment bool + // UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format + // when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value" + UnescapeValueDoubleQuotes bool + // UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format + // when value is NOT surrounded by any quotes. + // Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all. + UnescapeValueCommentSymbols bool + // UnparseableSections stores a list of blocks that are allowed with raw content which do not otherwise + // conform to key/value pairs. Specify the names of those blocks here. + UnparseableSections []string + // KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:". + KeyValueDelimiters string + // PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes). + PreserveSurroundedQuote bool +} + +func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) { + sources := make([]dataSource, len(others)+1) + sources[0], err = parseDataSource(source) + if err != nil { + return nil, err + } + for i := range others { + sources[i+1], err = parseDataSource(others[i]) + if err != nil { + return nil, err + } + } + f := newFile(sources, opts) + if err = f.Reload(); err != nil { + return nil, err + } + return f, nil +} + +// Load loads and parses from INI data sources. +// Arguments can be mixed of file name with string type, or raw data in []byte. +// It will return error if list contains nonexistent files. +func Load(source interface{}, others ...interface{}) (*File, error) { + return LoadSources(LoadOptions{}, source, others...) +} + +// LooseLoad has exactly same functionality as Load function +// except it ignores nonexistent files instead of returning error. +func LooseLoad(source interface{}, others ...interface{}) (*File, error) { + return LoadSources(LoadOptions{Loose: true}, source, others...) +} + +// InsensitiveLoad has exactly same functionality as Load function +// except it forces all section and key names to be lowercased. +func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) { + return LoadSources(LoadOptions{Insensitive: true}, source, others...) +} + +// ShadowLoad has exactly same functionality as Load function +// except it allows have shadow keys. +func ShadowLoad(source interface{}, others ...interface{}) (*File, error) { + return LoadSources(LoadOptions{AllowShadows: true}, source, others...) +} diff --git a/vendor/github.com/go-ini/ini/key.go b/vendor/github.com/go-ini/ini/key.go new file mode 100644 index 00000000..0fee0dc7 --- /dev/null +++ b/vendor/github.com/go-ini/ini/key.go @@ -0,0 +1,752 @@ +// Copyright 2014 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// Key represents a key under a section. +type Key struct { + s *Section + Comment string + name string + value string + isAutoIncrement bool + isBooleanType bool + + isShadow bool + shadows []*Key + + nestedValues []string +} + +// newKey simply return a key object with given values. +func newKey(s *Section, name, val string) *Key { + return &Key{ + s: s, + name: name, + value: val, + } +} + +func (k *Key) addShadow(val string) error { + if k.isShadow { + return errors.New("cannot add shadow to another shadow key") + } else if k.isAutoIncrement || k.isBooleanType { + return errors.New("cannot add shadow to auto-increment or boolean key") + } + + shadow := newKey(k.s, k.name, val) + shadow.isShadow = true + k.shadows = append(k.shadows, shadow) + return nil +} + +// AddShadow adds a new shadow key to itself. +func (k *Key) AddShadow(val string) error { + if !k.s.f.options.AllowShadows { + return errors.New("shadow key is not allowed") + } + return k.addShadow(val) +} + +func (k *Key) addNestedValue(val string) error { + if k.isAutoIncrement || k.isBooleanType { + return errors.New("cannot add nested value to auto-increment or boolean key") + } + + k.nestedValues = append(k.nestedValues, val) + return nil +} + +func (k *Key) AddNestedValue(val string) error { + if !k.s.f.options.AllowNestedValues { + return errors.New("nested value is not allowed") + } + return k.addNestedValue(val) +} + +// ValueMapper represents a mapping function for values, e.g. os.ExpandEnv +type ValueMapper func(string) string + +// Name returns name of key. +func (k *Key) Name() string { + return k.name +} + +// Value returns raw value of key for performance purpose. +func (k *Key) Value() string { + return k.value +} + +// ValueWithShadows returns raw values of key and its shadows if any. +func (k *Key) ValueWithShadows() []string { + if len(k.shadows) == 0 { + return []string{k.value} + } + vals := make([]string, len(k.shadows)+1) + vals[0] = k.value + for i := range k.shadows { + vals[i+1] = k.shadows[i].value + } + return vals +} + +// NestedValues returns nested values stored in the key. +// It is possible returned value is nil if no nested values stored in the key. +func (k *Key) NestedValues() []string { + return k.nestedValues +} + +// transformValue takes a raw value and transforms to its final string. +func (k *Key) transformValue(val string) string { + if k.s.f.ValueMapper != nil { + val = k.s.f.ValueMapper(val) + } + + // Fail-fast if no indicate char found for recursive value + if !strings.Contains(val, "%") { + return val + } + for i := 0; i < _DEPTH_VALUES; i++ { + vr := varPattern.FindString(val) + if len(vr) == 0 { + break + } + + // Take off leading '%(' and trailing ')s'. + noption := vr[2 : len(vr)-2] + + // Search in the same section. + nk, err := k.s.GetKey(noption) + if err != nil || k == nk { + // Search again in default section. + nk, _ = k.s.f.Section("").GetKey(noption) + } + + // Substitute by new value and take off leading '%(' and trailing ')s'. + val = strings.Replace(val, vr, nk.value, -1) + } + return val +} + +// String returns string representation of value. +func (k *Key) String() string { + return k.transformValue(k.value) +} + +// Validate accepts a validate function which can +// return modifed result as key value. +func (k *Key) Validate(fn func(string) string) string { + return fn(k.String()) +} + +// parseBool returns the boolean value represented by the string. +// +// It accepts 1, t, T, TRUE, true, True, YES, yes, Yes, y, ON, on, On, +// 0, f, F, FALSE, false, False, NO, no, No, n, OFF, off, Off. +// Any other value returns an error. +func parseBool(str string) (value bool, err error) { + switch str { + case "1", "t", "T", "true", "TRUE", "True", "YES", "yes", "Yes", "y", "ON", "on", "On": + return true, nil + case "0", "f", "F", "false", "FALSE", "False", "NO", "no", "No", "n", "OFF", "off", "Off": + return false, nil + } + return false, fmt.Errorf("parsing \"%s\": invalid syntax", str) +} + +// Bool returns bool type value. +func (k *Key) Bool() (bool, error) { + return parseBool(k.String()) +} + +// Float64 returns float64 type value. +func (k *Key) Float64() (float64, error) { + return strconv.ParseFloat(k.String(), 64) +} + +// Int returns int type value. +func (k *Key) Int() (int, error) { + v, err := strconv.ParseInt(k.String(), 0, 64) + return int(v), err +} + +// Int64 returns int64 type value. +func (k *Key) Int64() (int64, error) { + return strconv.ParseInt(k.String(), 0, 64) +} + +// Uint returns uint type valued. +func (k *Key) Uint() (uint, error) { + u, e := strconv.ParseUint(k.String(), 0, 64) + return uint(u), e +} + +// Uint64 returns uint64 type value. +func (k *Key) Uint64() (uint64, error) { + return strconv.ParseUint(k.String(), 0, 64) +} + +// Duration returns time.Duration type value. +func (k *Key) Duration() (time.Duration, error) { + return time.ParseDuration(k.String()) +} + +// TimeFormat parses with given format and returns time.Time type value. +func (k *Key) TimeFormat(format string) (time.Time, error) { + return time.Parse(format, k.String()) +} + +// Time parses with RFC3339 format and returns time.Time type value. +func (k *Key) Time() (time.Time, error) { + return k.TimeFormat(time.RFC3339) +} + +// MustString returns default value if key value is empty. +func (k *Key) MustString(defaultVal string) string { + val := k.String() + if len(val) == 0 { + k.value = defaultVal + return defaultVal + } + return val +} + +// MustBool always returns value without error, +// it returns false if error occurs. +func (k *Key) MustBool(defaultVal ...bool) bool { + val, err := k.Bool() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatBool(defaultVal[0]) + return defaultVal[0] + } + return val +} + +// MustFloat64 always returns value without error, +// it returns 0.0 if error occurs. +func (k *Key) MustFloat64(defaultVal ...float64) float64 { + val, err := k.Float64() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatFloat(defaultVal[0], 'f', -1, 64) + return defaultVal[0] + } + return val +} + +// MustInt always returns value without error, +// it returns 0 if error occurs. +func (k *Key) MustInt(defaultVal ...int) int { + val, err := k.Int() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatInt(int64(defaultVal[0]), 10) + return defaultVal[0] + } + return val +} + +// MustInt64 always returns value without error, +// it returns 0 if error occurs. +func (k *Key) MustInt64(defaultVal ...int64) int64 { + val, err := k.Int64() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatInt(defaultVal[0], 10) + return defaultVal[0] + } + return val +} + +// MustUint always returns value without error, +// it returns 0 if error occurs. +func (k *Key) MustUint(defaultVal ...uint) uint { + val, err := k.Uint() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatUint(uint64(defaultVal[0]), 10) + return defaultVal[0] + } + return val +} + +// MustUint64 always returns value without error, +// it returns 0 if error occurs. +func (k *Key) MustUint64(defaultVal ...uint64) uint64 { + val, err := k.Uint64() + if len(defaultVal) > 0 && err != nil { + k.value = strconv.FormatUint(defaultVal[0], 10) + return defaultVal[0] + } + return val +} + +// MustDuration always returns value without error, +// it returns zero value if error occurs. +func (k *Key) MustDuration(defaultVal ...time.Duration) time.Duration { + val, err := k.Duration() + if len(defaultVal) > 0 && err != nil { + k.value = defaultVal[0].String() + return defaultVal[0] + } + return val +} + +// MustTimeFormat always parses with given format and returns value without error, +// it returns zero value if error occurs. +func (k *Key) MustTimeFormat(format string, defaultVal ...time.Time) time.Time { + val, err := k.TimeFormat(format) + if len(defaultVal) > 0 && err != nil { + k.value = defaultVal[0].Format(format) + return defaultVal[0] + } + return val +} + +// MustTime always parses with RFC3339 format and returns value without error, +// it returns zero value if error occurs. +func (k *Key) MustTime(defaultVal ...time.Time) time.Time { + return k.MustTimeFormat(time.RFC3339, defaultVal...) +} + +// In always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) In(defaultVal string, candidates []string) string { + val := k.String() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InFloat64 always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InFloat64(defaultVal float64, candidates []float64) float64 { + val := k.MustFloat64() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InInt always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InInt(defaultVal int, candidates []int) int { + val := k.MustInt() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InInt64 always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InInt64(defaultVal int64, candidates []int64) int64 { + val := k.MustInt64() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InUint always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InUint(defaultVal uint, candidates []uint) uint { + val := k.MustUint() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InUint64 always returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InUint64(defaultVal uint64, candidates []uint64) uint64 { + val := k.MustUint64() + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InTimeFormat always parses with given format and returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InTimeFormat(format string, defaultVal time.Time, candidates []time.Time) time.Time { + val := k.MustTimeFormat(format) + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// InTime always parses with RFC3339 format and returns value without error, +// it returns default value if error occurs or doesn't fit into candidates. +func (k *Key) InTime(defaultVal time.Time, candidates []time.Time) time.Time { + return k.InTimeFormat(time.RFC3339, defaultVal, candidates) +} + +// RangeFloat64 checks if value is in given range inclusively, +// and returns default value if it's not. +func (k *Key) RangeFloat64(defaultVal, min, max float64) float64 { + val := k.MustFloat64() + if val < min || val > max { + return defaultVal + } + return val +} + +// RangeInt checks if value is in given range inclusively, +// and returns default value if it's not. +func (k *Key) RangeInt(defaultVal, min, max int) int { + val := k.MustInt() + if val < min || val > max { + return defaultVal + } + return val +} + +// RangeInt64 checks if value is in given range inclusively, +// and returns default value if it's not. +func (k *Key) RangeInt64(defaultVal, min, max int64) int64 { + val := k.MustInt64() + if val < min || val > max { + return defaultVal + } + return val +} + +// RangeTimeFormat checks if value with given format is in given range inclusively, +// and returns default value if it's not. +func (k *Key) RangeTimeFormat(format string, defaultVal, min, max time.Time) time.Time { + val := k.MustTimeFormat(format) + if val.Unix() < min.Unix() || val.Unix() > max.Unix() { + return defaultVal + } + return val +} + +// RangeTime checks if value with RFC3339 format is in given range inclusively, +// and returns default value if it's not. +func (k *Key) RangeTime(defaultVal, min, max time.Time) time.Time { + return k.RangeTimeFormat(time.RFC3339, defaultVal, min, max) +} + +// Strings returns list of string divided by given delimiter. +func (k *Key) Strings(delim string) []string { + str := k.String() + if len(str) == 0 { + return []string{} + } + + runes := []rune(str) + vals := make([]string, 0, 2) + var buf bytes.Buffer + escape := false + idx := 0 + for { + if escape { + escape = false + if runes[idx] != '\\' && !strings.HasPrefix(string(runes[idx:]), delim) { + buf.WriteRune('\\') + } + buf.WriteRune(runes[idx]) + } else { + if runes[idx] == '\\' { + escape = true + } else if strings.HasPrefix(string(runes[idx:]), delim) { + idx += len(delim) - 1 + vals = append(vals, strings.TrimSpace(buf.String())) + buf.Reset() + } else { + buf.WriteRune(runes[idx]) + } + } + idx += 1 + if idx == len(runes) { + break + } + } + + if buf.Len() > 0 { + vals = append(vals, strings.TrimSpace(buf.String())) + } + + return vals +} + +// StringsWithShadows returns list of string divided by given delimiter. +// Shadows will also be appended if any. +func (k *Key) StringsWithShadows(delim string) []string { + vals := k.ValueWithShadows() + results := make([]string, 0, len(vals)*2) + for i := range vals { + if len(vals) == 0 { + continue + } + + results = append(results, strings.Split(vals[i], delim)...) + } + + for i := range results { + results[i] = k.transformValue(strings.TrimSpace(results[i])) + } + return results +} + +// Float64s returns list of float64 divided by given delimiter. Any invalid input will be treated as zero value. +func (k *Key) Float64s(delim string) []float64 { + vals, _ := k.parseFloat64s(k.Strings(delim), true, false) + return vals +} + +// Ints returns list of int divided by given delimiter. Any invalid input will be treated as zero value. +func (k *Key) Ints(delim string) []int { + vals, _ := k.parseInts(k.Strings(delim), true, false) + return vals +} + +// Int64s returns list of int64 divided by given delimiter. Any invalid input will be treated as zero value. +func (k *Key) Int64s(delim string) []int64 { + vals, _ := k.parseInt64s(k.Strings(delim), true, false) + return vals +} + +// Uints returns list of uint divided by given delimiter. Any invalid input will be treated as zero value. +func (k *Key) Uints(delim string) []uint { + vals, _ := k.parseUints(k.Strings(delim), true, false) + return vals +} + +// Uint64s returns list of uint64 divided by given delimiter. Any invalid input will be treated as zero value. +func (k *Key) Uint64s(delim string) []uint64 { + vals, _ := k.parseUint64s(k.Strings(delim), true, false) + return vals +} + +// TimesFormat parses with given format and returns list of time.Time divided by given delimiter. +// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC). +func (k *Key) TimesFormat(format, delim string) []time.Time { + vals, _ := k.parseTimesFormat(format, k.Strings(delim), true, false) + return vals +} + +// Times parses with RFC3339 format and returns list of time.Time divided by given delimiter. +// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC). +func (k *Key) Times(delim string) []time.Time { + return k.TimesFormat(time.RFC3339, delim) +} + +// ValidFloat64s returns list of float64 divided by given delimiter. If some value is not float, then +// it will not be included to result list. +func (k *Key) ValidFloat64s(delim string) []float64 { + vals, _ := k.parseFloat64s(k.Strings(delim), false, false) + return vals +} + +// ValidInts returns list of int divided by given delimiter. If some value is not integer, then it will +// not be included to result list. +func (k *Key) ValidInts(delim string) []int { + vals, _ := k.parseInts(k.Strings(delim), false, false) + return vals +} + +// ValidInt64s returns list of int64 divided by given delimiter. If some value is not 64-bit integer, +// then it will not be included to result list. +func (k *Key) ValidInt64s(delim string) []int64 { + vals, _ := k.parseInt64s(k.Strings(delim), false, false) + return vals +} + +// ValidUints returns list of uint divided by given delimiter. If some value is not unsigned integer, +// then it will not be included to result list. +func (k *Key) ValidUints(delim string) []uint { + vals, _ := k.parseUints(k.Strings(delim), false, false) + return vals +} + +// ValidUint64s returns list of uint64 divided by given delimiter. If some value is not 64-bit unsigned +// integer, then it will not be included to result list. +func (k *Key) ValidUint64s(delim string) []uint64 { + vals, _ := k.parseUint64s(k.Strings(delim), false, false) + return vals +} + +// ValidTimesFormat parses with given format and returns list of time.Time divided by given delimiter. +func (k *Key) ValidTimesFormat(format, delim string) []time.Time { + vals, _ := k.parseTimesFormat(format, k.Strings(delim), false, false) + return vals +} + +// ValidTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter. +func (k *Key) ValidTimes(delim string) []time.Time { + return k.ValidTimesFormat(time.RFC3339, delim) +} + +// StrictFloat64s returns list of float64 divided by given delimiter or error on first invalid input. +func (k *Key) StrictFloat64s(delim string) ([]float64, error) { + return k.parseFloat64s(k.Strings(delim), false, true) +} + +// StrictInts returns list of int divided by given delimiter or error on first invalid input. +func (k *Key) StrictInts(delim string) ([]int, error) { + return k.parseInts(k.Strings(delim), false, true) +} + +// StrictInt64s returns list of int64 divided by given delimiter or error on first invalid input. +func (k *Key) StrictInt64s(delim string) ([]int64, error) { + return k.parseInt64s(k.Strings(delim), false, true) +} + +// StrictUints returns list of uint divided by given delimiter or error on first invalid input. +func (k *Key) StrictUints(delim string) ([]uint, error) { + return k.parseUints(k.Strings(delim), false, true) +} + +// StrictUint64s returns list of uint64 divided by given delimiter or error on first invalid input. +func (k *Key) StrictUint64s(delim string) ([]uint64, error) { + return k.parseUint64s(k.Strings(delim), false, true) +} + +// StrictTimesFormat parses with given format and returns list of time.Time divided by given delimiter +// or error on first invalid input. +func (k *Key) StrictTimesFormat(format, delim string) ([]time.Time, error) { + return k.parseTimesFormat(format, k.Strings(delim), false, true) +} + +// StrictTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter +// or error on first invalid input. +func (k *Key) StrictTimes(delim string) ([]time.Time, error) { + return k.StrictTimesFormat(time.RFC3339, delim) +} + +// parseFloat64s transforms strings to float64s. +func (k *Key) parseFloat64s(strs []string, addInvalid, returnOnInvalid bool) ([]float64, error) { + vals := make([]float64, 0, len(strs)) + for _, str := range strs { + val, err := strconv.ParseFloat(str, 64) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, val) + } + } + return vals, nil +} + +// parseInts transforms strings to ints. +func (k *Key) parseInts(strs []string, addInvalid, returnOnInvalid bool) ([]int, error) { + vals := make([]int, 0, len(strs)) + for _, str := range strs { + valInt64, err := strconv.ParseInt(str, 0, 64) + val := int(valInt64) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, val) + } + } + return vals, nil +} + +// parseInt64s transforms strings to int64s. +func (k *Key) parseInt64s(strs []string, addInvalid, returnOnInvalid bool) ([]int64, error) { + vals := make([]int64, 0, len(strs)) + for _, str := range strs { + val, err := strconv.ParseInt(str, 0, 64) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, val) + } + } + return vals, nil +} + +// parseUints transforms strings to uints. +func (k *Key) parseUints(strs []string, addInvalid, returnOnInvalid bool) ([]uint, error) { + vals := make([]uint, 0, len(strs)) + for _, str := range strs { + val, err := strconv.ParseUint(str, 0, 0) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, uint(val)) + } + } + return vals, nil +} + +// parseUint64s transforms strings to uint64s. +func (k *Key) parseUint64s(strs []string, addInvalid, returnOnInvalid bool) ([]uint64, error) { + vals := make([]uint64, 0, len(strs)) + for _, str := range strs { + val, err := strconv.ParseUint(str, 0, 64) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, val) + } + } + return vals, nil +} + +// parseTimesFormat transforms strings to times in given format. +func (k *Key) parseTimesFormat(format string, strs []string, addInvalid, returnOnInvalid bool) ([]time.Time, error) { + vals := make([]time.Time, 0, len(strs)) + for _, str := range strs { + val, err := time.Parse(format, str) + if err != nil && returnOnInvalid { + return nil, err + } + if err == nil || addInvalid { + vals = append(vals, val) + } + } + return vals, nil +} + +// SetValue changes key value. +func (k *Key) SetValue(v string) { + if k.s.f.BlockMode { + k.s.f.lock.Lock() + defer k.s.f.lock.Unlock() + } + + k.value = v + k.s.keysHash[k.name] = v +} diff --git a/vendor/github.com/go-ini/ini/parser.go b/vendor/github.com/go-ini/ini/parser.go new file mode 100644 index 00000000..f20073d1 --- /dev/null +++ b/vendor/github.com/go-ini/ini/parser.go @@ -0,0 +1,488 @@ +// Copyright 2015 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "bufio" + "bytes" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "unicode" +) + +var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]+)") + +type tokenType int + +const ( + _TOKEN_INVALID tokenType = iota + _TOKEN_COMMENT + _TOKEN_SECTION + _TOKEN_KEY +) + +type parser struct { + buf *bufio.Reader + isEOF bool + count int + comment *bytes.Buffer +} + +func newParser(r io.Reader) *parser { + return &parser{ + buf: bufio.NewReader(r), + count: 1, + comment: &bytes.Buffer{}, + } +} + +// BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format. +// http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding +func (p *parser) BOM() error { + mask, err := p.buf.Peek(2) + if err != nil && err != io.EOF { + return err + } else if len(mask) < 2 { + return nil + } + + switch { + case mask[0] == 254 && mask[1] == 255: + fallthrough + case mask[0] == 255 && mask[1] == 254: + p.buf.Read(mask) + case mask[0] == 239 && mask[1] == 187: + mask, err := p.buf.Peek(3) + if err != nil && err != io.EOF { + return err + } else if len(mask) < 3 { + return nil + } + if mask[2] == 191 { + p.buf.Read(mask) + } + } + return nil +} + +func (p *parser) readUntil(delim byte) ([]byte, error) { + data, err := p.buf.ReadBytes(delim) + if err != nil { + if err == io.EOF { + p.isEOF = true + } else { + return nil, err + } + } + return data, nil +} + +func cleanComment(in []byte) ([]byte, bool) { + i := bytes.IndexAny(in, "#;") + if i == -1 { + return nil, false + } + return in[i:], true +} + +func readKeyName(delimiters string, in []byte) (string, int, error) { + line := string(in) + + // Check if key name surrounded by quotes. + var keyQuote string + if line[0] == '"' { + if len(line) > 6 && string(line[0:3]) == `"""` { + keyQuote = `"""` + } else { + keyQuote = `"` + } + } else if line[0] == '`' { + keyQuote = "`" + } + + // Get out key name + endIdx := -1 + if len(keyQuote) > 0 { + startIdx := len(keyQuote) + // FIXME: fail case -> """"""name"""=value + pos := strings.Index(line[startIdx:], keyQuote) + if pos == -1 { + return "", -1, fmt.Errorf("missing closing key quote: %s", line) + } + pos += startIdx + + // Find key-value delimiter + i := strings.IndexAny(line[pos+startIdx:], delimiters) + if i < 0 { + return "", -1, ErrDelimiterNotFound{line} + } + endIdx = pos + i + return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil + } + + endIdx = strings.IndexAny(line, delimiters) + if endIdx < 0 { + return "", -1, ErrDelimiterNotFound{line} + } + return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil +} + +func (p *parser) readMultilines(line, val, valQuote string) (string, error) { + for { + data, err := p.readUntil('\n') + if err != nil { + return "", err + } + next := string(data) + + pos := strings.LastIndex(next, valQuote) + if pos > -1 { + val += next[:pos] + + comment, has := cleanComment([]byte(next[pos:])) + if has { + p.comment.Write(bytes.TrimSpace(comment)) + } + break + } + val += next + if p.isEOF { + return "", fmt.Errorf("missing closing key quote from '%s' to '%s'", line, next) + } + } + return val, nil +} + +func (p *parser) readContinuationLines(val string) (string, error) { + for { + data, err := p.readUntil('\n') + if err != nil { + return "", err + } + next := strings.TrimSpace(string(data)) + + if len(next) == 0 { + break + } + val += next + if val[len(val)-1] != '\\' { + break + } + val = val[:len(val)-1] + } + return val, nil +} + +// hasSurroundedQuote check if and only if the first and last characters +// are quotes \" or \'. +// It returns false if any other parts also contain same kind of quotes. +func hasSurroundedQuote(in string, quote byte) bool { + return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote && + strings.IndexByte(in[1:], quote) == len(in)-2 +} + +func (p *parser) readValue(in []byte, + parserBufferSize int, + ignoreContinuation, ignoreInlineComment, unescapeValueDoubleQuotes, unescapeValueCommentSymbols, allowPythonMultilines, spaceBeforeInlineComment, preserveSurroundedQuote bool) (string, error) { + + line := strings.TrimLeftFunc(string(in), unicode.IsSpace) + if len(line) == 0 { + return "", nil + } + + var valQuote string + if len(line) > 3 && string(line[0:3]) == `"""` { + valQuote = `"""` + } else if line[0] == '`' { + valQuote = "`" + } else if unescapeValueDoubleQuotes && line[0] == '"' { + valQuote = `"` + } + + if len(valQuote) > 0 { + startIdx := len(valQuote) + pos := strings.LastIndex(line[startIdx:], valQuote) + // Check for multi-line value + if pos == -1 { + return p.readMultilines(line, line[startIdx:], valQuote) + } + + if unescapeValueDoubleQuotes && valQuote == `"` { + return strings.Replace(line[startIdx:pos+startIdx], `\"`, `"`, -1), nil + } + return line[startIdx : pos+startIdx], nil + } + + lastChar := line[len(line)-1] + // Won't be able to reach here if value only contains whitespace + line = strings.TrimSpace(line) + trimmedLastChar := line[len(line)-1] + + // Check continuation lines when desired + if !ignoreContinuation && trimmedLastChar == '\\' { + return p.readContinuationLines(line[:len(line)-1]) + } + + // Check if ignore inline comment + if !ignoreInlineComment { + var i int + if spaceBeforeInlineComment { + i = strings.Index(line, " #") + if i == -1 { + i = strings.Index(line, " ;") + } + + } else { + i = strings.IndexAny(line, "#;") + } + + if i > -1 { + p.comment.WriteString(line[i:]) + line = strings.TrimSpace(line[:i]) + } + + } + + // Trim single and double quotes + if (hasSurroundedQuote(line, '\'') || + hasSurroundedQuote(line, '"')) && !preserveSurroundedQuote { + line = line[1 : len(line)-1] + } else if len(valQuote) == 0 && unescapeValueCommentSymbols { + if strings.Contains(line, `\;`) { + line = strings.Replace(line, `\;`, ";", -1) + } + if strings.Contains(line, `\#`) { + line = strings.Replace(line, `\#`, "#", -1) + } + } else if allowPythonMultilines && lastChar == '\n' { + parserBufferPeekResult, _ := p.buf.Peek(parserBufferSize) + peekBuffer := bytes.NewBuffer(parserBufferPeekResult) + + val := line + + for { + peekData, peekErr := peekBuffer.ReadBytes('\n') + if peekErr != nil { + if peekErr == io.EOF { + return val, nil + } + return "", peekErr + } + + peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) + if len(peekMatches) != 3 { + return val, nil + } + + // NOTE: Return if not a python-ini multi-line value. + currentIdentSize := len(peekMatches[1]) + if currentIdentSize <= 0 { + return val, nil + } + + // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer. + _, err := p.readUntil('\n') + if err != nil { + return "", err + } + + val += fmt.Sprintf("\n%s", peekMatches[2]) + } + } + + return line, nil +} + +// parse parses data through an io.Reader. +func (f *File) parse(reader io.Reader) (err error) { + p := newParser(reader) + if err = p.BOM(); err != nil { + return fmt.Errorf("BOM: %v", err) + } + + // Ignore error because default section name is never empty string. + name := DEFAULT_SECTION + if f.options.Insensitive { + name = strings.ToLower(DEFAULT_SECTION) + } + section, _ := f.NewSection(name) + + // This "last" is not strictly equivalent to "previous one" if current key is not the first nested key + var isLastValueEmpty bool + var lastRegularKey *Key + + var line []byte + var inUnparseableSection bool + + // NOTE: Iterate and increase `currentPeekSize` until + // the size of the parser buffer is found. + // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. + parserBufferSize := 0 + // NOTE: Peek 1kb at a time. + currentPeekSize := 1024 + + if f.options.AllowPythonMultilineValues { + for { + peekBytes, _ := p.buf.Peek(currentPeekSize) + peekBytesLength := len(peekBytes) + + if parserBufferSize >= peekBytesLength { + break + } + + currentPeekSize *= 2 + parserBufferSize = peekBytesLength + } + } + + for !p.isEOF { + line, err = p.readUntil('\n') + if err != nil { + return err + } + + if f.options.AllowNestedValues && + isLastValueEmpty && len(line) > 0 { + if line[0] == ' ' || line[0] == '\t' { + lastRegularKey.addNestedValue(string(bytes.TrimSpace(line))) + continue + } + } + + line = bytes.TrimLeftFunc(line, unicode.IsSpace) + if len(line) == 0 { + continue + } + + // Comments + if line[0] == '#' || line[0] == ';' { + // Note: we do not care ending line break, + // it is needed for adding second line, + // so just clean it once at the end when set to value. + p.comment.Write(line) + continue + } + + // Section + if line[0] == '[' { + // Read to the next ']' (TODO: support quoted strings) + closeIdx := bytes.LastIndexByte(line, ']') + if closeIdx == -1 { + return fmt.Errorf("unclosed section: %s", line) + } + + name := string(line[1:closeIdx]) + section, err = f.NewSection(name) + if err != nil { + return err + } + + comment, has := cleanComment(line[closeIdx+1:]) + if has { + p.comment.Write(comment) + } + + section.Comment = strings.TrimSpace(p.comment.String()) + + // Reset aotu-counter and comments + p.comment.Reset() + p.count = 1 + + inUnparseableSection = false + for i := range f.options.UnparseableSections { + if f.options.UnparseableSections[i] == name || + (f.options.Insensitive && strings.ToLower(f.options.UnparseableSections[i]) == strings.ToLower(name)) { + inUnparseableSection = true + continue + } + } + continue + } + + if inUnparseableSection { + section.isRawSection = true + section.rawBody += string(line) + continue + } + + kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line) + if err != nil { + // Treat as boolean key when desired, and whole line is key name. + if IsErrDelimiterNotFound(err) { + switch { + case f.options.AllowBooleanKeys: + kname, err := p.readValue(line, + parserBufferSize, + f.options.IgnoreContinuation, + f.options.IgnoreInlineComment, + f.options.UnescapeValueDoubleQuotes, + f.options.UnescapeValueCommentSymbols, + f.options.AllowPythonMultilineValues, + f.options.SpaceBeforeInlineComment, + f.options.PreserveSurroundedQuote) + if err != nil { + return err + } + key, err := section.NewBooleanKey(kname) + if err != nil { + return err + } + key.Comment = strings.TrimSpace(p.comment.String()) + p.comment.Reset() + continue + + case f.options.SkipUnrecognizableLines: + continue + } + } + return err + } + + // Auto increment. + isAutoIncr := false + if kname == "-" { + isAutoIncr = true + kname = "#" + strconv.Itoa(p.count) + p.count++ + } + + value, err := p.readValue(line[offset:], + parserBufferSize, + f.options.IgnoreContinuation, + f.options.IgnoreInlineComment, + f.options.UnescapeValueDoubleQuotes, + f.options.UnescapeValueCommentSymbols, + f.options.AllowPythonMultilineValues, + f.options.SpaceBeforeInlineComment, + f.options.PreserveSurroundedQuote) + if err != nil { + return err + } + isLastValueEmpty = len(value) == 0 + + key, err := section.NewKey(kname, value) + if err != nil { + return err + } + key.isAutoIncrement = isAutoIncr + key.Comment = strings.TrimSpace(p.comment.String()) + p.comment.Reset() + lastRegularKey = key + } + return nil +} diff --git a/vendor/github.com/go-ini/ini/section.go b/vendor/github.com/go-ini/ini/section.go new file mode 100644 index 00000000..bc32c620 --- /dev/null +++ b/vendor/github.com/go-ini/ini/section.go @@ -0,0 +1,259 @@ +// Copyright 2014 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "errors" + "fmt" + "strings" +) + +// Section represents a config section. +type Section struct { + f *File + Comment string + name string + keys map[string]*Key + keyList []string + keysHash map[string]string + + isRawSection bool + rawBody string +} + +func newSection(f *File, name string) *Section { + return &Section{ + f: f, + name: name, + keys: make(map[string]*Key), + keyList: make([]string, 0, 10), + keysHash: make(map[string]string), + } +} + +// Name returns name of Section. +func (s *Section) Name() string { + return s.name +} + +// Body returns rawBody of Section if the section was marked as unparseable. +// It still follows the other rules of the INI format surrounding leading/trailing whitespace. +func (s *Section) Body() string { + return strings.TrimSpace(s.rawBody) +} + +// SetBody updates body content only if section is raw. +func (s *Section) SetBody(body string) { + if !s.isRawSection { + return + } + s.rawBody = body +} + +// NewKey creates a new key to given section. +func (s *Section) NewKey(name, val string) (*Key, error) { + if len(name) == 0 { + return nil, errors.New("error creating new key: empty key name") + } else if s.f.options.Insensitive { + name = strings.ToLower(name) + } + + if s.f.BlockMode { + s.f.lock.Lock() + defer s.f.lock.Unlock() + } + + if inSlice(name, s.keyList) { + if s.f.options.AllowShadows { + if err := s.keys[name].addShadow(val); err != nil { + return nil, err + } + } else { + s.keys[name].value = val + s.keysHash[name] = val + } + return s.keys[name], nil + } + + s.keyList = append(s.keyList, name) + s.keys[name] = newKey(s, name, val) + s.keysHash[name] = val + return s.keys[name], nil +} + +// NewBooleanKey creates a new boolean type key to given section. +func (s *Section) NewBooleanKey(name string) (*Key, error) { + key, err := s.NewKey(name, "true") + if err != nil { + return nil, err + } + + key.isBooleanType = true + return key, nil +} + +// GetKey returns key in section by given name. +func (s *Section) GetKey(name string) (*Key, error) { + // FIXME: change to section level lock? + if s.f.BlockMode { + s.f.lock.RLock() + } + if s.f.options.Insensitive { + name = strings.ToLower(name) + } + key := s.keys[name] + if s.f.BlockMode { + s.f.lock.RUnlock() + } + + if key == nil { + // Check if it is a child-section. + sname := s.name + for { + if i := strings.LastIndex(sname, "."); i > -1 { + sname = sname[:i] + sec, err := s.f.GetSection(sname) + if err != nil { + continue + } + return sec.GetKey(name) + } else { + break + } + } + return nil, fmt.Errorf("error when getting key of section '%s': key '%s' not exists", s.name, name) + } + return key, nil +} + +// HasKey returns true if section contains a key with given name. +func (s *Section) HasKey(name string) bool { + key, _ := s.GetKey(name) + return key != nil +} + +// Haskey is a backwards-compatible name for HasKey. +// TODO: delete me in v2 +func (s *Section) Haskey(name string) bool { + return s.HasKey(name) +} + +// HasValue returns true if section contains given raw value. +func (s *Section) HasValue(value string) bool { + if s.f.BlockMode { + s.f.lock.RLock() + defer s.f.lock.RUnlock() + } + + for _, k := range s.keys { + if value == k.value { + return true + } + } + return false +} + +// Key assumes named Key exists in section and returns a zero-value when not. +func (s *Section) Key(name string) *Key { + key, err := s.GetKey(name) + if err != nil { + // It's OK here because the only possible error is empty key name, + // but if it's empty, this piece of code won't be executed. + key, _ = s.NewKey(name, "") + return key + } + return key +} + +// Keys returns list of keys of section. +func (s *Section) Keys() []*Key { + keys := make([]*Key, len(s.keyList)) + for i := range s.keyList { + keys[i] = s.Key(s.keyList[i]) + } + return keys +} + +// ParentKeys returns list of keys of parent section. +func (s *Section) ParentKeys() []*Key { + var parentKeys []*Key + sname := s.name + for { + if i := strings.LastIndex(sname, "."); i > -1 { + sname = sname[:i] + sec, err := s.f.GetSection(sname) + if err != nil { + continue + } + parentKeys = append(parentKeys, sec.Keys()...) + } else { + break + } + + } + return parentKeys +} + +// KeyStrings returns list of key names of section. +func (s *Section) KeyStrings() []string { + list := make([]string, len(s.keyList)) + copy(list, s.keyList) + return list +} + +// KeysHash returns keys hash consisting of names and values. +func (s *Section) KeysHash() map[string]string { + if s.f.BlockMode { + s.f.lock.RLock() + defer s.f.lock.RUnlock() + } + + hash := map[string]string{} + for key, value := range s.keysHash { + hash[key] = value + } + return hash +} + +// DeleteKey deletes a key from section. +func (s *Section) DeleteKey(name string) { + if s.f.BlockMode { + s.f.lock.Lock() + defer s.f.lock.Unlock() + } + + for i, k := range s.keyList { + if k == name { + s.keyList = append(s.keyList[:i], s.keyList[i+1:]...) + delete(s.keys, name) + delete(s.keysHash, name) + return + } + } +} + +// ChildSections returns a list of child sections of current section. +// For example, "[parent.child1]" and "[parent.child12]" are child sections +// of section "[parent]". +func (s *Section) ChildSections() []*Section { + prefix := s.name + "." + children := make([]*Section, 0, 3) + for _, name := range s.f.sectionList { + if strings.HasPrefix(name, prefix) { + children = append(children, s.f.sections[name]) + } + } + return children +} diff --git a/vendor/github.com/go-ini/ini/struct.go b/vendor/github.com/go-ini/ini/struct.go new file mode 100644 index 00000000..a9dfed07 --- /dev/null +++ b/vendor/github.com/go-ini/ini/struct.go @@ -0,0 +1,512 @@ +// Copyright 2014 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "strings" + "time" + "unicode" +) + +// NameMapper represents a ini tag name mapper. +type NameMapper func(string) string + +// Built-in name getters. +var ( + // AllCapsUnderscore converts to format ALL_CAPS_UNDERSCORE. + AllCapsUnderscore NameMapper = func(raw string) string { + newstr := make([]rune, 0, len(raw)) + for i, chr := range raw { + if isUpper := 'A' <= chr && chr <= 'Z'; isUpper { + if i > 0 { + newstr = append(newstr, '_') + } + } + newstr = append(newstr, unicode.ToUpper(chr)) + } + return string(newstr) + } + // TitleUnderscore converts to format title_underscore. + TitleUnderscore NameMapper = func(raw string) string { + newstr := make([]rune, 0, len(raw)) + for i, chr := range raw { + if isUpper := 'A' <= chr && chr <= 'Z'; isUpper { + if i > 0 { + newstr = append(newstr, '_') + } + chr -= ('A' - 'a') + } + newstr = append(newstr, chr) + } + return string(newstr) + } +) + +func (s *Section) parseFieldName(raw, actual string) string { + if len(actual) > 0 { + return actual + } + if s.f.NameMapper != nil { + return s.f.NameMapper(raw) + } + return raw +} + +func parseDelim(actual string) string { + if len(actual) > 0 { + return actual + } + return "," +} + +var reflectTime = reflect.TypeOf(time.Now()).Kind() + +// setSliceWithProperType sets proper values to slice based on its type. +func setSliceWithProperType(key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error { + var strs []string + if allowShadow { + strs = key.StringsWithShadows(delim) + } else { + strs = key.Strings(delim) + } + + numVals := len(strs) + if numVals == 0 { + return nil + } + + var vals interface{} + var err error + + sliceOf := field.Type().Elem().Kind() + switch sliceOf { + case reflect.String: + vals = strs + case reflect.Int: + vals, err = key.parseInts(strs, true, false) + case reflect.Int64: + vals, err = key.parseInt64s(strs, true, false) + case reflect.Uint: + vals, err = key.parseUints(strs, true, false) + case reflect.Uint64: + vals, err = key.parseUint64s(strs, true, false) + case reflect.Float64: + vals, err = key.parseFloat64s(strs, true, false) + case reflectTime: + vals, err = key.parseTimesFormat(time.RFC3339, strs, true, false) + default: + return fmt.Errorf("unsupported type '[]%s'", sliceOf) + } + if err != nil && isStrict { + return err + } + + slice := reflect.MakeSlice(field.Type(), numVals, numVals) + for i := 0; i < numVals; i++ { + switch sliceOf { + case reflect.String: + slice.Index(i).Set(reflect.ValueOf(vals.([]string)[i])) + case reflect.Int: + slice.Index(i).Set(reflect.ValueOf(vals.([]int)[i])) + case reflect.Int64: + slice.Index(i).Set(reflect.ValueOf(vals.([]int64)[i])) + case reflect.Uint: + slice.Index(i).Set(reflect.ValueOf(vals.([]uint)[i])) + case reflect.Uint64: + slice.Index(i).Set(reflect.ValueOf(vals.([]uint64)[i])) + case reflect.Float64: + slice.Index(i).Set(reflect.ValueOf(vals.([]float64)[i])) + case reflectTime: + slice.Index(i).Set(reflect.ValueOf(vals.([]time.Time)[i])) + } + } + field.Set(slice) + return nil +} + +func wrapStrictError(err error, isStrict bool) error { + if isStrict { + return err + } + return nil +} + +// setWithProperType sets proper value to field based on its type, +// but it does not return error for failing parsing, +// because we want to use default value that is already assigned to strcut. +func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error { + switch t.Kind() { + case reflect.String: + if len(key.String()) == 0 { + return nil + } + field.SetString(key.String()) + case reflect.Bool: + boolVal, err := key.Bool() + if err != nil { + return wrapStrictError(err, isStrict) + } + field.SetBool(boolVal) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + durationVal, err := key.Duration() + // Skip zero value + if err == nil && int64(durationVal) > 0 { + field.Set(reflect.ValueOf(durationVal)) + return nil + } + + intVal, err := key.Int64() + if err != nil { + return wrapStrictError(err, isStrict) + } + field.SetInt(intVal) + // byte is an alias for uint8, so supporting uint8 breaks support for byte + case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64: + durationVal, err := key.Duration() + // Skip zero value + if err == nil && uint64(durationVal) > 0 { + field.Set(reflect.ValueOf(durationVal)) + return nil + } + + uintVal, err := key.Uint64() + if err != nil { + return wrapStrictError(err, isStrict) + } + field.SetUint(uintVal) + + case reflect.Float32, reflect.Float64: + floatVal, err := key.Float64() + if err != nil { + return wrapStrictError(err, isStrict) + } + field.SetFloat(floatVal) + case reflectTime: + timeVal, err := key.Time() + if err != nil { + return wrapStrictError(err, isStrict) + } + field.Set(reflect.ValueOf(timeVal)) + case reflect.Slice: + return setSliceWithProperType(key, field, delim, allowShadow, isStrict) + default: + return fmt.Errorf("unsupported type '%s'", t) + } + return nil +} + +func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool) { + opts := strings.SplitN(tag, ",", 3) + rawName = opts[0] + if len(opts) > 1 { + omitEmpty = opts[1] == "omitempty" + } + if len(opts) > 2 { + allowShadow = opts[2] == "allowshadow" + } + return rawName, omitEmpty, allowShadow +} + +func (s *Section) mapTo(val reflect.Value, isStrict bool) error { + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + typ := val.Type() + + for i := 0; i < typ.NumField(); i++ { + field := val.Field(i) + tpField := typ.Field(i) + + tag := tpField.Tag.Get("ini") + if tag == "-" { + continue + } + + rawName, _, allowShadow := parseTagOptions(tag) + fieldName := s.parseFieldName(tpField.Name, rawName) + if len(fieldName) == 0 || !field.CanSet() { + continue + } + + isAnonymous := tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous + isStruct := tpField.Type.Kind() == reflect.Struct + if isAnonymous { + field.Set(reflect.New(tpField.Type.Elem())) + } + + if isAnonymous || isStruct { + if sec, err := s.f.GetSection(fieldName); err == nil { + if err = sec.mapTo(field, isStrict); err != nil { + return fmt.Errorf("error mapping field(%s): %v", fieldName, err) + } + continue + } + } + + if key, err := s.GetKey(fieldName); err == nil { + delim := parseDelim(tpField.Tag.Get("delim")) + if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil { + return fmt.Errorf("error mapping field(%s): %v", fieldName, err) + } + } + } + return nil +} + +// MapTo maps section to given struct. +func (s *Section) MapTo(v interface{}) error { + typ := reflect.TypeOf(v) + val := reflect.ValueOf(v) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + val = val.Elem() + } else { + return errors.New("cannot map to non-pointer struct") + } + + return s.mapTo(val, false) +} + +// MapTo maps section to given struct in strict mode, +// which returns all possible error including value parsing error. +func (s *Section) StrictMapTo(v interface{}) error { + typ := reflect.TypeOf(v) + val := reflect.ValueOf(v) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + val = val.Elem() + } else { + return errors.New("cannot map to non-pointer struct") + } + + return s.mapTo(val, true) +} + +// MapTo maps file to given struct. +func (f *File) MapTo(v interface{}) error { + return f.Section("").MapTo(v) +} + +// MapTo maps file to given struct in strict mode, +// which returns all possible error including value parsing error. +func (f *File) StrictMapTo(v interface{}) error { + return f.Section("").StrictMapTo(v) +} + +// MapTo maps data sources to given struct with name mapper. +func MapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error { + cfg, err := Load(source, others...) + if err != nil { + return err + } + cfg.NameMapper = mapper + return cfg.MapTo(v) +} + +// StrictMapToWithMapper maps data sources to given struct with name mapper in strict mode, +// which returns all possible error including value parsing error. +func StrictMapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error { + cfg, err := Load(source, others...) + if err != nil { + return err + } + cfg.NameMapper = mapper + return cfg.StrictMapTo(v) +} + +// MapTo maps data sources to given struct. +func MapTo(v, source interface{}, others ...interface{}) error { + return MapToWithMapper(v, nil, source, others...) +} + +// StrictMapTo maps data sources to given struct in strict mode, +// which returns all possible error including value parsing error. +func StrictMapTo(v, source interface{}, others ...interface{}) error { + return StrictMapToWithMapper(v, nil, source, others...) +} + +// reflectSliceWithProperType does the opposite thing as setSliceWithProperType. +func reflectSliceWithProperType(key *Key, field reflect.Value, delim string) error { + slice := field.Slice(0, field.Len()) + if field.Len() == 0 { + return nil + } + + var buf bytes.Buffer + sliceOf := field.Type().Elem().Kind() + for i := 0; i < field.Len(); i++ { + switch sliceOf { + case reflect.String: + buf.WriteString(slice.Index(i).String()) + case reflect.Int, reflect.Int64: + buf.WriteString(fmt.Sprint(slice.Index(i).Int())) + case reflect.Uint, reflect.Uint64: + buf.WriteString(fmt.Sprint(slice.Index(i).Uint())) + case reflect.Float64: + buf.WriteString(fmt.Sprint(slice.Index(i).Float())) + case reflectTime: + buf.WriteString(slice.Index(i).Interface().(time.Time).Format(time.RFC3339)) + default: + return fmt.Errorf("unsupported type '[]%s'", sliceOf) + } + buf.WriteString(delim) + } + key.SetValue(buf.String()[:buf.Len()-1]) + return nil +} + +// reflectWithProperType does the opposite thing as setWithProperType. +func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string) error { + switch t.Kind() { + case reflect.String: + key.SetValue(field.String()) + case reflect.Bool: + key.SetValue(fmt.Sprint(field.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + key.SetValue(fmt.Sprint(field.Int())) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + key.SetValue(fmt.Sprint(field.Uint())) + case reflect.Float32, reflect.Float64: + key.SetValue(fmt.Sprint(field.Float())) + case reflectTime: + key.SetValue(fmt.Sprint(field.Interface().(time.Time).Format(time.RFC3339))) + case reflect.Slice: + return reflectSliceWithProperType(key, field, delim) + default: + return fmt.Errorf("unsupported type '%s'", t) + } + return nil +} + +// CR: copied from encoding/json/encode.go with modifications of time.Time support. +// TODO: add more test coverage. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflectTime: + t, ok := v.Interface().(time.Time) + return ok && t.IsZero() + } + return false +} + +func (s *Section) reflectFrom(val reflect.Value) error { + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + typ := val.Type() + + for i := 0; i < typ.NumField(); i++ { + field := val.Field(i) + tpField := typ.Field(i) + + tag := tpField.Tag.Get("ini") + if tag == "-" { + continue + } + + opts := strings.SplitN(tag, ",", 2) + if len(opts) == 2 && opts[1] == "omitempty" && isEmptyValue(field) { + continue + } + + fieldName := s.parseFieldName(tpField.Name, opts[0]) + if len(fieldName) == 0 || !field.CanSet() { + continue + } + + if (tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous) || + (tpField.Type.Kind() == reflect.Struct && tpField.Type.Name() != "Time") { + // Note: The only error here is section doesn't exist. + sec, err := s.f.GetSection(fieldName) + if err != nil { + // Note: fieldName can never be empty here, ignore error. + sec, _ = s.f.NewSection(fieldName) + } + + // Add comment from comment tag + if len(sec.Comment) == 0 { + sec.Comment = tpField.Tag.Get("comment") + } + + if err = sec.reflectFrom(field); err != nil { + return fmt.Errorf("error reflecting field (%s): %v", fieldName, err) + } + continue + } + + // Note: Same reason as secion. + key, err := s.GetKey(fieldName) + if err != nil { + key, _ = s.NewKey(fieldName, "") + } + + // Add comment from comment tag + if len(key.Comment) == 0 { + key.Comment = tpField.Tag.Get("comment") + } + + if err = reflectWithProperType(tpField.Type, key, field, parseDelim(tpField.Tag.Get("delim"))); err != nil { + return fmt.Errorf("error reflecting field (%s): %v", fieldName, err) + } + + } + return nil +} + +// ReflectFrom reflects secion from given struct. +func (s *Section) ReflectFrom(v interface{}) error { + typ := reflect.TypeOf(v) + val := reflect.ValueOf(v) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + val = val.Elem() + } else { + return errors.New("cannot reflect from non-pointer struct") + } + + return s.reflectFrom(val) +} + +// ReflectFrom reflects file from given struct. +func (f *File) ReflectFrom(v interface{}) error { + return f.Section("").ReflectFrom(v) +} + +// ReflectFrom reflects data sources from given struct with name mapper. +func ReflectFromWithMapper(cfg *File, v interface{}, mapper NameMapper) error { + cfg.NameMapper = mapper + return cfg.ReflectFrom(v) +} + +// ReflectFrom reflects data sources from given struct. +func ReflectFrom(cfg *File, v interface{}) error { + return ReflectFromWithMapper(cfg, v, nil) +}