From 71bb38f496b8fe01b84839b55918c5248f9bd883 Mon Sep 17 00:00:00 2001 From: Alexander Kanevskiy Date: Sat, 24 Aug 2019 01:24:33 +0300 Subject: [PATCH] Implemented native FPGA flashing Removed dependency to OPAE libraries --- cmd/fpga_crihook/main.go | 50 ++---- cmd/fpga_tool/fpga_tool.go | 87 ++++----- pkg/fpga/bitstream/aocx.go | 3 +- pkg/fpga/bitstream/bitstream.go | 137 +-------------- pkg/fpga/bitstream/gbs.go | 1 - pkg/fpga/device/device.go | 189 -------------------- pkg/fpga/linux/dfl.go | 302 ++++++++++++++++++++++++++++---- pkg/fpga/linux/fpga.go | 117 +++++++++++++ pkg/fpga/linux/intel-fpga.go | 296 +++++++++++++++++++++++++++---- pkg/fpga/linux/interfaces.go | 53 +++++- pkg/fpga/linux/ioctl.go | 11 ++ pkg/fpga/linux/pci.go | 2 + pkg/fpga/linux/utils.go | 40 ++++- 13 files changed, 796 insertions(+), 492 deletions(-) delete mode 100644 pkg/fpga/device/device.go create mode 100644 pkg/fpga/linux/fpga.go diff --git a/cmd/fpga_crihook/main.go b/cmd/fpga_crihook/main.go index 321ecbb1..a0dea139 100644 --- a/cmd/fpga_crihook/main.go +++ b/cmd/fpga_crihook/main.go @@ -23,7 +23,7 @@ import ( "strings" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device" + fpga "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux" "github.com/pkg/errors" utilsexec "k8s.io/utils/exec" ) @@ -144,10 +144,10 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) { splitted := strings.SplitN(env, "=", 2) if strings.HasPrefix(splitted[0], fpgaRegionEnvPrefix) { num := strings.Split(splitted[0], fpgaRegionEnvPrefix)[1] - regionEnv[num] = device.CanonizeID(splitted[1]) + regionEnv[num] = fpga.CanonizeID(splitted[1]) } else if strings.HasPrefix(splitted[0], fpgaAfuEnvPrefix) { num := strings.Split(splitted[0], fpgaAfuEnvPrefix)[1] - afuEnv[num] = device.CanonizeID(splitted[1]) + afuEnv[num] = fpga.CanonizeID(splitted[1]) } } @@ -175,7 +175,7 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) { for _, dev := range config.Linux.Devices { deviceName := dev.getName() // skip non-FPGA devices - if !device.IsFPGADevice(deviceName) { + if !fpga.IsFpgaPort(deviceName) { continue } @@ -183,16 +183,15 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) { if dev.processed { continue } - fme, err := device.GetFMEDevice(he.sysFsPrefix, deviceName) + port, err := fpga.NewFpgaPort(deviceName) if err != nil { return nil, err } - - if fme.ID == region { + if interfaceUUID := port.GetInterfaceUUID(); interfaceUUID == region { params = append(params, fpgaParams{ afu: afu, - region: fme.ID, + region: interfaceUUID, portDevice: deviceName, }, ) @@ -253,12 +252,13 @@ func (he *hookEnv) process(reader io.Reader) error { } for _, params := range paramslist { - programmedAfu, err := device.GetAFUDevice(he.sysFsPrefix, params.portDevice) + port, err := fpga.NewFpgaPort(params.portDevice) if err != nil { return err } - if programmedAfu.ID == params.afu { + programmedAfu := port.GetAcceleratorTypeUUID() + if programmedAfu == params.afu { // Afu is already programmed return nil } @@ -267,34 +267,14 @@ func (he *hookEnv) process(reader io.Reader) error { if err != nil { return err } + defer bitstream.Close() - err = bitstream.Init() - if err != nil { - return err - } + err = port.PR(bitstream, false) - err = bitstream.Validate(params.region, params.afu) - if err != nil { - return err - } + programmedAfu = port.GetAcceleratorTypeUUID() - fme, err := device.GetFMEDevice(he.sysFsPrefix, params.portDevice) - if err != nil { - return err - } - - err = bitstream.Program(fme, he.execer) - if err != nil { - return err - } - - programmedAfu, err = device.GetAFUDevice(he.sysFsPrefix, params.portDevice) - if err != nil { - return err - } - - if programmedAfu.ID != params.afu { - return errors.Errorf("programmed function %s instead of %s", programmedAfu.ID, params.afu) + if programmedAfu != bitstream.AcceleratorTypeUUID() { + return errors.Errorf("programmed function %s instead of %s", programmedAfu, bitstream.AcceleratorTypeUUID()) } } diff --git a/cmd/fpga_tool/fpga_tool.go b/cmd/fpga_tool/fpga_tool.go index b074ede1..f8edb268 100644 --- a/cmd/fpga_tool/fpga_tool.go +++ b/cmd/fpga_tool/fpga_tool.go @@ -21,12 +21,10 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/pkg/errors" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device" fpga "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux" ) @@ -106,16 +104,7 @@ func validateFlags(cmd, bitstream, device string) error { // WIP testing command func magic(dev string) (err error) { - d, err := device.GetFMEDevice("", dev) - fmt.Printf("%+v %+v\n", d, err) - - d1, err := fpga.FindSysFsDevice(dev) - fmt.Printf("%+v %+v\n", d1, err) - if err != nil { - return - } - d2, err := fpga.NewPCIDevice(d1) - fmt.Printf("%+v %+v\n", d2, err) + fmt.Println(fpga.ListFpgaDevices()) return } @@ -153,9 +142,9 @@ func installBitstream(fname string, dryRun bool) (err error) { func fpgaInfo(fname string) error { switch { - case strings.HasPrefix(fname, "/dev/dfl-fme."), strings.HasPrefix(fname, "/dev/intel-fpga-fme."): + case fpga.IsFpgaFME(fname): return fmeInfo(fname) - case strings.HasPrefix(fname, "/dev/dfl-port."), strings.HasPrefix(fname, "/dev/intel-fpga-port."): + case fpga.IsFpgaPort(fname): return portInfo(fname) } return errors.Errorf("unknown FPGA device file %s", fname) @@ -185,14 +174,7 @@ func printBitstreamInfo(fname string) (err error) { func fmeInfo(fname string) error { var f fpga.FpgaFME var err error - switch { - case strings.HasPrefix(fname, "/dev/dfl-fme."): - f, err = fpga.NewDflFME(fname) - case strings.HasPrefix(fname, "/dev/intel-fpga-fme."): - f, err = fpga.NewIntelFpgaFME(fname) - default: - return errors.Errorf("unknow type of FME %s", fname) - } + f, err = fpga.NewFpgaFME(fname) if err != nil { return err } @@ -201,20 +183,21 @@ func fmeInfo(fname string) error { fmt.Println(f.GetAPIVersion()) fmt.Print("CheckExtension:") fmt.Println(f.CheckExtension()) + + fmt.Println("GetDevPath: ", f.GetDevPath()) + fmt.Println("GetSysFsPath: ", f.GetSysFsPath()) + fmt.Println("GetName: ", f.GetName()) + pci, err := f.GetPCIDevice() + fmt.Printf("GetPCIDevice: %+v %+v\n", pci, err) + fmt.Println("GetInterfaceUUID: ", f.GetInterfaceUUID()) + fmt.Println("GetPortNums: ", f.GetPortsNum()) return nil } func portInfo(fname string) error { var f fpga.FpgaPort var err error - switch { - case strings.HasPrefix(fname, "/dev/dfl-port."): - f, err = fpga.NewDflPort(fname) - case strings.HasPrefix(fname, "/dev/intel-fpga-port."): - f, err = fpga.NewIntelFpgaPort(fname) - default: - err = errors.Errorf("unknown type of port %s", fname) - } + f, err = fpga.NewFpgaPort(fname) if err != nil { return err } @@ -234,42 +217,38 @@ func portInfo(fname string) error { fmt.Println(f.PortGetRegionInfo(uint32(idx))) } } + + fmt.Println("GetDevPath: ", f.GetDevPath()) + fmt.Println("GetSysFsPath: ", f.GetSysFsPath()) + fmt.Println("GetName: ", f.GetName()) + pci, err := f.GetPCIDevice() + fmt.Printf("GetPCIDevice: %+v %+v\n", pci, err) + id, err := f.GetPortID() + fmt.Printf("GetPort: %+v %+v\n", id, err) + fmt.Println("GetAcceleratorTypeUUID: ", f.GetAcceleratorTypeUUID()) + fmt.Println("GetInterfaceUUID: ", f.GetInterfaceUUID()) + fme, err := f.GetFME() + fmt.Printf("GetFME: %+v %+v\n", fme, err) + return nil } -func doPR(fme, bs string, dryRun bool) error { - var f fpga.FpgaFME - var err error - switch { - case strings.HasPrefix(fme, "/dev/dfl-fme."): - f, err = fpga.NewDflFME(fme) - case strings.HasPrefix(fme, "/dev/intel-fpga-fme."): - f, err = fpga.NewIntelFpgaFME(fme) - default: - return errors.Errorf("unknown FME %s", fme) - } - fmt.Printf("Trying to program %s to port 0 of %s", bs, fme) +func doPR(dev, bs string, dryRun bool) error { + + f, err := fpga.NewFpgaPort(dev) if err != nil { return err } defer f.Close() - fmt.Print("API:") - fmt.Println(f.GetAPIVersion()) m, err := bitstream.Open(bs) if err != nil { return err } defer m.Close() - rawBistream, err := m.RawBitstreamData() - if err != nil { - return err - } - if dryRun { - fmt.Println("Dry-Run: Skipping actual programming") - return nil - } - fmt.Print("Trying to PR, brace yourself! :") - fmt.Println(f.PortPR(0, rawBistream)) + fmt.Printf("Before programming I %q A %q\n", f.GetInterfaceUUID(), f.GetAcceleratorTypeUUID()) + fmt.Printf("Trying to program %s to port %s: ", bs, dev) + fmt.Println(f.PR(m, dryRun)) + fmt.Printf("After programming I %q A %q\n", f.GetInterfaceUUID(), f.GetAcceleratorTypeUUID()) return nil } diff --git a/pkg/fpga/bitstream/aocx.go b/pkg/fpga/bitstream/aocx.go index 482f6b16..8d47eba7 100644 --- a/pkg/fpga/bitstream/aocx.go +++ b/pkg/fpga/bitstream/aocx.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "github.com/pkg/errors" ) @@ -113,7 +114,7 @@ func setSection(f *FileAOCX, section *elf.Section) error { if err != nil { return errors.Wrapf(err, "%s: unable to get section data", name) } - *field = string(data) + *field = strings.TrimSpace(string(data)) } return nil } diff --git a/pkg/fpga/bitstream/bitstream.go b/pkg/fpga/bitstream/bitstream.go index 65002d0b..e3f6af23 100644 --- a/pkg/fpga/bitstream/bitstream.go +++ b/pkg/fpga/bitstream/bitstream.go @@ -15,61 +15,14 @@ package bitstream import ( - "fmt" "os" "path/filepath" - "regexp" - "strings" - utilsexec "k8s.io/utils/exec" - - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device" - "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux" "github.com/pkg/errors" ) -const ( - fpgaconf = "/opt/intel/fpga-sw/opae/fpgaconf-wrapper" - pciAddressRegex = `^[[:xdigit:]]{4}:([[:xdigit:]]{2}):([[:xdigit:]]{2})\.([[:xdigit:]])$` -) - -var ( - pciAddressRE = regexp.MustCompile(pciAddressRegex) -) - -// FPGABitstream defines common interface for OPAE and OpenCL bitstreams -type FPGABitstream interface { - Init() error - Validate(region, afu string) error - Program(fme *device.FPGADevice, execer utilsexec.Interface) error -} - -// OPAEBitstream defines structure of .gbs file -type OPAEBitstream struct { - Path string - Region string - AFU string -} - -// Init gets Region and AFU from .gbs file -func (bitstream *OPAEBitstream) Init() error { - gbs, err := OpenGBS(bitstream.Path) - if err != nil { - return errors.Wrapf(err, "%s: can't get bitstream info", bitstream.Path) - } - - if len(gbs.Metadata.AfuImage.AcceleratorClusters) != 1 { - return errors.Errorf("%s: 'accelerator-clusters' field not found", bitstream.Path) - } - - bitstream.Region = device.CanonizeID(gbs.Metadata.AfuImage.InterfaceUUID) - bitstream.AFU = device.CanonizeID(gbs.Metadata.AfuImage.AcceleratorClusters[0].AcceleratorTypeUUID) - - return nil -} - // GetFPGABitstream scans bitstream storage and returns first found bitstream by region and afu id -func GetFPGABitstream(bitstreamDir, region, afu string) (FPGABitstream, error) { +func GetFPGABitstream(bitstreamDir, region, afu string) (File, error) { bitstreamPath := "" // Temporarily only support gbs bitstreams // for _, ext := range []string{".gbs", ".aocx"} { @@ -84,97 +37,11 @@ func GetFPGABitstream(bitstreamDir, region, afu string) (FPGABitstream, error) { return nil, errors.Errorf("%s: stat error: %v", bitstreamPath, err) } - if ext == ".gbs" { - return &OPAEBitstream{ - Path: bitstreamPath, - Region: region, - AFU: afu}, nil - } else if ext == ".aocx" { - return &OpenCLBitstream{ - Path: bitstreamPath, - Region: region, - AFU: afu}, nil - } + return Open(bitstreamPath) } return nil, errors.Errorf("%s/%s: bitstream not found", region, afu) } -func validate(region, afu, expectedRegion, expectedAFU string) error { - if expectedRegion != region { - return errors.Errorf("bitstream is not for this device: region(%s) and interface-uuid(%s) don't match", region, expectedRegion) - } - - if expectedAFU != afu { - return errors.Errorf("incorrect bitstream: AFU(%s) and accelerator-type-uuid(%s) don't match", afu, expectedAFU) - } - - return nil -} - -// Validate checks if region and afu parameters mutch bitstream parameters -func (bitstream *OPAEBitstream) Validate(region, afu string) error { - return validate(bitstream.Region, bitstream.AFU, region, afu) -} - -func getFpgaConfArgs(devNode string) ([]string, error) { - realDevPath, err := linux.FindSysFsDevice(devNode) - if err != nil { - return nil, err - } - if realDevPath == "" { - return nil, nil - } - for p := realDevPath; strings.HasPrefix(p, "/sys/devices/pci"); p = filepath.Dir(p) { - pciDevPath, err := filepath.EvalSymlinks(filepath.Join(p, "device")) - if err != nil { - continue - } - subs := pciAddressRE.FindStringSubmatch(filepath.Base(pciDevPath)) - if subs == nil || len(subs) != 4 { - return nil, errors.Errorf("unable to parse PCI address %s", pciDevPath) - } - return []string{"-B", "0x" + subs[1], "-D", "0x" + subs[2], "-F", "0x" + subs[3]}, nil - } - return nil, errors.Errorf("can't find PCI device address for sysfs entry %s", realDevPath) -} - -// Program programs OPAE gbs bitstream -func (bitstream *OPAEBitstream) Program(device *device.FPGADevice, execer utilsexec.Interface) error { - args, err := getFpgaConfArgs(device.DevNode) - if err != nil { - return errors.Wrapf(err, "failed get fpgaconf args for %s", device.DevNode) - } - args = append(args, bitstream.Path) - output, err := execer.Command(fpgaconf, args...).CombinedOutput() - if err != nil { - return errors.Wrapf(err, "failed to program AFU %s to device %s, region %s: output: %s", bitstream.AFU, device.DevNode, bitstream.Region, string(output)) - } - return nil -} - -// OpenCLBitstream defines parameters of .aocx file -type OpenCLBitstream struct { - Path string - Region string - AFU string -} - -// Init stub for OpenCLBitstream -func (bitstream *OpenCLBitstream) Init() error { - // Unpack .gbs and call OPAEBitstream.init() here - return nil -} - -// Validate stub for OpenCLBitstream -func (bitstream *OpenCLBitstream) Validate(region, afu string) error { - return validate(bitstream.Region, bitstream.AFU, region, afu) -} - -// Program stub for OpenCLBitstream -func (bitstream *OpenCLBitstream) Program(device *device.FPGADevice, execer utilsexec.Interface) error { - return fmt.Errorf("Not implemented") -} - // Open bitstream file, detecting type based on the filename extension. func Open(fname string) (File, error) { switch filepath.Ext(fname) { diff --git a/pkg/fpga/bitstream/gbs.go b/pkg/fpga/bitstream/gbs.go index 5c635827..c85e273a 100644 --- a/pkg/fpga/bitstream/gbs.go +++ b/pkg/fpga/bitstream/gbs.go @@ -153,7 +153,6 @@ func NewFileGBS(r bitstreamReader) (*FileGBS, error) { sr := io.NewSectionReader(r, 0, 1<<63-1) f := new(FileGBS) - // TODO: // 1. Read file header sr.Seek(0, io.SeekStart) if err := binary.Read(sr, binary.LittleEndian, &f.Header); err != nil { diff --git a/pkg/fpga/device/device.go b/pkg/fpga/device/device.go deleted file mode 100644 index d34bcde2..00000000 --- a/pkg/fpga/device/device.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2019 Intel Corporation. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package device - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - - "github.com/pkg/errors" -) - -const ( - sysfsDirectoryOPAE = "/sys/class/fpga" - interfaceIDTemplateOPAE = "/sys/class/fpga/%s/*/pr/interface_id" - afuIDTemplateOPAE = "/sys/class/fpga/intel-fpga-dev.*/%s/afu_id" - fpgaPortDevRegexOPAE = `^intel-fpga-port\.[0-9]+$` - devTemplateOPAE = "/sys/class/fpga/intel-fpga-dev.*/%s/dev" - - sysfsDirectoryDFL = "/sys/class/fpga_region" - interfaceIDTemplateDFL = "/sys/class/fpga_region/%s/*/dfl-fme-region.*/fpga_region/region*/compat_id" - afuIDTemplateDFL = "/sys/class/fpga_region/region*/%s/afu_id" - devTemplateDFL = "/sys/class/fpga_region/region*/%s/dev" - fpgaPortDevRegexDFL = `^dfl-port\.[0-9]+$` - - fpgaDevRegex = `.*/sys/class/fpga[^/]*/([^/]+)/.+$` - fpgaPortDevRegex = `.*/sys/class/fpga[^/]*/[^/]+/([^/]+)/.+$` -) - -var ( - fpgaPortDevRegDFL = regexp.MustCompile(fpgaPortDevRegexDFL) - fpgaPortDevRegOPAE = regexp.MustCompile(fpgaPortDevRegexOPAE) - fpgaDevReg = regexp.MustCompile(fpgaDevRegex) - fpgaPortDevReg = regexp.MustCompile(fpgaPortDevRegex) -) - -// FPGADevice represents FME and port(AFU) devices -type FPGADevice struct { - ID string // Interface or AFU id - Name string // basename of DevNode - SysfsPath string // path to ID file - DevNode string - Minor uint32 - Major uint32 -} - -// CanonizeID canonizes Interface and AFU ids -func CanonizeID(ID string) string { - return strings.ToLower(strings.Replace(strings.TrimSpace(ID), "-", "", -1)) -} - -// IsFPGADevice returns true if device is either DFL or OPAE device -func IsFPGADevice(deviceName string) bool { - return fpgaPortDevRegDFL.MatchString(deviceName) || fpgaPortDevRegOPAE.MatchString(deviceName) -} - -func getTemplates(sysFsPrefix string) (string, string, string, error) { - var afuIDTemplate, interfaceIDTemplate, devTemplate string - if _, err := os.Stat(path.Join(sysFsPrefix, sysfsDirectoryOPAE)); err == nil { - afuIDTemplate = afuIDTemplateOPAE - interfaceIDTemplate = interfaceIDTemplateOPAE - devTemplate = devTemplateOPAE - } else if _, err := os.Stat(path.Join(sysFsPrefix, sysfsDirectoryDFL)); err == nil { - afuIDTemplate = afuIDTemplateDFL - interfaceIDTemplate = interfaceIDTemplateDFL - devTemplate = devTemplateDFL - } else { - return "", "", "", fmt.Errorf("kernel driver is not loaded: neither %s nor %s sysfs directory is accessible", sysfsDirectoryOPAE, sysfsDirectoryDFL) - } - return path.Join(sysFsPrefix, afuIDTemplate), path.Join(sysFsPrefix, interfaceIDTemplate), path.Join(sysFsPrefix, devTemplate), nil -} - -// read file by filepath.Glob pattern -func readFileByPattern(pattern string) (string, []byte, error) { - matches, err := filepath.Glob(pattern) - if err != nil { - return "", nil, err - } - if len(matches) == 0 { - return "", nil, fmt.Errorf("no file found with pattern '%s'", pattern) - } - if len(matches) > 1 { - return "", nil, fmt.Errorf("path pattern '%s' matches multiple files", pattern) - } - data, err := ioutil.ReadFile(matches[0]) - if err != nil { - return matches[0], nil, errors.WithStack(err) - } - return matches[0], data, nil -} - -func parseDev(devData string) (uint32, uint32, error) { - devData = strings.TrimSpace(devData) - numbers := strings.SplitN(devData, ":", 2) - minor := numbers[0] - major := numbers[1] - minorInt, err := strconv.ParseInt(minor, 10, 32) - if err != nil { - return 0, 0, errors.Wrapf(err, "can't convert device minor %s to a number", minor) - } - majorInt, err := strconv.ParseInt(major, 10, 32) - if err != nil { - return 0, 0, errors.Wrapf(err, "can't convert device major %s to a number", major) - } - - return uint32(minorInt), uint32(majorInt), nil -} - -func getFPGADevice(afuIDPattern, devPattern string) (*FPGADevice, error) { - idPath, idData, err := readFileByPattern(afuIDPattern) - if err != nil { - return nil, err - } - - // Get minor:major from the dev file - _, devData, err := readFileByPattern(devPattern) - if err != nil { - return nil, err - } - - major, minor, err := parseDev(string(devData)) - if err != nil { - return nil, err - } - - // Find fpga device name - subs := fpgaPortDevReg.FindStringSubmatch(idPath) - if len(subs) < 2 { - return nil, fmt.Errorf("can't parse afu_id path: %s", idPath) - } - name := subs[1] - - return &FPGADevice{ - ID: CanonizeID(string(idData)), - Name: name, - SysfsPath: idPath, - DevNode: path.Join("/dev", name), - Minor: minor, - Major: major, - }, nil -} - -// GetAFUDevice creates FPGADevice struct for using port device name -func GetAFUDevice(sysFsPrefix, portDeviceName string) (*FPGADevice, error) { - afuIDTemplate, _, devTemplate, err := getTemplates(sysFsPrefix) - if err != nil { - return nil, err - } - return getFPGADevice(fmt.Sprintf(afuIDTemplate, portDeviceName), fmt.Sprintf(devTemplate, portDeviceName)) -} - -// GetFMEDevice reads FME properties from SysFs -func GetFMEDevice(sysFsPrefix, portDeviceName string) (*FPGADevice, error) { - portDeviceName = strings.TrimPrefix(portDeviceName, "/dev/") - afuDevice, err := GetAFUDevice(sysFsPrefix, portDeviceName) - if err != nil { - return nil, err - } - // Get top level region name from the AFU path - subs := fpgaDevReg.FindStringSubmatch(afuDevice.SysfsPath) - if len(subs) < 2 { - return nil, fmt.Errorf("can't parse sysfs path: %s", afuDevice.SysfsPath) - } - region := subs[1] - - _, interfaceIDTemplate, devTemplate, err := getTemplates(sysFsPrefix) - if err != nil { - return nil, err - } - - return getFPGADevice(fmt.Sprintf(interfaceIDTemplate, region), fmt.Sprintf(devTemplate, portDeviceName)) -} diff --git a/pkg/fpga/linux/dfl.go b/pkg/fpga/linux/dfl.go index e33ef683..2e82e9a3 100644 --- a/pkg/fpga/linux/dfl.go +++ b/pkg/fpga/linux/dfl.go @@ -18,104 +18,133 @@ package linux import ( - "os" + "math" + "path/filepath" + "strconv" "unsafe" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" "github.com/pkg/errors" ) +const ( + dflFpgaFmePrefix = "dfl-fme." + dflFpgaPortPrefix = "dfl-port." + dflFpgaFmeGlobPCI = "fpga_region/region*/dfl-fme.*" +) + // DflFME represent DFL FPGA FME device type DflFME struct { FpgaFME - DevPath string - f *os.File + DevPath string + SysFsPath string + Name string + PCIDevice *PCIDevice + SocketID string + Dev string + CompatID string + BitstreamID string + BitstreamMetadata string + PortsNum string } // Close closes open device func (f *DflFME) Close() error { - if f.f != nil { - return f.f.Close() - } + // if f.f != nil { + // return f.f.Close() + // } return nil } // NewDflFME Opens device func NewDflFME(dev string) (FpgaFME, error) { - f, err := os.OpenFile(dev, os.O_RDWR, 0644) - if err != nil { - return nil, err - } - fme := &DflFME{DevPath: dev, f: f} + fme := &DflFME{DevPath: dev} // check that kernel API is compatible if _, err := fme.GetAPIVersion(); err != nil { - fme.Close() return nil, errors.Wrap(err, "kernel API mismatch") } + if err := checkVendorAndClass(fme); err != nil { + return nil, err + } + if err := fme.updateProperties(); err != nil { + return nil, err + } return fme, nil } // DflPort represent DFL FPGA Port device type DflPort struct { FpgaPort - DevPath string - f *os.File + DevPath string + SysFsPath string + Name string + PCIDevice *PCIDevice + Dev string + AFUID string + ID string + FME FpgaFME } // Close closes open device func (f *DflPort) Close() error { - if f.f != nil { - return f.f.Close() + if f.FME != nil { + return f.FME.Close() } + // if f.f != nil { + // return f.f.Close() + // } return nil } // NewDflPort Opens device func NewDflPort(dev string) (FpgaPort, error) { - f, err := os.OpenFile(dev, os.O_RDWR, 0644) - if err != nil { - return nil, err - } - port := &DflPort{DevPath: dev, f: f} + port := &DflPort{DevPath: dev} // check that kernel API is compatible if _, err := port.GetAPIVersion(); err != nil { - port.Close() return nil, errors.Wrap(err, "kernel API mismatch") } + if err := checkVendorAndClass(port); err != nil { + return nil, err + } + if err := port.updateProperties(); err != nil { + return nil, err + } return port, nil } // common ioctls for FME and Port -func commonGetAPIVersion(fd uintptr) (int, error) { - v, err := ioctl(fd, DFL_FPGA_GET_API_VERSION, 0) +func commonDflGetAPIVersion(dev string) (int, error) { + v, err := ioctlDev(dev, DFL_FPGA_GET_API_VERSION, 0) return int(v), err } -func commonCheckExtension(fd uintptr) (int, error) { - v, err := ioctl(fd, DFL_FPGA_CHECK_EXTENSION, 0) +func commonDflCheckExtension(dev string) (int, error) { + v, err := ioctlDev(dev, DFL_FPGA_CHECK_EXTENSION, 0) return int(v), err } // GetAPIVersion Report the version of the driver API. // * Return: Driver API Version. func (f *DflFME) GetAPIVersion() (int, error) { - return commonGetAPIVersion(f.f.Fd()) + return commonDflGetAPIVersion(f.DevPath) } // CheckExtension Check whether an extension is supported. // * Return: 0 if not supported, otherwise the extension is supported. func (f *DflFME) CheckExtension() (int, error) { - return commonCheckExtension(f.f.Fd()) + // return commonCheckExtension(f.f.Fd()) + return commonDflCheckExtension(f.DevPath) } // GetAPIVersion Report the version of the driver API. // * Return: Driver API Version. func (f *DflPort) GetAPIVersion() (int, error) { - return commonGetAPIVersion(f.f.Fd()) + return commonDflGetAPIVersion(f.DevPath) } // CheckExtension Check whether an extension is supported. // * Return: 0 if not supported, otherwise the extension is supported. func (f *DflPort) CheckExtension() (int, error) { - return commonCheckExtension(f.f.Fd()) + return commonDflCheckExtension(f.DevPath) } // PortReset Reset the FPGA Port and its AFU. No parameters are supported. @@ -124,7 +153,7 @@ func (f *DflPort) CheckExtension() (int, error) { // (e.g. DMA or PR operation failure) and be recoverable from the failure. // * Return: 0 on success, -errno of failure func (f *DflPort) PortReset() error { - _, err := ioctl(f.f.Fd(), DFL_FPGA_PORT_RESET, 0) + _, err := ioctlDev(f.DevPath, DFL_FPGA_PORT_RESET, 0) return err } @@ -134,7 +163,7 @@ func (f *DflPort) PortReset() error { func (f *DflPort) PortGetInfo() (ret FpgaPortInfo, err error) { var value DflFpgaPortInfo value.Argsz = uint32(unsafe.Sizeof(value)) - _, err = ioctl(f.f.Fd(), DFL_FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) + _, err = ioctlDev(f.DevPath, DFL_FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) if err == nil { ret.Flags = value.Flags ret.Regions = value.Regions @@ -152,7 +181,7 @@ func (f *DflPort) PortGetRegionInfo(index uint32) (ret FpgaPortRegionInfo, err e var value DflFpgaPortRegionInfo value.Argsz = uint32(unsafe.Sizeof(value)) value.Index = index - _, err = ioctl(f.f.Fd(), DFL_FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) + _, err = ioctlDev(f.DevPath, DFL_FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) if err == nil { ret.Flags = value.Flags ret.Index = value.Index @@ -174,6 +203,211 @@ func (f *DflFME) PortPR(port uint32, bitstream []byte) error { value.Port_id = port value.Buffer_size = uint32(len(bitstream)) value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0]))) - _, err := ioctl(f.f.Fd(), DFL_FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) + _, err := ioctlDev(f.DevPath, DFL_FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) return err } + +// FME interfaces + +// GetDevPath returns path to device node +func (f *DflFME) GetDevPath() string { + return f.DevPath +} + +// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items) +func (f *DflFME) GetSysFsPath() string { + if f.SysFsPath != "" { + return f.SysFsPath + } + sysfs, err := FindSysFsDevice(f.DevPath) + if err != nil { + return "" + } + f.SysFsPath = sysfs + return f.SysFsPath +} + +// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/ +func (f *DflFME) GetName() string { + if f.Name != "" { + return f.Name + } + f.Name = filepath.Base(f.GetSysFsPath()) + return f.Name +} + +// GetPCIDevice returns PCIDevice for this device +func (f *DflFME) GetPCIDevice() (*PCIDevice, error) { + if f.PCIDevice != nil { + return f.PCIDevice, nil + } + pci, err := NewPCIDevice(f.GetSysFsPath()) + if err != nil { + return nil, err + } + f.PCIDevice = pci + return f.PCIDevice, nil +} + +// GetPortsNum returns amount of FPGA Ports associated to this FME +func (f *DflFME) GetPortsNum() int { + if f.PortsNum == "" { + err := f.updateProperties() + if err != nil { + return -1 + } + } + n, err := strconv.ParseUint(f.PortsNum, 10, 32) + if err != nil { + return -1 + } + return int(n) +} + +// GetInterfaceUUID returns Interface UUID for FME +func (f *DflFME) GetInterfaceUUID() (id string) { + if f.CompatID == "" { + err := f.updateProperties() + if err != nil { + return "" + } + } + return f.CompatID +} + +// Update properties from sysfs +func (f *DflFME) updateProperties() error { + pci, err := f.GetPCIDevice() + if err != nil { + return err + } + fileMap := map[string]*string{ + "bitstream_id": &f.BitstreamID, + "bitstream_metadata": &f.BitstreamMetadata, + "dev": &f.Dev, + "ports_num": &f.PortsNum, + "socket_id": &f.SocketID, + "dfl-fme-region.*/fpga_region/region*/compat_id": &f.CompatID, + } + return readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, dflFpgaFmeGlobPCI)) +} + +// Port interfaces + +// GetDevPath returns path to device node +func (f *DflPort) GetDevPath() string { + return f.DevPath +} + +// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items) +func (f *DflPort) GetSysFsPath() string { + if f.SysFsPath != "" { + return f.SysFsPath + } + sysfs, err := FindSysFsDevice(f.DevPath) + if err != nil { + return "" + } + f.SysFsPath = sysfs + return f.SysFsPath +} + +// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/ +func (f *DflPort) GetName() string { + if f.Name != "" { + return f.Name + } + f.Name = filepath.Base(f.GetSysFsPath()) + return f.Name +} + +// GetPCIDevice returns PCIDevice for this device +func (f *DflPort) GetPCIDevice() (*PCIDevice, error) { + if f.PCIDevice != nil { + return f.PCIDevice, nil + } + pci, err := NewPCIDevice(f.GetSysFsPath()) + if err != nil { + return nil, err + } + f.PCIDevice = pci + return f.PCIDevice, nil +} + +// GetFME returns FPGA FME device for this port +func (f *DflPort) GetFME() (fme FpgaFME, err error) { + if f.FME != nil { + return f.FME, nil + } + pci, err := f.GetPCIDevice() + if err != nil { + return + } + if pci.PhysFn != nil { + pci = pci.PhysFn + } + + var dev string + fileMap := map[string]*string{ + "dev": &dev, + } + if err = readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, dflFpgaFmeGlobPCI)); err != nil { + return + } + realDev, err := filepath.EvalSymlinks(filepath.Join("/dev/char", dev)) + if err != nil { + return + } + fme, err = NewDflFME(realDev) + if err != nil { + return + } + f.FME = fme + return +} + +// GetPortID returns ID of the FPGA port within physical device +func (f *DflPort) GetPortID() (uint32, error) { + if f.ID == "" { + err := f.updateProperties() + if err != nil { + return math.MaxUint32, err + } + } + id, err := strconv.ParseUint(f.ID, 10, 32) + return uint32(id), err +} + +// GetAcceleratorTypeUUID returns AFU UUID for port +func (f *DflPort) GetAcceleratorTypeUUID() (afuID string) { + err := f.updateProperties() + if err != nil || f.AFUID == "" { + return "" + } + return f.AFUID +} + +// GetInterfaceUUID returns Interface UUID for FME +func (f *DflPort) GetInterfaceUUID() (id string) { + fme, err := f.GetFME() + if err != nil { + return "" + } + defer fme.Close() + return fme.GetInterfaceUUID() +} + +// PR programs specified bitstream to port +func (f *DflPort) PR(bs bitstream.File, dryRun bool) error { + return genericPortPR(f, bs, dryRun) +} + +// Update properties from sysfs +func (f *DflPort) updateProperties() error { + fileMap := map[string]*string{ + "afu_id": &f.AFUID, + "dev": &f.Dev, + "id": &f.ID, + } + return readFilesInDirectory(fileMap, f.GetSysFsPath()) +} diff --git a/pkg/fpga/linux/fpga.go b/pkg/fpga/linux/fpga.go new file mode 100644 index 00000000..f601da90 --- /dev/null +++ b/pkg/fpga/linux/fpga.go @@ -0,0 +1,117 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +build linux +// + +package linux + +import ( + "io/ioutil" + "path/filepath" + "strings" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" + + "github.com/pkg/errors" +) + +// IsFpgaFME returns true if the name looks like any supported FME device +func IsFpgaFME(name string) bool { + devName := cleanBasename(name) + return strings.HasPrefix(devName, dflFpgaFmePrefix) || strings.HasPrefix(devName, intelFpgaFmePrefix) +} + +// IsFpgaPort returns true if the name looks like any supported FME device +func IsFpgaPort(name string) bool { + devName := cleanBasename(name) + return strings.HasPrefix(devName, dflFpgaPortPrefix) || strings.HasPrefix(devName, intelFpgaPortPrefix) +} + +// CanonizeID canonizes Interface and AFU ids +func CanonizeID(ID string) string { + return strings.ToLower(strings.Replace(strings.TrimSpace(ID), "-", "", -1)) +} + +// NewFpgaPort returns FpgaPort for specified device node +func NewFpgaPort(fname string) (FpgaPort, error) { + if strings.IndexByte(fname, byte('/')) < 0 { + fname = filepath.Join("/dev", fname) + } + devName := cleanBasename(fname) + switch { + case strings.HasPrefix(devName, dflFpgaPortPrefix): + return NewDflPort(fname) + case strings.HasPrefix(devName, intelFpgaPortPrefix): + return NewIntelFpgaPort(fname) + } + return nil, errors.Errorf("unknown type of FPGA port %s", devName) +} + +// NewFpgaFME returns FpgaFME for specified device node +func NewFpgaFME(fname string) (FpgaFME, error) { + if strings.IndexByte(fname, byte('/')) < 0 { + fname = filepath.Join("/dev", fname) + } + devName := cleanBasename(fname) + switch { + case strings.HasPrefix(devName, dflFpgaFmePrefix): + return NewDflFME(fname) + case strings.HasPrefix(devName, intelFpgaFmePrefix): + return NewIntelFpgaFME(fname) + } + return nil, errors.Errorf("unknown type of FPGA FME %s", devName) +} + +// ListFpgaDevices returns two lists of FPGA device nodes: FMEs and Ports +func ListFpgaDevices() (FMEs, Ports []string) { + files, err := ioutil.ReadDir("/sys/bus/platform/devices") + if err != nil { + return + } + for _, file := range files { + fname := file.Name() + switch { + case IsFpgaFME(fname): + FMEs = append(FMEs, fname) + case IsFpgaPort(fname): + Ports = append(Ports, fname) + } + } + return +} + +func genericPortPR(f FpgaPort, bs bitstream.File, dryRun bool) error { + fme, err := f.GetFME() + if err != nil { + return err + } + ifID := fme.GetInterfaceUUID() + bsID := bs.InterfaceUUID() + if ifID != bsID { + return errors.Errorf("FME interface UUID %q is not compatible with bitstream UUID %q ", ifID, bsID) + } + pNum, err := f.GetPortID() + if err != nil { + return err + } + rawBistream, err := bs.RawBitstreamData() + if err != nil { + return err + } + if dryRun { + return nil + } + return fme.PortPR(pNum, rawBistream) +} diff --git a/pkg/fpga/linux/intel-fpga.go b/pkg/fpga/linux/intel-fpga.go index ff90e567..e2cda365 100644 --- a/pkg/fpga/linux/intel-fpga.go +++ b/pkg/fpga/linux/intel-fpga.go @@ -18,104 +18,129 @@ package linux import ( - "os" + "math" + "path/filepath" + "strconv" "unsafe" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" "github.com/pkg/errors" ) +const ( + intelFpgaFmePrefix = "intel-fpga-fme." + intelFpgaPortPrefix = "intel-fpga-port." + intelFpgaFmeGlobPCI = "fpga/intel-fpga-dev.*/intel-fpga-fme.*" +) + // IntelFpgaFME represent Intel FPGA FME device type IntelFpgaFME struct { FpgaFME - DevPath string - f *os.File + DevPath string + SysFsPath string + Name string + PCIDevice *PCIDevice + SocketID string + Dev string + CompatID string + BitstreamID string + BitstreamMetadata string + PortsNum string } // Close closes open device func (f *IntelFpgaFME) Close() error { - if f.f != nil { - return f.f.Close() - } return nil } // NewIntelFpgaFME Opens device func NewIntelFpgaFME(dev string) (FpgaFME, error) { - f, err := os.OpenFile(dev, os.O_RDWR, 0644) - if err != nil { - return nil, err - } - fme := &IntelFpgaFME{DevPath: dev, f: f} + fme := &IntelFpgaFME{DevPath: dev} // check that kernel API is compatible if _, err := fme.GetAPIVersion(); err != nil { - fme.Close() return nil, errors.Wrap(err, "kernel API mismatch") } + if err := checkVendorAndClass(fme); err != nil { + return nil, err + } + if err := fme.updateProperties(); err != nil { + return nil, err + } return fme, nil } // IntelFpgaPort represent IntelFpga FPGA Port device type IntelFpgaPort struct { FpgaPort - DevPath string - f *os.File + DevPath string + SysFsPath string + Name string + PCIDevice *PCIDevice + Dev string + AFUID string + ID string + FME FpgaFME } // Close closes open device func (f *IntelFpgaPort) Close() error { - if f.f != nil { - return f.f.Close() + if f.FME != nil { + defer f.FME.Close() } return nil } // NewIntelFpgaPort Opens device func NewIntelFpgaPort(dev string) (FpgaPort, error) { - f, err := os.OpenFile(dev, os.O_RDWR, 0644) - if err != nil { - return nil, err - } - port := &IntelFpgaPort{DevPath: dev, f: f} + port := &IntelFpgaPort{DevPath: dev} // check that kernel API is compatible if _, err := port.GetAPIVersion(); err != nil { port.Close() return nil, errors.Wrap(err, "kernel API mismatch") } + if err := checkVendorAndClass(port); err != nil { + port.Close() + return nil, err + } + if err := port.updateProperties(); err != nil { + port.Close() + return nil, err + } return port, nil } // common ioctls for FME and Port -func commonIntelFpgaGetAPIVersion(fd uintptr) (int, error) { - v, err := ioctl(fd, FPGA_GET_API_VERSION, 0) +func commonIntelFpgaGetAPIVersion(fd string) (int, error) { + v, err := ioctlDev(fd, FPGA_GET_API_VERSION, 0) return int(v), err } -func commonIntelFpgaCheckExtension(fd uintptr) (int, error) { - v, err := ioctl(fd, FPGA_CHECK_EXTENSION, 0) +func commonIntelFpgaCheckExtension(fd string) (int, error) { + v, err := ioctlDev(fd, FPGA_CHECK_EXTENSION, 0) return int(v), err } // GetAPIVersion Report the version of the driver API. // * Return: Driver API Version. func (f *IntelFpgaFME) GetAPIVersion() (int, error) { - return commonIntelFpgaGetAPIVersion(f.f.Fd()) + return commonIntelFpgaGetAPIVersion(f.DevPath) } // CheckExtension Check whether an extension is supported. // * Return: 0 if not supported, otherwise the extension is supported. func (f *IntelFpgaFME) CheckExtension() (int, error) { - return commonIntelFpgaCheckExtension(f.f.Fd()) + return commonIntelFpgaCheckExtension(f.DevPath) } // GetAPIVersion Report the version of the driver API. // * Return: Driver API Version. func (f *IntelFpgaPort) GetAPIVersion() (int, error) { - return commonIntelFpgaGetAPIVersion(f.f.Fd()) + return commonIntelFpgaGetAPIVersion(f.DevPath) } // CheckExtension Check whether an extension is supported. // * Return: 0 if not supported, otherwise the extension is supported. func (f *IntelFpgaPort) CheckExtension() (int, error) { - return commonIntelFpgaCheckExtension(f.f.Fd()) + return commonIntelFpgaCheckExtension(f.DevPath) } // PortReset Reset the FPGA Port and its AFU. No parameters are supported. @@ -124,7 +149,7 @@ func (f *IntelFpgaPort) CheckExtension() (int, error) { // (e.g. DMA or PR operation failure) and be recoverable from the failure. // * Return: 0 on success, -errno of failure func (f *IntelFpgaPort) PortReset() error { - _, err := ioctl(f.f.Fd(), FPGA_PORT_RESET, 0) + _, err := ioctlDev(f.DevPath, FPGA_PORT_RESET, 0) return err } @@ -134,7 +159,7 @@ func (f *IntelFpgaPort) PortReset() error { func (f *IntelFpgaPort) PortGetInfo() (ret FpgaPortInfo, err error) { var value IntelFpgaPortInfo value.Argsz = uint32(unsafe.Sizeof(value)) - _, err = ioctl(f.f.Fd(), FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) + _, err = ioctlDev(f.DevPath, FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) if err == nil { ret.Flags = value.Flags ret.Regions = value.Regions @@ -152,7 +177,7 @@ func (f *IntelFpgaPort) PortGetRegionInfo(index uint32) (ret FpgaPortRegionInfo, var value IntelFpgaPortRegionInfo value.Argsz = uint32(unsafe.Sizeof(value)) value.Index = index - _, err = ioctl(f.f.Fd(), FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) + _, err = ioctlDev(f.DevPath, FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) if err == nil { ret.Flags = value.Flags ret.Index = value.Index @@ -174,6 +199,211 @@ func (f *IntelFpgaFME) PortPR(port uint32, bitstream []byte) error { value.Port_id = port value.Buffer_size = uint32(len(bitstream)) value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0]))) - _, err := ioctl(f.f.Fd(), FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) + _, err := ioctlDev(f.DevPath, FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) return err } + +// FME interfaces + +// GetDevPath returns path to device node +func (f *IntelFpgaFME) GetDevPath() string { + return f.DevPath +} + +// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items) +func (f *IntelFpgaFME) GetSysFsPath() string { + if f.SysFsPath != "" { + return f.SysFsPath + } + sysfs, err := FindSysFsDevice(f.DevPath) + if err != nil { + return "" + } + f.SysFsPath = sysfs + return f.SysFsPath +} + +// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/ +func (f *IntelFpgaFME) GetName() string { + if f.Name != "" { + return f.Name + } + f.Name = filepath.Base(f.GetSysFsPath()) + return f.Name +} + +// GetPCIDevice returns PCIDevice for this device +func (f *IntelFpgaFME) GetPCIDevice() (*PCIDevice, error) { + if f.PCIDevice != nil { + return f.PCIDevice, nil + } + pci, err := NewPCIDevice(f.GetSysFsPath()) + if err != nil { + return nil, err + } + f.PCIDevice = pci + return f.PCIDevice, nil +} + +// GetPortsNum returns amount of FPGA Ports associated to this FME +func (f *IntelFpgaFME) GetPortsNum() int { + if f.PortsNum == "" { + err := f.updateProperties() + if err != nil { + return -1 + } + } + n, err := strconv.ParseUint(f.PortsNum, 10, 32) + if err != nil { + return -1 + } + return int(n) +} + +// GetInterfaceUUID returns Interface UUID for FME +func (f *IntelFpgaFME) GetInterfaceUUID() (id string) { + if f.CompatID == "" { + err := f.updateProperties() + if err != nil { + return "" + } + } + return f.CompatID +} + +// Update properties from sysfs +func (f *IntelFpgaFME) updateProperties() error { + pci, err := f.GetPCIDevice() + if err != nil { + return err + } + fileMap := map[string]*string{ + "bitstream_id": &f.BitstreamID, + "bitstream_metadata": &f.BitstreamMetadata, + "dev": &f.Dev, + "ports_num": &f.PortsNum, + "socket_id": &f.SocketID, + "pr/interface_id": &f.CompatID, + } + return readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, intelFpgaFmeGlobPCI)) +} + +// Port interfaces + +// GetDevPath returns path to device node +func (f *IntelFpgaPort) GetDevPath() string { + return f.DevPath +} + +// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items) +func (f *IntelFpgaPort) GetSysFsPath() string { + if f.SysFsPath != "" { + return f.SysFsPath + } + sysfs, err := FindSysFsDevice(f.DevPath) + if err != nil { + return "" + } + f.SysFsPath = sysfs + return f.SysFsPath +} + +// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/ +func (f *IntelFpgaPort) GetName() string { + if f.Name != "" { + return f.Name + } + f.Name = filepath.Base(f.GetSysFsPath()) + return f.Name +} + +// GetPCIDevice returns PCIDevice for this device +func (f *IntelFpgaPort) GetPCIDevice() (*PCIDevice, error) { + if f.PCIDevice != nil { + return f.PCIDevice, nil + } + pci, err := NewPCIDevice(f.GetSysFsPath()) + if err != nil { + return nil, err + } + f.PCIDevice = pci + return f.PCIDevice, nil +} + +// GetFME returns FPGA FME device for this port +func (f *IntelFpgaPort) GetFME() (fme FpgaFME, err error) { + if f.FME != nil { + return f.FME, nil + } + pci, err := f.GetPCIDevice() + if err != nil { + return + } + if pci.PhysFn != nil { + pci = pci.PhysFn + } + + var dev string + fileMap := map[string]*string{ + "dev": &dev, + } + if err = readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, intelFpgaFmeGlobPCI)); err != nil { + return + } + realDev, err := filepath.EvalSymlinks(filepath.Join("/dev/char", dev)) + if err != nil { + return + } + fme, err = NewIntelFpgaFME(realDev) + if err != nil { + return + } + f.FME = fme + return +} + +// GetPortID returns ID of the FPGA port within physical device +func (f *IntelFpgaPort) GetPortID() (uint32, error) { + if f.ID == "" { + err := f.updateProperties() + if err != nil { + return math.MaxUint32, err + } + } + id, err := strconv.ParseUint(f.ID, 10, 32) + return uint32(id), err +} + +// GetAcceleratorTypeUUID returns AFU UUID for port +func (f *IntelFpgaPort) GetAcceleratorTypeUUID() string { + err := f.updateProperties() + if err != nil || f.AFUID == "" { + return "" + } + return f.AFUID +} + +// GetInterfaceUUID returns Interface UUID for FME +func (f *IntelFpgaPort) GetInterfaceUUID() (id string) { + fme, err := f.GetFME() + if err != nil { + return "" + } + defer fme.Close() + return fme.GetInterfaceUUID() +} + +// PR programs specified bitstream to port +func (f *IntelFpgaPort) PR(bs bitstream.File, dryRun bool) error { + return genericPortPR(f, bs, dryRun) +} + +// Update properties from sysfs +func (f *IntelFpgaPort) updateProperties() error { + fileMap := map[string]*string{ + "afu_id": &f.AFUID, + "dev": &f.Dev, + "id": &f.ID, + } + return readFilesInDirectory(fileMap, f.GetSysFsPath()) +} diff --git a/pkg/fpga/linux/interfaces.go b/pkg/fpga/linux/interfaces.go index 6bdabd02..002ae32d 100644 --- a/pkg/fpga/linux/interfaces.go +++ b/pkg/fpga/linux/interfaces.go @@ -17,9 +17,13 @@ package linux -import "io" +import ( + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" + "io" +) -type commonAPI interface { +type commonFpgaAPI interface { + // Generic interfaces provided by FPGA ports and FMEs io.Closer // GetAPIVersion Report the version of the driver API. // * Return: Driver API Version. @@ -27,11 +31,23 @@ type commonAPI interface { // CheckExtension Check whether an extension is supported. // * Return: 0 if not supported, otherwise the extension is supported. CheckExtension() (int, error) + + // Interfaces for device discovery and accessing properties + + // GetDevPath returns path to device node + GetDevPath() string + // GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items) + GetSysFsPath() string + // GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/ + GetName() string + // GetPCIDevice returns PCIDevice for this device + GetPCIDevice() (*PCIDevice, error) } // FpgaFME represent interfaces provided by management interface of FPGA type FpgaFME interface { - commonAPI + // Kernel IOCTL interfaces for FPGA ports: + commonFpgaAPI // PortPR does Partial Reconfiguration based on Port ID and Buffer (Image) // provided by caller. // * Return: 0 on success, -errno on failure. @@ -39,16 +55,25 @@ type FpgaFME interface { // some errors during PR, under this case, the user can fetch HW error info // from the status of FME's fpga manager. PortPR(uint32, []byte) error - - // TODO: + // TODO: (not implemented IOCTLs) // Port release / assign // Get Info // Set IRQ err + + // Interfaces for device discovery and accessing properties + + // GetPortsNum returns amount of FPGA Ports associated to this FME + GetPortsNum() int + // InterfaceUUID returns Interface UUID for FME + GetInterfaceUUID() string + // GetPort returns FpgaPort of the desired FPGA port within that FME + // GetPort(uint32) (FpgaPort, error) } // FpgaPort represent interfaces provided by AFU port of FPGA type FpgaPort interface { - commonAPI + // Kernel IOCTL interfaces for FPGA ports: + commonFpgaAPI // PortReset Reset the FPGA Port and its AFU. No parameters are supported. // Userspace can do Port reset at any time, e.g. during DMA or PR. But // it should never cause any system level issue, only functional failure @@ -65,11 +90,23 @@ type FpgaPort interface { // * Driver returns the region info in other fields. // * Return: 0 on success, -errno on failure. PortGetRegionInfo(index uint32) (FpgaPortRegionInfo, error) - - // TODO: + // TODO: (not implemented IOCTLs) // Port DMA map / unmap // UMSG enable / disable / set-mode / set-base-addr (intel-fpga) // Set IRQ: err, uafu (intel-fpga) + + // Interfaces for device discovery and accessing properties + + // GetFME returns FPGA FME device for this port + GetFME() (FpgaFME, error) + // GetPortID returns ID of the FPGA port within physical device + GetPortID() (uint32, error) + // GetAcceleratorTypeUUID returns AFU UUID for port + GetAcceleratorTypeUUID() string + // InterfaceUUID returns Interface UUID for FME + GetInterfaceUUID() string + // PR programs specified bitstream to port + PR(bitstream.File, bool) error } // FpgaPortInfo is a unified port info between drivers diff --git a/pkg/fpga/linux/ioctl.go b/pkg/fpga/linux/ioctl.go index 49f88494..fc9af466 100644 --- a/pkg/fpga/linux/ioctl.go +++ b/pkg/fpga/linux/ioctl.go @@ -18,6 +18,7 @@ package linux import ( + "os" "syscall" ) @@ -28,3 +29,13 @@ func ioctl(fd uintptr, req uint, arg uintptr) (ret uintptr, err error) { } return } + +// Same as above, but open device only for single operation +func ioctlDev(dev string, req uint, arg uintptr) (ret uintptr, err error) { + f, err := os.OpenFile(dev, os.O_RDWR, 0644) + if err != nil { + return + } + defer f.Close() + return ioctl(f.Fd(), req, arg) +} diff --git a/pkg/fpga/linux/pci.go b/pkg/fpga/linux/pci.go index e08a2267..1367e03e 100644 --- a/pkg/fpga/linux/pci.go +++ b/pkg/fpga/linux/pci.go @@ -32,6 +32,8 @@ import ( const ( pciAddressRegex = `^([[:xdigit:]]{4}):([[:xdigit:]]{2}):([[:xdigit:]]{2})\.([[:xdigit:]])$` + vendorIntel = "0x8086" + fpgaClass = "0x120000" ) var ( diff --git a/pkg/fpga/linux/utils.go b/pkg/fpga/linux/utils.go index f3b09d0e..61348619 100644 --- a/pkg/fpga/linux/utils.go +++ b/pkg/fpga/linux/utils.go @@ -15,17 +15,32 @@ package linux import ( - "github.com/pkg/errors" "io/ioutil" "os" "path/filepath" "strings" + + "github.com/pkg/errors" ) // small helper function that reads several files into provided set of variables func readFilesInDirectory(fileMap map[string]*string, dir string) error { for k, v := range fileMap { - b, err := ioutil.ReadFile(filepath.Join(dir, k)) + fname := filepath.Join(dir, k) + if strings.ContainsAny(fname, "?*[") { + // path contains wildcards, let's find by Glob needed file. + files, err := filepath.Glob(fname) + switch { + case err != nil: + continue + case len(files) != 1: + // doesn't match unique file, skip it + // fmt.Println("KAD2: ", files) + continue + } + fname = files[0] + } + b, err := ioutil.ReadFile(fname) if err != nil { if os.IsNotExist(err) { continue @@ -36,3 +51,24 @@ func readFilesInDirectory(fileMap map[string]*string, dir string) error { } return nil } + +// returns filename of the argument after resolving symlinks +func cleanBasename(name string) string { + realPath, err := filepath.EvalSymlinks(name) + if err != nil { + realPath = name + } + return filepath.Base(realPath) +} + +// check that FPGA device is a compatible PCI device +func checkVendorAndClass(dev commonFpgaAPI) error { + pci, err := dev.GetPCIDevice() + if err != nil { + return err + } + if pci.Vendor != vendorIntel || pci.Class != fpgaClass { + return errors.Errorf("unsupported PCI device %s VID=%s PID=%s Class=%s", pci.BDF, pci.Vendor, pci.Device, pci.Class) + } + return nil +}