github/intel-device-plugins-for-kubernetes
7
0
Fork 0
mirror of https://github.com/intel/intel-device-plugins-for-kubernetes.git synced 2025-06-03 03:59:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented native FPGA flashing

Browse Source 
Removed dependency to OPAE libraries
This commit is contained in:
Alexander Kanevskiy 2019-08-24 01:24:33 +03:00
parent 04be52e0a1
commit 71bb38f496
13 changed files with 796 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
cmd/fpga_crihook/main.go
View File

@ -23,7 +23,7 @@ import (
"strings" "strings"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device" fpga "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux"
"github.com/pkg/errors" "github.com/pkg/errors"
utilsexec "k8s.io/utils/exec" utilsexec "k8s.io/utils/exec"
) )
@ -144,10 +144,10 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) {
splitted := strings.SplitN(env, "=", 2) splitted := strings.SplitN(env, "=", 2)
if strings.HasPrefix(splitted[0], fpgaRegionEnvPrefix) { if strings.HasPrefix(splitted[0], fpgaRegionEnvPrefix) {
num := strings.Split(splitted[0], fpgaRegionEnvPrefix)[1] num := strings.Split(splitted[0], fpgaRegionEnvPrefix)[1]
regionEnv[num] = device.CanonizeID(splitted[1]) regionEnv[num] = fpga.CanonizeID(splitted[1])
} else if strings.HasPrefix(splitted[0], fpgaAfuEnvPrefix) { } else if strings.HasPrefix(splitted[0], fpgaAfuEnvPrefix) {
num := strings.Split(splitted[0], fpgaAfuEnvPrefix)[1] num := strings.Split(splitted[0], fpgaAfuEnvPrefix)[1]
afuEnv[num] = device.CanonizeID(splitted[1]) afuEnv[num] = fpga.CanonizeID(splitted[1])
} }
} }
@ -175,7 +175,7 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) {
for _, dev := range config.Linux.Devices { for _, dev := range config.Linux.Devices {
deviceName := dev.getName() deviceName := dev.getName()
// skip non-FPGA devices // skip non-FPGA devices
if !device.IsFPGADevice(deviceName) { if !fpga.IsFpgaPort(deviceName) {
continue continue
} }
@ -183,16 +183,15 @@ func (he *hookEnv) getFPGAParams(config *Config) ([]fpgaParams, error) {
if dev.processed { if dev.processed {
continue continue
} }
fme, err := device.GetFMEDevice(he.sysFsPrefix, deviceName) port, err := fpga.NewFpgaPort(deviceName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if interfaceUUID := port.GetInterfaceUUID(); interfaceUUID == region {
if fme.ID == region {
params = append(params, params = append(params,
fpgaParams{ fpgaParams{
afu: afu, afu: afu,
region: fme.ID, region: interfaceUUID,
portDevice: deviceName, portDevice: deviceName,
}, },
) )
@ -253,12 +252,13 @@ func (he *hookEnv) process(reader io.Reader) error {
} }
for _, params := range paramslist { for _, params := range paramslist {
programmedAfu, err := device.GetAFUDevice(he.sysFsPrefix, params.portDevice) port, err := fpga.NewFpgaPort(params.portDevice)
if err != nil { if err != nil {
return err return err
} }
if programmedAfu.ID == params.afu { programmedAfu := port.GetAcceleratorTypeUUID()
if programmedAfu == params.afu {
// Afu is already programmed // Afu is already programmed
return nil return nil
} }
@ -267,34 +267,14 @@ func (he *hookEnv) process(reader io.Reader) error {
if err != nil { if err != nil {
return err return err
} }
defer bitstream.Close()
err = bitstream.Init() err = port.PR(bitstream, false)
if err != nil {
return err
}
err = bitstream.Validate(params.region, params.afu) programmedAfu = port.GetAcceleratorTypeUUID()
if err != nil {
return err
}
fme, err := device.GetFMEDevice(he.sysFsPrefix, params.portDevice) if programmedAfu != bitstream.AcceleratorTypeUUID() {
if err != nil { return errors.Errorf("programmed function %s instead of %s", programmedAfu, bitstream.AcceleratorTypeUUID())
return err
}
err = bitstream.Program(fme, he.execer)
if err != nil {
return err
}
programmedAfu, err = device.GetAFUDevice(he.sysFsPrefix, params.portDevice)
if err != nil {
return err
}
if programmedAfu.ID != params.afu {
return errors.Errorf("programmed function %s instead of %s", programmedAfu.ID, params.afu)
} }
} }

87
cmd/fpga_tool/fpga_tool.go
View File

@ -21,12 +21,10 @@ import (
"log" "log"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device"
fpga "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux" fpga "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux"
) )
@ -106,16 +104,7 @@ func validateFlags(cmd, bitstream, device string) error {
// WIP testing command // WIP testing command
func magic(dev string) (err error) { func magic(dev string) (err error) {
d, err := device.GetFMEDevice("", dev) fmt.Println(fpga.ListFpgaDevices())
fmt.Printf("%+v %+v\n", d, err)
d1, err := fpga.FindSysFsDevice(dev)
fmt.Printf("%+v %+v\n", d1, err)
if err != nil {
return
}
d2, err := fpga.NewPCIDevice(d1)
fmt.Printf("%+v %+v\n", d2, err)
return return
} }
@ -153,9 +142,9 @@ func installBitstream(fname string, dryRun bool) (err error) {
func fpgaInfo(fname string) error { func fpgaInfo(fname string) error {
switch { switch {
case strings.HasPrefix(fname, "/dev/dfl-fme."), strings.HasPrefix(fname, "/dev/intel-fpga-fme."): case fpga.IsFpgaFME(fname):
return fmeInfo(fname) return fmeInfo(fname)
case strings.HasPrefix(fname, "/dev/dfl-port."), strings.HasPrefix(fname, "/dev/intel-fpga-port."): case fpga.IsFpgaPort(fname):
return portInfo(fname) return portInfo(fname)
} }
return errors.Errorf("unknown FPGA device file %s", fname) return errors.Errorf("unknown FPGA device file %s", fname)
@ -185,14 +174,7 @@ func printBitstreamInfo(fname string) (err error) {
func fmeInfo(fname string) error { func fmeInfo(fname string) error {
var f fpga.FpgaFME var f fpga.FpgaFME
var err error var err error
switch { f, err = fpga.NewFpgaFME(fname)
case strings.HasPrefix(fname, "/dev/dfl-fme."):
f, err = fpga.NewDflFME(fname)
case strings.HasPrefix(fname, "/dev/intel-fpga-fme."):
f, err = fpga.NewIntelFpgaFME(fname)
default:
return errors.Errorf("unknow type of FME %s", fname)
}
if err != nil { if err != nil {
return err return err
} }
@ -201,20 +183,21 @@ func fmeInfo(fname string) error {
fmt.Println(f.GetAPIVersion()) fmt.Println(f.GetAPIVersion())
fmt.Print("CheckExtension:") fmt.Print("CheckExtension:")
fmt.Println(f.CheckExtension()) fmt.Println(f.CheckExtension())
fmt.Println("GetDevPath: ", f.GetDevPath())
fmt.Println("GetSysFsPath: ", f.GetSysFsPath())
fmt.Println("GetName: ", f.GetName())
pci, err := f.GetPCIDevice()
fmt.Printf("GetPCIDevice: %+v %+v\n", pci, err)
fmt.Println("GetInterfaceUUID: ", f.GetInterfaceUUID())
fmt.Println("GetPortNums: ", f.GetPortsNum())
return nil return nil
} }
func portInfo(fname string) error { func portInfo(fname string) error {
var f fpga.FpgaPort var f fpga.FpgaPort
var err error var err error
switch { f, err = fpga.NewFpgaPort(fname)
case strings.HasPrefix(fname, "/dev/dfl-port."):
f, err = fpga.NewDflPort(fname)
case strings.HasPrefix(fname, "/dev/intel-fpga-port."):
f, err = fpga.NewIntelFpgaPort(fname)
default:
err = errors.Errorf("unknown type of port %s", fname)
}
if err != nil { if err != nil {
return err return err
} }
@ -234,42 +217,38 @@ func portInfo(fname string) error {
fmt.Println(f.PortGetRegionInfo(uint32(idx))) fmt.Println(f.PortGetRegionInfo(uint32(idx)))
} }
} }
fmt.Println("GetDevPath: ", f.GetDevPath())
fmt.Println("GetSysFsPath: ", f.GetSysFsPath())
fmt.Println("GetName: ", f.GetName())
pci, err := f.GetPCIDevice()
fmt.Printf("GetPCIDevice: %+v %+v\n", pci, err)
id, err := f.GetPortID()
fmt.Printf("GetPort: %+v %+v\n", id, err)
fmt.Println("GetAcceleratorTypeUUID: ", f.GetAcceleratorTypeUUID())
fmt.Println("GetInterfaceUUID: ", f.GetInterfaceUUID())
fme, err := f.GetFME()
fmt.Printf("GetFME: %+v %+v\n", fme, err)
return nil return nil
} }
func doPR(fme, bs string, dryRun bool) error { func doPR(dev, bs string, dryRun bool) error {
var f fpga.FpgaFME
var err error f, err := fpga.NewFpgaPort(dev)
switch {
case strings.HasPrefix(fme, "/dev/dfl-fme."):
f, err = fpga.NewDflFME(fme)
case strings.HasPrefix(fme, "/dev/intel-fpga-fme."):
f, err = fpga.NewIntelFpgaFME(fme)
default:
return errors.Errorf("unknown FME %s", fme)
}
fmt.Printf("Trying to program %s to port 0 of %s", bs, fme)
if err != nil { if err != nil {
return err return err
} }
defer f.Close() defer f.Close()
fmt.Print("API:")
fmt.Println(f.GetAPIVersion())
m, err := bitstream.Open(bs) m, err := bitstream.Open(bs)
if err != nil { if err != nil {
return err return err
} }
defer m.Close() defer m.Close()
rawBistream, err := m.RawBitstreamData() fmt.Printf("Before programming I %q A %q\n", f.GetInterfaceUUID(), f.GetAcceleratorTypeUUID())
if err != nil { fmt.Printf("Trying to program %s to port %s: ", bs, dev)
return err fmt.Println(f.PR(m, dryRun))
} fmt.Printf("After programming I %q A %q\n", f.GetInterfaceUUID(), f.GetAcceleratorTypeUUID())
if dryRun {
fmt.Println("Dry-Run: Skipping actual programming")
return nil
}
fmt.Print("Trying to PR, brace yourself! :")
fmt.Println(f.PortPR(0, rawBistream))
return nil return nil
} }

3
pkg/fpga/bitstream/aocx.go
View File

@ -23,6 +23,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -113,7 +114,7 @@ func setSection(f *FileAOCX, section *elf.Section) error {
if err != nil { if err != nil {
return errors.Wrapf(err, "%s: unable to get section data", name) return errors.Wrapf(err, "%s: unable to get section data", name)
} }
*field = string(data) *field = strings.TrimSpace(string(data))
} }
return nil return nil
} }

137
pkg/fpga/bitstream/bitstream.go
View File

@ -15,61 +15,14 @@
package bitstream package bitstream
import ( import (
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings"
utilsexec "k8s.io/utils/exec"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/device"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/linux"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const (
fpgaconf = "/opt/intel/fpga-sw/opae/fpgaconf-wrapper"
pciAddressRegex = `^[[:xdigit:]]{4}:([[:xdigit:]]{2}):([[:xdigit:]]{2})\.([[:xdigit:]])$`
)
var (
pciAddressRE = regexp.MustCompile(pciAddressRegex)
)
// FPGABitstream defines common interface for OPAE and OpenCL bitstreams
type FPGABitstream interface {
Init() error
Validate(region, afu string) error
Program(fme *device.FPGADevice, execer utilsexec.Interface) error
}
// OPAEBitstream defines structure of .gbs file
type OPAEBitstream struct {
Path string
Region string
AFU string
}
// Init gets Region and AFU from .gbs file
func (bitstream *OPAEBitstream) Init() error {
gbs, err := OpenGBS(bitstream.Path)
if err != nil {
return errors.Wrapf(err, "%s: can't get bitstream info", bitstream.Path)
}
if len(gbs.Metadata.AfuImage.AcceleratorClusters) != 1 {
return errors.Errorf("%s: 'accelerator-clusters' field not found", bitstream.Path)
}
bitstream.Region = device.CanonizeID(gbs.Metadata.AfuImage.InterfaceUUID)
bitstream.AFU = device.CanonizeID(gbs.Metadata.AfuImage.AcceleratorClusters[0].AcceleratorTypeUUID)
return nil
}
// GetFPGABitstream scans bitstream storage and returns first found bitstream by region and afu id // GetFPGABitstream scans bitstream storage and returns first found bitstream by region and afu id
func GetFPGABitstream(bitstreamDir, region, afu string) (FPGABitstream, error) { func GetFPGABitstream(bitstreamDir, region, afu string) (File, error) {
bitstreamPath := "" bitstreamPath := ""
// Temporarily only support gbs bitstreams // Temporarily only support gbs bitstreams
// for _, ext := range []string{".gbs", ".aocx"} { // for _, ext := range []string{".gbs", ".aocx"} {
@ -84,97 +37,11 @@ func GetFPGABitstream(bitstreamDir, region, afu string) (FPGABitstream, error) {
return nil, errors.Errorf("%s: stat error: %v", bitstreamPath, err) return nil, errors.Errorf("%s: stat error: %v", bitstreamPath, err)
} }
if ext == ".gbs" { return Open(bitstreamPath)
return &OPAEBitstream{
Path: bitstreamPath,
Region: region,
AFU: afu}, nil
} else if ext == ".aocx" {
return &OpenCLBitstream{
Path: bitstreamPath,
Region: region,
AFU: afu}, nil
}
} }
return nil, errors.Errorf("%s/%s: bitstream not found", region, afu) return nil, errors.Errorf("%s/%s: bitstream not found", region, afu)
} }
func validate(region, afu, expectedRegion, expectedAFU string) error {
if expectedRegion != region {
return errors.Errorf("bitstream is not for this device: region(%s) and interface-uuid(%s) don't match", region, expectedRegion)
}
if expectedAFU != afu {
return errors.Errorf("incorrect bitstream: AFU(%s) and accelerator-type-uuid(%s) don't match", afu, expectedAFU)
}
return nil
}
// Validate checks if region and afu parameters mutch bitstream parameters
func (bitstream *OPAEBitstream) Validate(region, afu string) error {
return validate(bitstream.Region, bitstream.AFU, region, afu)
}
func getFpgaConfArgs(devNode string) ([]string, error) {
realDevPath, err := linux.FindSysFsDevice(devNode)
if err != nil {
return nil, err
}
if realDevPath == "" {
return nil, nil
}
for p := realDevPath; strings.HasPrefix(p, "/sys/devices/pci"); p = filepath.Dir(p) {
pciDevPath, err := filepath.EvalSymlinks(filepath.Join(p, "device"))
if err != nil {
continue
}
subs := pciAddressRE.FindStringSubmatch(filepath.Base(pciDevPath))
if subs == nil || len(subs) != 4 {
return nil, errors.Errorf("unable to parse PCI address %s", pciDevPath)
}
return []string{"-B", "0x" + subs[1], "-D", "0x" + subs[2], "-F", "0x" + subs[3]}, nil
}
return nil, errors.Errorf("can't find PCI device address for sysfs entry %s", realDevPath)
}
// Program programs OPAE gbs bitstream
func (bitstream *OPAEBitstream) Program(device *device.FPGADevice, execer utilsexec.Interface) error {
args, err := getFpgaConfArgs(device.DevNode)
if err != nil {
return errors.Wrapf(err, "failed get fpgaconf args for %s", device.DevNode)
}
args = append(args, bitstream.Path)
output, err := execer.Command(fpgaconf, args...).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "failed to program AFU %s to device %s, region %s: output: %s", bitstream.AFU, device.DevNode, bitstream.Region, string(output))
}
return nil
}
// OpenCLBitstream defines parameters of .aocx file
type OpenCLBitstream struct {
Path string
Region string
AFU string
}
// Init stub for OpenCLBitstream
func (bitstream *OpenCLBitstream) Init() error {
// Unpack .gbs and call OPAEBitstream.init() here
return nil
}
// Validate stub for OpenCLBitstream
func (bitstream *OpenCLBitstream) Validate(region, afu string) error {
return validate(bitstream.Region, bitstream.AFU, region, afu)
}
// Program stub for OpenCLBitstream
func (bitstream *OpenCLBitstream) Program(device *device.FPGADevice, execer utilsexec.Interface) error {
return fmt.Errorf("Not implemented")
}
// Open bitstream file, detecting type based on the filename extension. // Open bitstream file, detecting type based on the filename extension.
func Open(fname string) (File, error) { func Open(fname string) (File, error) {
switch filepath.Ext(fname) { switch filepath.Ext(fname) {

1
pkg/fpga/bitstream/gbs.go
View File

@ -153,7 +153,6 @@ func NewFileGBS(r bitstreamReader) (*FileGBS, error) {
sr := io.NewSectionReader(r, 0, 1<<63-1) sr := io.NewSectionReader(r, 0, 1<<63-1)
f := new(FileGBS) f := new(FileGBS)
// TODO:
// 1. Read file header // 1. Read file header
sr.Seek(0, io.SeekStart) sr.Seek(0, io.SeekStart)
if err := binary.Read(sr, binary.LittleEndian, &f.Header); err != nil { if err := binary.Read(sr, binary.LittleEndian, &f.Header); err != nil {

189
pkg/fpga/device/device.go
View File

@ -1,189 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package device
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/pkg/errors"
)
const (
sysfsDirectoryOPAE = "/sys/class/fpga"
interfaceIDTemplateOPAE = "/sys/class/fpga/%s/*/pr/interface_id"
afuIDTemplateOPAE = "/sys/class/fpga/intel-fpga-dev.*/%s/afu_id"
fpgaPortDevRegexOPAE = `^intel-fpga-port\.[0-9]+$`
devTemplateOPAE = "/sys/class/fpga/intel-fpga-dev.*/%s/dev"
sysfsDirectoryDFL = "/sys/class/fpga_region"
interfaceIDTemplateDFL = "/sys/class/fpga_region/%s/*/dfl-fme-region.*/fpga_region/region*/compat_id"
afuIDTemplateDFL = "/sys/class/fpga_region/region*/%s/afu_id"
devTemplateDFL = "/sys/class/fpga_region/region*/%s/dev"
fpgaPortDevRegexDFL = `^dfl-port\.[0-9]+$`
fpgaDevRegex = `.*/sys/class/fpga[^/]*/([^/]+)/.+$`
fpgaPortDevRegex = `.*/sys/class/fpga[^/]*/[^/]+/([^/]+)/.+$`
)
var (
fpgaPortDevRegDFL = regexp.MustCompile(fpgaPortDevRegexDFL)
fpgaPortDevRegOPAE = regexp.MustCompile(fpgaPortDevRegexOPAE)
fpgaDevReg = regexp.MustCompile(fpgaDevRegex)
fpgaPortDevReg = regexp.MustCompile(fpgaPortDevRegex)
)
// FPGADevice represents FME and port(AFU) devices
type FPGADevice struct {
ID string // Interface or AFU id
Name string // basename of DevNode
SysfsPath string // path to ID file
DevNode string
Minor uint32
Major uint32
}
// CanonizeID canonizes Interface and AFU ids
func CanonizeID(ID string) string {
return strings.ToLower(strings.Replace(strings.TrimSpace(ID), "-", "", -1))
}
// IsFPGADevice returns true if device is either DFL or OPAE device
func IsFPGADevice(deviceName string) bool {
return fpgaPortDevRegDFL.MatchString(deviceName) || fpgaPortDevRegOPAE.MatchString(deviceName)
}
func getTemplates(sysFsPrefix string) (string, string, string, error) {
var afuIDTemplate, interfaceIDTemplate, devTemplate string
if _, err := os.Stat(path.Join(sysFsPrefix, sysfsDirectoryOPAE)); err == nil {
afuIDTemplate = afuIDTemplateOPAE
interfaceIDTemplate = interfaceIDTemplateOPAE
devTemplate = devTemplateOPAE
} else if _, err := os.Stat(path.Join(sysFsPrefix, sysfsDirectoryDFL)); err == nil {
afuIDTemplate = afuIDTemplateDFL
interfaceIDTemplate = interfaceIDTemplateDFL
devTemplate = devTemplateDFL
} else {
return "", "", "", fmt.Errorf("kernel driver is not loaded: neither %s nor %s sysfs directory is accessible", sysfsDirectoryOPAE, sysfsDirectoryDFL)
}
return path.Join(sysFsPrefix, afuIDTemplate), path.Join(sysFsPrefix, interfaceIDTemplate), path.Join(sysFsPrefix, devTemplate), nil
}
// read file by filepath.Glob pattern
func readFileByPattern(pattern string) (string, []byte, error) {
matches, err := filepath.Glob(pattern)
if err != nil {
return "", nil, err
}
if len(matches) == 0 {
return "", nil, fmt.Errorf("no file found with pattern '%s'", pattern)
}
if len(matches) > 1 {
return "", nil, fmt.Errorf("path pattern '%s' matches multiple files", pattern)
}
data, err := ioutil.ReadFile(matches[0])
if err != nil {
return matches[0], nil, errors.WithStack(err)
}
return matches[0], data, nil
}
func parseDev(devData string) (uint32, uint32, error) {
devData = strings.TrimSpace(devData)
numbers := strings.SplitN(devData, ":", 2)
minor := numbers[0]
major := numbers[1]
minorInt, err := strconv.ParseInt(minor, 10, 32)
if err != nil {
return 0, 0, errors.Wrapf(err, "can't convert device minor %s to a number", minor)
}
majorInt, err := strconv.ParseInt(major, 10, 32)
if err != nil {
return 0, 0, errors.Wrapf(err, "can't convert device major %s to a number", major)
}
return uint32(minorInt), uint32(majorInt), nil
}
func getFPGADevice(afuIDPattern, devPattern string) (*FPGADevice, error) {
idPath, idData, err := readFileByPattern(afuIDPattern)
if err != nil {
return nil, err
}
// Get minor:major from the dev file
_, devData, err := readFileByPattern(devPattern)
if err != nil {
return nil, err
}
major, minor, err := parseDev(string(devData))
if err != nil {
return nil, err
}
// Find fpga device name
subs := fpgaPortDevReg.FindStringSubmatch(idPath)
if len(subs) < 2 {
return nil, fmt.Errorf("can't parse afu_id path: %s", idPath)
}
name := subs[1]
return &FPGADevice{
ID: CanonizeID(string(idData)),
Name: name,
SysfsPath: idPath,
DevNode: path.Join("/dev", name),
Minor: minor,
Major: major,
}, nil
}
// GetAFUDevice creates FPGADevice struct for using port device name
func GetAFUDevice(sysFsPrefix, portDeviceName string) (*FPGADevice, error) {
afuIDTemplate, _, devTemplate, err := getTemplates(sysFsPrefix)
if err != nil {
return nil, err
}
return getFPGADevice(fmt.Sprintf(afuIDTemplate, portDeviceName), fmt.Sprintf(devTemplate, portDeviceName))
}
// GetFMEDevice reads FME properties from SysFs
func GetFMEDevice(sysFsPrefix, portDeviceName string) (*FPGADevice, error) {
portDeviceName = strings.TrimPrefix(portDeviceName, "/dev/")
afuDevice, err := GetAFUDevice(sysFsPrefix, portDeviceName)
if err != nil {
return nil, err
}
// Get top level region name from the AFU path
subs := fpgaDevReg.FindStringSubmatch(afuDevice.SysfsPath)
if len(subs) < 2 {
return nil, fmt.Errorf("can't parse sysfs path: %s", afuDevice.SysfsPath)
}
region := subs[1]
_, interfaceIDTemplate, devTemplate, err := getTemplates(sysFsPrefix)
if err != nil {
return nil, err
}
return getFPGADevice(fmt.Sprintf(interfaceIDTemplate, region), fmt.Sprintf(devTemplate, portDeviceName))
}

302
pkg/fpga/linux/dfl.go
View File

@ -18,104 +18,133 @@
package linux package linux
import ( import (
"os" "math"
"path/filepath"
"strconv"
"unsafe" "unsafe"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const (
dflFpgaFmePrefix = "dfl-fme."
dflFpgaPortPrefix = "dfl-port."
dflFpgaFmeGlobPCI = "fpga_region/region*/dfl-fme.*"
)
// DflFME represent DFL FPGA FME device // DflFME represent DFL FPGA FME device
type DflFME struct { type DflFME struct {
FpgaFME FpgaFME
DevPath string DevPath string
f *os.File SysFsPath string
Name string
PCIDevice *PCIDevice
SocketID string
Dev string
CompatID string
BitstreamID string
BitstreamMetadata string
PortsNum string
} }
// Close closes open device // Close closes open device
func (f *DflFME) Close() error { func (f *DflFME) Close() error {
if f.f != nil { // if f.f != nil {
return f.f.Close() // return f.f.Close()
} // }
return nil return nil
} }
// NewDflFME Opens device // NewDflFME Opens device
func NewDflFME(dev string) (FpgaFME, error) { func NewDflFME(dev string) (FpgaFME, error) {
f, err := os.OpenFile(dev, os.O_RDWR, 0644) fme := &DflFME{DevPath: dev}
if err != nil {
return nil, err
}
fme := &DflFME{DevPath: dev, f: f}
// check that kernel API is compatible // check that kernel API is compatible
if _, err := fme.GetAPIVersion(); err != nil { if _, err := fme.GetAPIVersion(); err != nil {
fme.Close()
return nil, errors.Wrap(err, "kernel API mismatch") return nil, errors.Wrap(err, "kernel API mismatch")
} }
if err := checkVendorAndClass(fme); err != nil {
return nil, err
}
if err := fme.updateProperties(); err != nil {
return nil, err
}
return fme, nil return fme, nil
} }
// DflPort represent DFL FPGA Port device // DflPort represent DFL FPGA Port device
type DflPort struct { type DflPort struct {
FpgaPort FpgaPort
DevPath string DevPath string
f *os.File SysFsPath string
Name string
PCIDevice *PCIDevice
Dev string
AFUID string
ID string
FME FpgaFME
} }
// Close closes open device // Close closes open device
func (f *DflPort) Close() error { func (f *DflPort) Close() error {
if f.f != nil { if f.FME != nil {
return f.f.Close() return f.FME.Close()
} }
// if f.f != nil {
// return f.f.Close()
// }
return nil return nil
} }
// NewDflPort Opens device // NewDflPort Opens device
func NewDflPort(dev string) (FpgaPort, error) { func NewDflPort(dev string) (FpgaPort, error) {
f, err := os.OpenFile(dev, os.O_RDWR, 0644) port := &DflPort{DevPath: dev}
if err != nil {
return nil, err
}
port := &DflPort{DevPath: dev, f: f}
// check that kernel API is compatible // check that kernel API is compatible
if _, err := port.GetAPIVersion(); err != nil { if _, err := port.GetAPIVersion(); err != nil {
port.Close()
return nil, errors.Wrap(err, "kernel API mismatch") return nil, errors.Wrap(err, "kernel API mismatch")
} }
if err := checkVendorAndClass(port); err != nil {
return nil, err
}
if err := port.updateProperties(); err != nil {
return nil, err
}
return port, nil return port, nil
} }
// common ioctls for FME and Port // common ioctls for FME and Port
func commonGetAPIVersion(fd uintptr) (int, error) { func commonDflGetAPIVersion(dev string) (int, error) {
v, err := ioctl(fd, DFL_FPGA_GET_API_VERSION, 0) v, err := ioctlDev(dev, DFL_FPGA_GET_API_VERSION, 0)
return int(v), err return int(v), err
} }
func commonCheckExtension(fd uintptr) (int, error) { func commonDflCheckExtension(dev string) (int, error) {
v, err := ioctl(fd, DFL_FPGA_CHECK_EXTENSION, 0) v, err := ioctlDev(dev, DFL_FPGA_CHECK_EXTENSION, 0)
return int(v), err return int(v), err
} }
// GetAPIVersion Report the version of the driver API. // GetAPIVersion Report the version of the driver API.
// * Return: Driver API Version. // * Return: Driver API Version.
func (f *DflFME) GetAPIVersion() (int, error) { func (f *DflFME) GetAPIVersion() (int, error) {
return commonGetAPIVersion(f.f.Fd()) return commonDflGetAPIVersion(f.DevPath)
} }
// CheckExtension Check whether an extension is supported. // CheckExtension Check whether an extension is supported.
// * Return: 0 if not supported, otherwise the extension is supported. // * Return: 0 if not supported, otherwise the extension is supported.
func (f *DflFME) CheckExtension() (int, error) { func (f *DflFME) CheckExtension() (int, error) {
return commonCheckExtension(f.f.Fd()) // return commonCheckExtension(f.f.Fd())
return commonDflCheckExtension(f.DevPath)
} }
// GetAPIVersion Report the version of the driver API. // GetAPIVersion Report the version of the driver API.
// * Return: Driver API Version. // * Return: Driver API Version.
func (f *DflPort) GetAPIVersion() (int, error) { func (f *DflPort) GetAPIVersion() (int, error) {
return commonGetAPIVersion(f.f.Fd()) return commonDflGetAPIVersion(f.DevPath)
} }
// CheckExtension Check whether an extension is supported. // CheckExtension Check whether an extension is supported.
// * Return: 0 if not supported, otherwise the extension is supported. // * Return: 0 if not supported, otherwise the extension is supported.
func (f *DflPort) CheckExtension() (int, error) { func (f *DflPort) CheckExtension() (int, error) {
return commonCheckExtension(f.f.Fd()) return commonDflCheckExtension(f.DevPath)
} }
// PortReset Reset the FPGA Port and its AFU. No parameters are supported. // PortReset Reset the FPGA Port and its AFU. No parameters are supported.
@ -124,7 +153,7 @@ func (f *DflPort) CheckExtension() (int, error) {
// (e.g. DMA or PR operation failure) and be recoverable from the failure. // (e.g. DMA or PR operation failure) and be recoverable from the failure.
// * Return: 0 on success, -errno of failure // * Return: 0 on success, -errno of failure
func (f *DflPort) PortReset() error { func (f *DflPort) PortReset() error {
_, err := ioctl(f.f.Fd(), DFL_FPGA_PORT_RESET, 0) _, err := ioctlDev(f.DevPath, DFL_FPGA_PORT_RESET, 0)
return err return err
} }
@ -134,7 +163,7 @@ func (f *DflPort) PortReset() error {
func (f *DflPort) PortGetInfo() (ret FpgaPortInfo, err error) { func (f *DflPort) PortGetInfo() (ret FpgaPortInfo, err error) {
var value DflFpgaPortInfo var value DflFpgaPortInfo
value.Argsz = uint32(unsafe.Sizeof(value)) value.Argsz = uint32(unsafe.Sizeof(value))
_, err = ioctl(f.f.Fd(), DFL_FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) _, err = ioctlDev(f.DevPath, DFL_FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value)))
if err == nil { if err == nil {
ret.Flags = value.Flags ret.Flags = value.Flags
ret.Regions = value.Regions ret.Regions = value.Regions
@ -152,7 +181,7 @@ func (f *DflPort) PortGetRegionInfo(index uint32) (ret FpgaPortRegionInfo, err e
var value DflFpgaPortRegionInfo var value DflFpgaPortRegionInfo
value.Argsz = uint32(unsafe.Sizeof(value)) value.Argsz = uint32(unsafe.Sizeof(value))
value.Index = index value.Index = index
_, err = ioctl(f.f.Fd(), DFL_FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) _, err = ioctlDev(f.DevPath, DFL_FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value)))
if err == nil { if err == nil {
ret.Flags = value.Flags ret.Flags = value.Flags
ret.Index = value.Index ret.Index = value.Index
@ -174,6 +203,211 @@ func (f *DflFME) PortPR(port uint32, bitstream []byte) error {
value.Port_id = port value.Port_id = port
value.Buffer_size = uint32(len(bitstream)) value.Buffer_size = uint32(len(bitstream))
value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0]))) value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0])))
_, err := ioctl(f.f.Fd(), DFL_FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) _, err := ioctlDev(f.DevPath, DFL_FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value)))
return err return err
} }
// FME interfaces
// GetDevPath returns path to device node
func (f *DflFME) GetDevPath() string {
return f.DevPath
}
// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items)
func (f *DflFME) GetSysFsPath() string {
if f.SysFsPath != "" {
return f.SysFsPath
}
sysfs, err := FindSysFsDevice(f.DevPath)
if err != nil {
return ""
}
f.SysFsPath = sysfs
return f.SysFsPath
}
// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/
func (f *DflFME) GetName() string {
if f.Name != "" {
return f.Name
}
f.Name = filepath.Base(f.GetSysFsPath())
return f.Name
}
// GetPCIDevice returns PCIDevice for this device
func (f *DflFME) GetPCIDevice() (*PCIDevice, error) {
if f.PCIDevice != nil {
return f.PCIDevice, nil
}
pci, err := NewPCIDevice(f.GetSysFsPath())
if err != nil {
return nil, err
}
f.PCIDevice = pci
return f.PCIDevice, nil
}
// GetPortsNum returns amount of FPGA Ports associated to this FME
func (f *DflFME) GetPortsNum() int {
if f.PortsNum == "" {
err := f.updateProperties()
if err != nil {
return -1
}
}
n, err := strconv.ParseUint(f.PortsNum, 10, 32)
if err != nil {
return -1
}
return int(n)
}
// GetInterfaceUUID returns Interface UUID for FME
func (f *DflFME) GetInterfaceUUID() (id string) {
if f.CompatID == "" {
err := f.updateProperties()
if err != nil {
return ""
}
}
return f.CompatID
}
// Update properties from sysfs
func (f *DflFME) updateProperties() error {
pci, err := f.GetPCIDevice()
if err != nil {
return err
}
fileMap := map[string]*string{
"bitstream_id": &f.BitstreamID,
"bitstream_metadata": &f.BitstreamMetadata,
"dev": &f.Dev,
"ports_num": &f.PortsNum,
"socket_id": &f.SocketID,
"dfl-fme-region.*/fpga_region/region*/compat_id": &f.CompatID,
}
return readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, dflFpgaFmeGlobPCI))
}
// Port interfaces
// GetDevPath returns path to device node
func (f *DflPort) GetDevPath() string {
return f.DevPath
}
// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items)
func (f *DflPort) GetSysFsPath() string {
if f.SysFsPath != "" {
return f.SysFsPath
}
sysfs, err := FindSysFsDevice(f.DevPath)
if err != nil {
return ""
}
f.SysFsPath = sysfs
return f.SysFsPath
}
// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/
func (f *DflPort) GetName() string {
if f.Name != "" {
return f.Name
}
f.Name = filepath.Base(f.GetSysFsPath())
return f.Name
}
// GetPCIDevice returns PCIDevice for this device
func (f *DflPort) GetPCIDevice() (*PCIDevice, error) {
if f.PCIDevice != nil {
return f.PCIDevice, nil
}
pci, err := NewPCIDevice(f.GetSysFsPath())
if err != nil {
return nil, err
}
f.PCIDevice = pci
return f.PCIDevice, nil
}
// GetFME returns FPGA FME device for this port
func (f *DflPort) GetFME() (fme FpgaFME, err error) {
if f.FME != nil {
return f.FME, nil
}
pci, err := f.GetPCIDevice()
if err != nil {
return
}
if pci.PhysFn != nil {
pci = pci.PhysFn
}
var dev string
fileMap := map[string]*string{
"dev": &dev,
}
if err = readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, dflFpgaFmeGlobPCI)); err != nil {
return
}
realDev, err := filepath.EvalSymlinks(filepath.Join("/dev/char", dev))
if err != nil {
return
}
fme, err = NewDflFME(realDev)
if err != nil {
return
}
f.FME = fme
return
}
// GetPortID returns ID of the FPGA port within physical device
func (f *DflPort) GetPortID() (uint32, error) {
if f.ID == "" {
err := f.updateProperties()
if err != nil {
return math.MaxUint32, err
}
}
id, err := strconv.ParseUint(f.ID, 10, 32)
return uint32(id), err
}
// GetAcceleratorTypeUUID returns AFU UUID for port
func (f *DflPort) GetAcceleratorTypeUUID() (afuID string) {
err := f.updateProperties()
if err != nil || f.AFUID == "" {
return ""
}
return f.AFUID
}
// GetInterfaceUUID returns Interface UUID for FME
func (f *DflPort) GetInterfaceUUID() (id string) {
fme, err := f.GetFME()
if err != nil {
return ""
}
defer fme.Close()
return fme.GetInterfaceUUID()
}
// PR programs specified bitstream to port
func (f *DflPort) PR(bs bitstream.File, dryRun bool) error {
return genericPortPR(f, bs, dryRun)
}
// Update properties from sysfs
func (f *DflPort) updateProperties() error {
fileMap := map[string]*string{
"afu_id": &f.AFUID,
"dev": &f.Dev,
"id": &f.ID,
}
return readFilesInDirectory(fileMap, f.GetSysFsPath())
}

117
pkg/fpga/linux/fpga.go Normal file
View File

@ -0,0 +1,117 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build linux
//
package linux
import (
"io/ioutil"
"path/filepath"
"strings"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"github.com/pkg/errors"
)
// IsFpgaFME returns true if the name looks like any supported FME device
func IsFpgaFME(name string) bool {
devName := cleanBasename(name)
return strings.HasPrefix(devName, dflFpgaFmePrefix) || strings.HasPrefix(devName, intelFpgaFmePrefix)
}
// IsFpgaPort returns true if the name looks like any supported FME device
func IsFpgaPort(name string) bool {
devName := cleanBasename(name)
return strings.HasPrefix(devName, dflFpgaPortPrefix) || strings.HasPrefix(devName, intelFpgaPortPrefix)
}
// CanonizeID canonizes Interface and AFU ids
func CanonizeID(ID string) string {
return strings.ToLower(strings.Replace(strings.TrimSpace(ID), "-", "", -1))
}
// NewFpgaPort returns FpgaPort for specified device node
func NewFpgaPort(fname string) (FpgaPort, error) {
if strings.IndexByte(fname, byte('/')) < 0 {
fname = filepath.Join("/dev", fname)
}
devName := cleanBasename(fname)
switch {
case strings.HasPrefix(devName, dflFpgaPortPrefix):
return NewDflPort(fname)
case strings.HasPrefix(devName, intelFpgaPortPrefix):
return NewIntelFpgaPort(fname)
}
return nil, errors.Errorf("unknown type of FPGA port %s", devName)
}
// NewFpgaFME returns FpgaFME for specified device node
func NewFpgaFME(fname string) (FpgaFME, error) {
if strings.IndexByte(fname, byte('/')) < 0 {
fname = filepath.Join("/dev", fname)
}
devName := cleanBasename(fname)
switch {
case strings.HasPrefix(devName, dflFpgaFmePrefix):
return NewDflFME(fname)
case strings.HasPrefix(devName, intelFpgaFmePrefix):
return NewIntelFpgaFME(fname)
}
return nil, errors.Errorf("unknown type of FPGA FME %s", devName)
}
// ListFpgaDevices returns two lists of FPGA device nodes: FMEs and Ports
func ListFpgaDevices() (FMEs, Ports []string) {
files, err := ioutil.ReadDir("/sys/bus/platform/devices")
if err != nil {
return
}
for _, file := range files {
fname := file.Name()
switch {
case IsFpgaFME(fname):
FMEs = append(FMEs, fname)
case IsFpgaPort(fname):
Ports = append(Ports, fname)
}
}
return
}
func genericPortPR(f FpgaPort, bs bitstream.File, dryRun bool) error {
fme, err := f.GetFME()
if err != nil {
return err
}
ifID := fme.GetInterfaceUUID()
bsID := bs.InterfaceUUID()
if ifID != bsID {
return errors.Errorf("FME interface UUID %q is not compatible with bitstream UUID %q ", ifID, bsID)
}
pNum, err := f.GetPortID()
if err != nil {
return err
}
rawBistream, err := bs.RawBitstreamData()
if err != nil {
return err
}
if dryRun {
return nil
}
return fme.PortPR(pNum, rawBistream)
}

296
pkg/fpga/linux/intel-fpga.go
View File

@ -18,104 +18,129 @@
package linux package linux
import ( import (
"os" "math"
"path/filepath"
"strconv"
"unsafe" "unsafe"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const (
intelFpgaFmePrefix = "intel-fpga-fme."
intelFpgaPortPrefix = "intel-fpga-port."
intelFpgaFmeGlobPCI = "fpga/intel-fpga-dev.*/intel-fpga-fme.*"
)
// IntelFpgaFME represent Intel FPGA FME device // IntelFpgaFME represent Intel FPGA FME device
type IntelFpgaFME struct { type IntelFpgaFME struct {
FpgaFME FpgaFME
DevPath string DevPath string
f *os.File SysFsPath string
Name string
PCIDevice *PCIDevice
SocketID string
Dev string
CompatID string
BitstreamID string
BitstreamMetadata string
PortsNum string
} }
// Close closes open device // Close closes open device
func (f *IntelFpgaFME) Close() error { func (f *IntelFpgaFME) Close() error {
if f.f != nil {
return f.f.Close()
}
return nil return nil
} }
// NewIntelFpgaFME Opens device // NewIntelFpgaFME Opens device
func NewIntelFpgaFME(dev string) (FpgaFME, error) { func NewIntelFpgaFME(dev string) (FpgaFME, error) {
f, err := os.OpenFile(dev, os.O_RDWR, 0644) fme := &IntelFpgaFME{DevPath: dev}
if err != nil {
return nil, err
}
fme := &IntelFpgaFME{DevPath: dev, f: f}
// check that kernel API is compatible // check that kernel API is compatible
if _, err := fme.GetAPIVersion(); err != nil { if _, err := fme.GetAPIVersion(); err != nil {
fme.Close()
return nil, errors.Wrap(err, "kernel API mismatch") return nil, errors.Wrap(err, "kernel API mismatch")
} }
if err := checkVendorAndClass(fme); err != nil {
return nil, err
}
if err := fme.updateProperties(); err != nil {
return nil, err
}
return fme, nil return fme, nil
} }
// IntelFpgaPort represent IntelFpga FPGA Port device // IntelFpgaPort represent IntelFpga FPGA Port device
type IntelFpgaPort struct { type IntelFpgaPort struct {
FpgaPort FpgaPort
DevPath string DevPath string
f *os.File SysFsPath string
Name string
PCIDevice *PCIDevice
Dev string
AFUID string
ID string
FME FpgaFME
} }
// Close closes open device // Close closes open device
func (f *IntelFpgaPort) Close() error { func (f *IntelFpgaPort) Close() error {
if f.f != nil { if f.FME != nil {
return f.f.Close() defer f.FME.Close()
} }
return nil return nil
} }
// NewIntelFpgaPort Opens device // NewIntelFpgaPort Opens device
func NewIntelFpgaPort(dev string) (FpgaPort, error) { func NewIntelFpgaPort(dev string) (FpgaPort, error) {
f, err := os.OpenFile(dev, os.O_RDWR, 0644) port := &IntelFpgaPort{DevPath: dev}
if err != nil {
return nil, err
}
port := &IntelFpgaPort{DevPath: dev, f: f}
// check that kernel API is compatible // check that kernel API is compatible
if _, err := port.GetAPIVersion(); err != nil { if _, err := port.GetAPIVersion(); err != nil {
port.Close() port.Close()
return nil, errors.Wrap(err, "kernel API mismatch") return nil, errors.Wrap(err, "kernel API mismatch")
} }
if err := checkVendorAndClass(port); err != nil {
port.Close()
return nil, err
}
if err := port.updateProperties(); err != nil {
port.Close()
return nil, err
}
return port, nil return port, nil
} }
// common ioctls for FME and Port // common ioctls for FME and Port
func commonIntelFpgaGetAPIVersion(fd uintptr) (int, error) { func commonIntelFpgaGetAPIVersion(fd string) (int, error) {
v, err := ioctl(fd, FPGA_GET_API_VERSION, 0) v, err := ioctlDev(fd, FPGA_GET_API_VERSION, 0)
return int(v), err return int(v), err
} }
func commonIntelFpgaCheckExtension(fd uintptr) (int, error) { func commonIntelFpgaCheckExtension(fd string) (int, error) {
v, err := ioctl(fd, FPGA_CHECK_EXTENSION, 0) v, err := ioctlDev(fd, FPGA_CHECK_EXTENSION, 0)
return int(v), err return int(v), err
} }
// GetAPIVersion Report the version of the driver API. // GetAPIVersion Report the version of the driver API.
// * Return: Driver API Version. // * Return: Driver API Version.
func (f *IntelFpgaFME) GetAPIVersion() (int, error) { func (f *IntelFpgaFME) GetAPIVersion() (int, error) {
return commonIntelFpgaGetAPIVersion(f.f.Fd()) return commonIntelFpgaGetAPIVersion(f.DevPath)
} }
// CheckExtension Check whether an extension is supported. // CheckExtension Check whether an extension is supported.
// * Return: 0 if not supported, otherwise the extension is supported. // * Return: 0 if not supported, otherwise the extension is supported.
func (f *IntelFpgaFME) CheckExtension() (int, error) { func (f *IntelFpgaFME) CheckExtension() (int, error) {
return commonIntelFpgaCheckExtension(f.f.Fd()) return commonIntelFpgaCheckExtension(f.DevPath)
} }
// GetAPIVersion Report the version of the driver API. // GetAPIVersion Report the version of the driver API.
// * Return: Driver API Version. // * Return: Driver API Version.
func (f *IntelFpgaPort) GetAPIVersion() (int, error) { func (f *IntelFpgaPort) GetAPIVersion() (int, error) {
return commonIntelFpgaGetAPIVersion(f.f.Fd()) return commonIntelFpgaGetAPIVersion(f.DevPath)
} }
// CheckExtension Check whether an extension is supported. // CheckExtension Check whether an extension is supported.
// * Return: 0 if not supported, otherwise the extension is supported. // * Return: 0 if not supported, otherwise the extension is supported.
func (f *IntelFpgaPort) CheckExtension() (int, error) { func (f *IntelFpgaPort) CheckExtension() (int, error) {
return commonIntelFpgaCheckExtension(f.f.Fd()) return commonIntelFpgaCheckExtension(f.DevPath)
} }
// PortReset Reset the FPGA Port and its AFU. No parameters are supported. // PortReset Reset the FPGA Port and its AFU. No parameters are supported.
@ -124,7 +149,7 @@ func (f *IntelFpgaPort) CheckExtension() (int, error) {
// (e.g. DMA or PR operation failure) and be recoverable from the failure. // (e.g. DMA or PR operation failure) and be recoverable from the failure.
// * Return: 0 on success, -errno of failure // * Return: 0 on success, -errno of failure
func (f *IntelFpgaPort) PortReset() error { func (f *IntelFpgaPort) PortReset() error {
_, err := ioctl(f.f.Fd(), FPGA_PORT_RESET, 0) _, err := ioctlDev(f.DevPath, FPGA_PORT_RESET, 0)
return err return err
} }
@ -134,7 +159,7 @@ func (f *IntelFpgaPort) PortReset() error {
func (f *IntelFpgaPort) PortGetInfo() (ret FpgaPortInfo, err error) { func (f *IntelFpgaPort) PortGetInfo() (ret FpgaPortInfo, err error) {
var value IntelFpgaPortInfo var value IntelFpgaPortInfo
value.Argsz = uint32(unsafe.Sizeof(value)) value.Argsz = uint32(unsafe.Sizeof(value))
_, err = ioctl(f.f.Fd(), FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value))) _, err = ioctlDev(f.DevPath, FPGA_PORT_GET_INFO, uintptr(unsafe.Pointer(&value)))
if err == nil { if err == nil {
ret.Flags = value.Flags ret.Flags = value.Flags
ret.Regions = value.Regions ret.Regions = value.Regions
@ -152,7 +177,7 @@ func (f *IntelFpgaPort) PortGetRegionInfo(index uint32) (ret FpgaPortRegionInfo,
var value IntelFpgaPortRegionInfo var value IntelFpgaPortRegionInfo
value.Argsz = uint32(unsafe.Sizeof(value)) value.Argsz = uint32(unsafe.Sizeof(value))
value.Index = index value.Index = index
_, err = ioctl(f.f.Fd(), FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value))) _, err = ioctlDev(f.DevPath, FPGA_PORT_GET_REGION_INFO, uintptr(unsafe.Pointer(&value)))
if err == nil { if err == nil {
ret.Flags = value.Flags ret.Flags = value.Flags
ret.Index = value.Index ret.Index = value.Index
@ -174,6 +199,211 @@ func (f *IntelFpgaFME) PortPR(port uint32, bitstream []byte) error {
value.Port_id = port value.Port_id = port
value.Buffer_size = uint32(len(bitstream)) value.Buffer_size = uint32(len(bitstream))
value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0]))) value.Buffer_address = uint64(uintptr(unsafe.Pointer(&bitstream[0])))
_, err := ioctl(f.f.Fd(), FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value))) _, err := ioctlDev(f.DevPath, FPGA_FME_PORT_PR, uintptr(unsafe.Pointer(&value)))
return err return err
} }
// FME interfaces
// GetDevPath returns path to device node
func (f *IntelFpgaFME) GetDevPath() string {
return f.DevPath
}
// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items)
func (f *IntelFpgaFME) GetSysFsPath() string {
if f.SysFsPath != "" {
return f.SysFsPath
}
sysfs, err := FindSysFsDevice(f.DevPath)
if err != nil {
return ""
}
f.SysFsPath = sysfs
return f.SysFsPath
}
// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/
func (f *IntelFpgaFME) GetName() string {
if f.Name != "" {
return f.Name
}
f.Name = filepath.Base(f.GetSysFsPath())
return f.Name
}
// GetPCIDevice returns PCIDevice for this device
func (f *IntelFpgaFME) GetPCIDevice() (*PCIDevice, error) {
if f.PCIDevice != nil {
return f.PCIDevice, nil
}
pci, err := NewPCIDevice(f.GetSysFsPath())
if err != nil {
return nil, err
}
f.PCIDevice = pci
return f.PCIDevice, nil
}
// GetPortsNum returns amount of FPGA Ports associated to this FME
func (f *IntelFpgaFME) GetPortsNum() int {
if f.PortsNum == "" {
err := f.updateProperties()
if err != nil {
return -1
}
}
n, err := strconv.ParseUint(f.PortsNum, 10, 32)
if err != nil {
return -1
}
return int(n)
}
// GetInterfaceUUID returns Interface UUID for FME
func (f *IntelFpgaFME) GetInterfaceUUID() (id string) {
if f.CompatID == "" {
err := f.updateProperties()
if err != nil {
return ""
}
}
return f.CompatID
}
// Update properties from sysfs
func (f *IntelFpgaFME) updateProperties() error {
pci, err := f.GetPCIDevice()
if err != nil {
return err
}
fileMap := map[string]*string{
"bitstream_id": &f.BitstreamID,
"bitstream_metadata": &f.BitstreamMetadata,
"dev": &f.Dev,
"ports_num": &f.PortsNum,
"socket_id": &f.SocketID,
"pr/interface_id": &f.CompatID,
}
return readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, intelFpgaFmeGlobPCI))
}
// Port interfaces
// GetDevPath returns path to device node
func (f *IntelFpgaPort) GetDevPath() string {
return f.DevPath
}
// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items)
func (f *IntelFpgaPort) GetSysFsPath() string {
if f.SysFsPath != "" {
return f.SysFsPath
}
sysfs, err := FindSysFsDevice(f.DevPath)
if err != nil {
return ""
}
f.SysFsPath = sysfs
return f.SysFsPath
}
// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/
func (f *IntelFpgaPort) GetName() string {
if f.Name != "" {
return f.Name
}
f.Name = filepath.Base(f.GetSysFsPath())
return f.Name
}
// GetPCIDevice returns PCIDevice for this device
func (f *IntelFpgaPort) GetPCIDevice() (*PCIDevice, error) {
if f.PCIDevice != nil {
return f.PCIDevice, nil
}
pci, err := NewPCIDevice(f.GetSysFsPath())
if err != nil {
return nil, err
}
f.PCIDevice = pci
return f.PCIDevice, nil
}
// GetFME returns FPGA FME device for this port
func (f *IntelFpgaPort) GetFME() (fme FpgaFME, err error) {
if f.FME != nil {
return f.FME, nil
}
pci, err := f.GetPCIDevice()
if err != nil {
return
}
if pci.PhysFn != nil {
pci = pci.PhysFn
}
var dev string
fileMap := map[string]*string{
"dev": &dev,
}
if err = readFilesInDirectory(fileMap, filepath.Join(pci.SysFsPath, intelFpgaFmeGlobPCI)); err != nil {
return
}
realDev, err := filepath.EvalSymlinks(filepath.Join("/dev/char", dev))
if err != nil {
return
}
fme, err = NewIntelFpgaFME(realDev)
if err != nil {
return
}
f.FME = fme
return
}
// GetPortID returns ID of the FPGA port within physical device
func (f *IntelFpgaPort) GetPortID() (uint32, error) {
if f.ID == "" {
err := f.updateProperties()
if err != nil {
return math.MaxUint32, err
}
}
id, err := strconv.ParseUint(f.ID, 10, 32)
return uint32(id), err
}
// GetAcceleratorTypeUUID returns AFU UUID for port
func (f *IntelFpgaPort) GetAcceleratorTypeUUID() string {
err := f.updateProperties()
if err != nil || f.AFUID == "" {
return ""
}
return f.AFUID
}
// GetInterfaceUUID returns Interface UUID for FME
func (f *IntelFpgaPort) GetInterfaceUUID() (id string) {
fme, err := f.GetFME()
if err != nil {
return ""
}
defer fme.Close()
return fme.GetInterfaceUUID()
}
// PR programs specified bitstream to port
func (f *IntelFpgaPort) PR(bs bitstream.File, dryRun bool) error {
return genericPortPR(f, bs, dryRun)
}
// Update properties from sysfs
func (f *IntelFpgaPort) updateProperties() error {
fileMap := map[string]*string{
"afu_id": &f.AFUID,
"dev": &f.Dev,
"id": &f.ID,
}
return readFilesInDirectory(fileMap, f.GetSysFsPath())
}

53
pkg/fpga/linux/interfaces.go
View File

@ -17,9 +17,13 @@
package linux package linux
import "io" import (
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga/bitstream"
"io"
)
type commonAPI interface { type commonFpgaAPI interface {
// Generic interfaces provided by FPGA ports and FMEs
io.Closer io.Closer
// GetAPIVersion Report the version of the driver API. // GetAPIVersion Report the version of the driver API.
// * Return: Driver API Version. // * Return: Driver API Version.
@ -27,11 +31,23 @@ type commonAPI interface {
// CheckExtension Check whether an extension is supported. // CheckExtension Check whether an extension is supported.
// * Return: 0 if not supported, otherwise the extension is supported. // * Return: 0 if not supported, otherwise the extension is supported.
CheckExtension() (int, error) CheckExtension() (int, error)
// Interfaces for device discovery and accessing properties
// GetDevPath returns path to device node
GetDevPath() string
// GetSysFsPath returns sysfs entry for FPGA FME or Port (e.g. can be used for custom errors/perf items)
GetSysFsPath() string
// GetName returns simple FPGA name, derived from sysfs entry, can be used with /dev/ or /sys/bus/platform/
GetName() string
// GetPCIDevice returns PCIDevice for this device
GetPCIDevice() (*PCIDevice, error)
} }
// FpgaFME represent interfaces provided by management interface of FPGA // FpgaFME represent interfaces provided by management interface of FPGA
type FpgaFME interface { type FpgaFME interface {
commonAPI // Kernel IOCTL interfaces for FPGA ports:
commonFpgaAPI
// PortPR does Partial Reconfiguration based on Port ID and Buffer (Image) // PortPR does Partial Reconfiguration based on Port ID and Buffer (Image)
// provided by caller. // provided by caller.
// * Return: 0 on success, -errno on failure. // * Return: 0 on success, -errno on failure.
@ -39,16 +55,25 @@ type FpgaFME interface {
// some errors during PR, under this case, the user can fetch HW error info // some errors during PR, under this case, the user can fetch HW error info
// from the status of FME's fpga manager. // from the status of FME's fpga manager.
PortPR(uint32, []byte) error PortPR(uint32, []byte) error
// TODO: (not implemented IOCTLs)
// TODO:
// Port release / assign // Port release / assign
// Get Info // Get Info
// Set IRQ err // Set IRQ err
// Interfaces for device discovery and accessing properties
// GetPortsNum returns amount of FPGA Ports associated to this FME
GetPortsNum() int
// InterfaceUUID returns Interface UUID for FME
GetInterfaceUUID() string
// GetPort returns FpgaPort of the desired FPGA port within that FME
// GetPort(uint32) (FpgaPort, error)
} }
// FpgaPort represent interfaces provided by AFU port of FPGA // FpgaPort represent interfaces provided by AFU port of FPGA
type FpgaPort interface { type FpgaPort interface {
commonAPI // Kernel IOCTL interfaces for FPGA ports:
commonFpgaAPI
// PortReset Reset the FPGA Port and its AFU. No parameters are supported. // PortReset Reset the FPGA Port and its AFU. No parameters are supported.
// Userspace can do Port reset at any time, e.g. during DMA or PR. But // Userspace can do Port reset at any time, e.g. during DMA or PR. But
// it should never cause any system level issue, only functional failure // it should never cause any system level issue, only functional failure
@ -65,11 +90,23 @@ type FpgaPort interface {
// * Driver returns the region info in other fields. // * Driver returns the region info in other fields.
// * Return: 0 on success, -errno on failure. // * Return: 0 on success, -errno on failure.
PortGetRegionInfo(index uint32) (FpgaPortRegionInfo, error) PortGetRegionInfo(index uint32) (FpgaPortRegionInfo, error)
// TODO: (not implemented IOCTLs)
// TODO:
// Port DMA map / unmap // Port DMA map / unmap
// UMSG enable / disable / set-mode / set-base-addr (intel-fpga) // UMSG enable / disable / set-mode / set-base-addr (intel-fpga)
// Set IRQ: err, uafu (intel-fpga) // Set IRQ: err, uafu (intel-fpga)
// Interfaces for device discovery and accessing properties
// GetFME returns FPGA FME device for this port
GetFME() (FpgaFME, error)
// GetPortID returns ID of the FPGA port within physical device
GetPortID() (uint32, error)
// GetAcceleratorTypeUUID returns AFU UUID for port
GetAcceleratorTypeUUID() string
// InterfaceUUID returns Interface UUID for FME
GetInterfaceUUID() string
// PR programs specified bitstream to port
PR(bitstream.File, bool) error
} }
// FpgaPortInfo is a unified port info between drivers // FpgaPortInfo is a unified port info between drivers

11
pkg/fpga/linux/ioctl.go
View File

@ -18,6 +18,7 @@
package linux package linux
import ( import (
"os"
"syscall" "syscall"
) )
@ -28,3 +29,13 @@ func ioctl(fd uintptr, req uint, arg uintptr) (ret uintptr, err error) {
} }
return return
} }
// Same as above, but open device only for single operation
func ioctlDev(dev string, req uint, arg uintptr) (ret uintptr, err error) {
f, err := os.OpenFile(dev, os.O_RDWR, 0644)
if err != nil {
return
}
defer f.Close()
return ioctl(f.Fd(), req, arg)
}

2
pkg/fpga/linux/pci.go
View File

@ -32,6 +32,8 @@ import (
const ( const (
pciAddressRegex = `^([[:xdigit:]]{4}):([[:xdigit:]]{2}):([[:xdigit:]]{2})\.([[:xdigit:]])$` pciAddressRegex = `^([[:xdigit:]]{4}):([[:xdigit:]]{2}):([[:xdigit:]]{2})\.([[:xdigit:]])$`
vendorIntel = "0x8086"
fpgaClass = "0x120000"
) )
var ( var (

40
pkg/fpga/linux/utils.go
View File

@ -15,17 +15,32 @@
package linux package linux
import ( import (
"github.com/pkg/errors"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/pkg/errors"
) )
// small helper function that reads several files into provided set of variables // small helper function that reads several files into provided set of variables
func readFilesInDirectory(fileMap map[string]*string, dir string) error { func readFilesInDirectory(fileMap map[string]*string, dir string) error {
for k, v := range fileMap { for k, v := range fileMap {
b, err := ioutil.ReadFile(filepath.Join(dir, k)) fname := filepath.Join(dir, k)
if strings.ContainsAny(fname, "?*[") {
// path contains wildcards, let's find by Glob needed file.
files, err := filepath.Glob(fname)
switch {
case err != nil:
continue
case len(files) != 1:
// doesn't match unique file, skip it
// fmt.Println("KAD2: ", files)
continue
}
fname = files[0]
}
b, err := ioutil.ReadFile(fname)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
continue continue
@ -36,3 +51,24 @@ func readFilesInDirectory(fileMap map[string]*string, dir string) error {
} }
return nil return nil
} }
// returns filename of the argument after resolving symlinks
func cleanBasename(name string) string {
realPath, err := filepath.EvalSymlinks(name)
if err != nil {
realPath = name
}
return filepath.Base(realPath)
}
// check that FPGA device is a compatible PCI device
func checkVendorAndClass(dev commonFpgaAPI) error {
pci, err := dev.GetPCIDevice()
if err != nil {
return err
}
if pci.Vendor != vendorIntel || pci.Class != fpgaClass {
return errors.Errorf("unsupported PCI device %s VID=%s PID=%s Class=%s", pci.BDF, pci.Vendor, pci.Device, pci.Class)
}
return nil
}