Merge pull request #659 from almaslennikov/guid-config

Configure IB VFs' GUIDs using a statically provided GUID pool

doc/ib-vf-guid-static-configuration.md

@@ -0,0 +1,90 @@
+# Infiniband VF GUID Static Configuration
+
+SR-IOV Network Operator is able to use a static configuration file from the host filesystem to assign GUIDs to IB VFs.
+
+## Prerequisites
+
+- Infiniband NICs
+- Static GUID configuration file on the host filesystem
+
+### Configuration file
+
+Config file should be stored at `/etc/sriov-operator/infiniband/guids`. This location is writable across most cloud platforms.
+
+VF to GUID assignment, based on this file, is ordered. VF0 takes the GUID0, VF1 takes the GUID1 etc.
+
+Each PF has its own set of GUIDs.
+
+Example of the config file:
+
+```json
+[
+  {
+    "pci_address": "<pci_address_1>",
+    "guids": [
+      "02:00:00:00:00:00:00:00",
+      "02:00:00:00:00:00:00:01"
+    ]
+  },
+  {
+    "pf_guid": "<pf_guid_2>",
+    "guidsRange": {
+      "start": "02:00:00:00:00:aa:00:02",
+      "end": "02:00:00:00:00:aa:00:0a"
+    }
+  }
+]
+```
+
+Config file parameters:
+
+* `pci_address` is a PCI address of a PF
+* `pf_guid` is a GUID of a PF. Can be obtained with `ibstat` command.
+* `guids` is an array of VF GUID strings
+* `guidsRange` is an object representing the start and end of a VF GUID range. It has two fields:
+    * `start` is a VF GUID range start
+    * `end` is a VF GUID range end
+
+Requirements for the config file:
+
+* `pci_address` and `pf_guid` cannot be set at the same time for a single device - should return an error
+* if the list contains multiple entries for the same device, the first one shall be taken
+* `rangeStart` and `rangeEnd` are both included in the range
+* `guids` list and range cannot be both set at the same time for a single device - should return an error
+* GUIDs are assigned once and not change throughout the lifecycle of the host
+
+### Deploy SriovNetworkNodePolicy
+
+```yaml
+apiVersion: sriovnetwork.openshift.io/v1
+kind: SriovNetworkNodePolicy
+metadata:
+  name: ib-policy
+  namespace: sriov-network-operator
+spec:
+  nodeSelector:
+    feature.node.kubernetes.io/network-sriov.capable: "true"
+  resourceName: ib_vfs
+  priority: 10
+  numVfs: 2
+  nicSelector:
+    vendor: "15b3"
+    rootDevices:
+    - 0000:d8.00.0
+  deviceType: netdevice
+```
+
+## Verify assigned GUIDs
+
+Run ip link tool to verify assigned GUIDs:
+
+```bash
+ip link
+
+...
+6: ibp216s0f0: <BROADCAST,MULTICAST> mtu 4092 qdisc noop state UP mode DEFAULT group default qlen 256
+    link/infiniband 00:00:05:a9:fe:80:00:00:00:00:00:00:b8:59:9f:03:00:f9:8f:86 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
+    vf 0     link/infiniband ... NODE_GUID 02:00:00:00:00:00:00:00, PORT_GUID 02:00:00:00:00:00:00:00, link-state enable, trust off, query_rss off
+    vf 1     link/infiniband ... NODE_GUID 02:00:00:00:00:00:00:01, PORT_GUID 02:00:00:00:00:00:00:01, link-state enable, trust off, query_rss off
+```
+

pkg/consts/constants.go

@@ -138,6 +138,9 @@ const (
 
 	// ManageSoftwareBridgesFeatureGate: enables management of software bridges by the operator
 	ManageSoftwareBridgesFeatureGate = "manageSoftwareBridges"
+
+	// The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs
+	InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids"
 )
 
 const (

pkg/helper/host.go

@@ -35,7 +35,11 @@ func NewHostHelpers(utilsHelper utils.CmdInterface,
 func NewDefaultHostHelpers() (HostHelpersInterface, error) {
 	utilsHelper := utils.New()
 	mlxHelper := mlx.New(utilsHelper)
-	hostManager := host.NewHostManager(utilsHelper)
+	hostManager, err := host.NewHostManager(utilsHelper)
+	if err != nil {
+		log.Log.Error(err, "failed to create host manager")
+		return nil, err
+	}
 	storeManager, err := store.NewManager()
 	if err != nil {
 		log.Log.Error(err, "failed to create store manager")

pkg/host/internal/infiniband/guid.go

@@ -0,0 +1,62 @@
+package infiniband
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+)
+
+// GUID address is an uint64 encapsulation for network hardware address
+type GUID uint64
+
+const (
+	guidLength = 8
+	byteBitLen = 8
+	byteMask   = 0xff
+)
+
+// ParseGUID parses string only as GUID 64 bit
+func ParseGUID(s string) (GUID, error) {
+	ha, err := net.ParseMAC(s)
+	if err != nil {
+		return 0, err
+	}
+	if len(ha) != guidLength {
+		return 0, fmt.Errorf("invalid GUID address %s", s)
+	}
+	var guid uint64
+	for idx, octet := range ha {
+		guid |= uint64(octet) << uint(byteBitLen*(guidLength-1-idx))
+	}
+	return GUID(guid), nil
+}
+
+// String returns the string representation of GUID
+func (g GUID) String() string {
+	return g.HardwareAddr().String()
+}
+
+// HardwareAddr returns GUID representation as net.HardwareAddr
+func (g GUID) HardwareAddr() net.HardwareAddr {
+	value := uint64(g)
+	ha := make(net.HardwareAddr, guidLength)
+	for idx := guidLength - 1; idx >= 0; idx-- {
+		ha[idx] = byte(value & byteMask)
+		value >>= byteBitLen
+	}
+
+	return ha
+}
+
+func generateRandomGUID() net.HardwareAddr {
+	guid := make(net.HardwareAddr, 8)
+
+	// First field is 0x01 - xfe to avoid all zero and all F invalid guids
+	guid[0] = byte(1 + rand.Intn(0xfe))
+
+	for i := 1; i < len(guid); i++ {
+		guid[i] = byte(rand.Intn(0x100))
+	}
+
+	return guid
+}

pkg/host/internal/infiniband/guid_test.go

@@ -0,0 +1,29 @@
+package infiniband
+
+import (
+	"net"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("GUID", func() {
+	It("should parse and process GUIDs correctly", func() {
+		guidStr := "00:01:02:03:04:05:06:08"
+		nextGuidStr := "00:01:02:03:04:05:06:09"
+
+		guid, err := ParseGUID(guidStr)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(guid.String()).To(Equal(guidStr))
+		Expect((guid + 1).String()).To(Equal(nextGuidStr))
+	})
+	It("should represent GUID as HW address", func() {
+		guidStr := "00:01:02:03:04:05:06:08"
+
+		guid, err := ParseGUID(guidStr)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(guid.HardwareAddr()).To(Equal(net.HardwareAddr{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08}))
+	})
+})

pkg/host/internal/infiniband/ib_guid_config.go

@@ -0,0 +1,126 @@
+package infiniband
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
+)
+
+type ibPfGUIDJSONConfig struct {
+	PciAddress string         `json:"pciAddress,omitempty"`
+	PfGUID     string         `json:"pfGuid,omitempty"`
+	GUIDs      []string       `json:"guids,omitempty"`
+	GUIDsRange *GUIDRangeJSON `json:"guidsRange,omitempty"`
+}
+
+type GUIDRangeJSON struct {
+	Start string `json:"start,omitempty"`
+	End   string `json:"end,omitempty"`
+}
+
+type ibPfGUIDConfig struct {
+	GUIDs     []GUID
+	GUIDRange *GUIDRange
+}
+
+type GUIDRange struct {
+	Start GUID
+	End   GUID
+}
+
+func getIbGUIDConfig(configPath string, netlinkLib netlink.NetlinkLib, networkHelper types.NetworkInterface) (map[string]ibPfGUIDConfig, error) {
+	links, err := netlinkLib.LinkList()
+	if err != nil {
+		return nil, err
+	}
+	rawConfigs, err := readJSONConfig(configPath)
+	if err != nil {
+		return nil, err
+	}
+	resultConfigs := map[string]ibPfGUIDConfig{}
+	// Parse JSON config into an internal struct
+	for _, rawConfig := range rawConfigs {
+		pciAddress, err := getPfPciAddressFromRawConfig(rawConfig, links, networkHelper)
+		if err != nil {
+			return nil, fmt.Errorf("failed to extract pci address from ib guid config: %w", err)
+		}
+		if len(rawConfig.GUIDs) == 0 && (rawConfig.GUIDsRange == nil || (rawConfig.GUIDsRange.Start == "" || rawConfig.GUIDsRange.End == "")) {
+			return nil, fmt.Errorf("either guid list or guid range should be provided, got none")
+		}
+		if len(rawConfig.GUIDs) != 0 && rawConfig.GUIDsRange != nil {
+			return nil, fmt.Errorf("either guid list or guid range should be provided, got both")
+		}
+		if rawConfig.GUIDsRange != nil && ((rawConfig.GUIDsRange.Start != "" && rawConfig.GUIDsRange.End == "") || (rawConfig.GUIDsRange.Start == "" && rawConfig.GUIDsRange.End != "")) {
+			return nil, fmt.Errorf("both guid rangeStart and rangeEnd should be provided, got one")
+		}
+		if len(rawConfig.GUIDs) != 0 {
+			var guids []GUID
+			for _, guidStr := range rawConfig.GUIDs {
+				guid, err := ParseGUID(guidStr)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse ib guid %s: %w", guidStr, err)
+				}
+				guids = append(guids, guid)
+			}
+			resultConfigs[pciAddress] = ibPfGUIDConfig{
+				GUIDs: guids,
+			}
+			continue
+		}
+
+		rangeStart, err := ParseGUID(rawConfig.GUIDsRange.Start)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse ib guid range start: %w", err)
+		}
+		rangeEnd, err := ParseGUID(rawConfig.GUIDsRange.End)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse ib guid range end: %w", err)
+		}
+		if rangeEnd < rangeStart {
+			return nil, fmt.Errorf("range end cannot be less then range start")
+		}
+		resultConfigs[pciAddress] = ibPfGUIDConfig{
+			GUIDRange: &GUIDRange{
+				Start: rangeStart,
+				End:   rangeEnd,
+			},
+		}
+	}
+	return resultConfigs, nil
+}
+
+// readJSONConfig reads the file at the given path and unmarshals the contents into an array of ibPfGUIDJSONConfig structs
+func readJSONConfig(configPath string) ([]ibPfGUIDJSONConfig, error) {
+	data, err := os.ReadFile(utils.GetHostExtensionPath(configPath))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read ib guid config: %w", err)
+	}
+	var configs []ibPfGUIDJSONConfig
+	if err := json.Unmarshal(data, &configs); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal content of ib guid config: %w", err)
+	}
+	return configs, nil
+}
+
+func getPfPciAddressFromRawConfig(pfRawConfig ibPfGUIDJSONConfig, links []netlink.Link, networkHelper types.NetworkInterface) (string, error) {
+	if pfRawConfig.PciAddress != "" && pfRawConfig.PfGUID != "" {
+		return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, both provided")
+	}
+	if pfRawConfig.PciAddress == "" && pfRawConfig.PfGUID == "" {
+		return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, none provided")
+	}
+	if pfRawConfig.PciAddress != "" {
+		return pfRawConfig.PciAddress, nil
+	}
+	// PfGUID is provided, need to resolve the pci address
+	for _, link := range links {
+		if link.Attrs().HardwareAddr.String() == pfRawConfig.PfGUID {
+			return networkHelper.GetPciAddressFromInterfaceName(link.Attrs().Name)
+		}
+	}
+	return "", fmt.Errorf("no matching link found for pf guid: %s", pfRawConfig.PfGUID)
+}