Add HostDeviceNetwork basic support

README.md

@@ -4,7 +4,7 @@
 
 - [Nvidia Network Operator](#nvidia-network-operator)
   * [Prerequisites](#prerequisites)
-    + [Kubernetes Node Feature Discovery (NFD)](#kubernetes-node-feature-discovery--nfd-)
+    + [Kubernetes Node Feature Discovery (NFD)](#kubernetes-node-feature-discovery-nfd)
   * [Resource Definitions](#resource-definitions)
     + [NICClusterPolicy CRD](#nicclusterpolicy-crd)
       - [NICClusterPolicy spec](#nicclusterpolicy-spec)
@@ -14,6 +14,9 @@
     + [MacvlanNetwork CRD](#macvlannetwork-crd)
       - [MacvlanNetwork spec](#macvlannetwork-spec)
         * [Example for MacvlanNetwork resource:](#example-for-macvlannetwork-resource)
+    + [HostDeviceNetwork CRD](#hostdevicenetwork-crd)
+      - [HostDeviceNetwork spec](#hostdevicenetwork-spec)
+        * [Example for HostDeviceNetwork resource:](#example-for-hostdevicenetwork-resource)
   * [System Requirements](#system-requirements)
   * [Deployment Example](#deployment-example)
   * [Driver Containers](#driver-containers)
@@ -115,6 +118,23 @@ spec:
           }
         ]
       }
+  sriovDevicePlugin:
+    image: sriov-device-plugin
+    repository: docker.io/nfvpe
+    version: v3.3
+    config: |
+      {
+        "resourceList": [
+            {
+                "resourcePrefix": "nvidia.com",
+                "resourceName": "hostdev",
+                "selectors": {
+                    "vendors": ["15b3"],
+                    "isRdma": true
+                }
+            }
+        ]
+      }
   secondaryNetwork:
     cniPlugins:
       image: containernetworking-plugins
@@ -211,6 +231,45 @@ spec:
 
 Can be found at: `example/crs/mellanox.com_v1alpha1_macvlannetwork_cr.yaml`
 
+### HostDeviceNetwork CRD
+This CRD defines a HostDevice secondary network. It is translated by the Operator to a `NetworkAttachmentDefinition` instance as defined in [k8snetworkplumbingwg/multi-net-spec](https://github.com/k8snetworkplumbingwg/multi-net-spec).
+
+#### HostDeviceNetwork spec:
+HostDeviceNetwork CRD Spec includes the following fields:
+- `networkNamespace`: Namespace for NetworkAttachmentDefinition related to this HostDeviceNetwork CRD.
+- `ResourceName`: Host device resource pool.
+- `ipam`: IPAM configuration to be used for this network.
+
+##### Example for HostDeviceNetwork resource:
+In the example below we deploy HostDeviceNetwork CRD instance with "hostdev" resource pool, that will be used to deploy NetworkAttachmentDefinition for HostDevice network to default namespace.
+
+```
+apiVersion: mellanox.com/v1alpha1
+kind: HostDeviceNetwork
+metadata:
+  name: example-hostdevice-network
+spec:
+  networkNamespace: "default"
+  ResourceName: "hostdev"
+  ipam: |
+    {
+      "type": "whereabouts",
+      "datastore": "kubernetes",
+      "kubernetes": {
+        "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig"
+      },
+      "range": "192.168.3.225/28",
+      "exclude": [
+       "192.168.3.229/30",
+       "192.168.3.236/32"
+      ],
+      "log_file" : "/var/log/whereabouts.log",
+      "log_level" : "info"
+    }
+```
+
+Can be found at: `mellanox.com_v1alpha1_hostdevicenetwork_cr.yaml`
+
 ## System Requirements
 * RDMA capable hardware: Mellanox ConnectX-4 NIC or newer.
 * NVIDIA GPU and driver supporting GPUDirect e.g Quadro RTX 6000/8000 or Tesla T4 or Tesla V100 or Tesla V100.

api/v1alpha1/hostdevicenetwork_types.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2021 NVIDIA
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	HostDeviceNetworkCRDName = "HostDeviceNetwork"
+)
+
+// HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork
+type HostDeviceNetworkSpec struct {
+	// Namespace of the NetworkAttachmentDefinition custom resource
+	NetworkNamespace string `json:"networkNamespace,omitempty"`
+	// Host device resource pool name
+	ResourceName string `json:"resourceName,omitempty"`
+	// IPAM configuration to be used for this network
+	IPAM string `json:"ipam,omitempty"`
+}
+
+// HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork
+type HostDeviceNetworkStatus struct {
+	// Reflects the state of the HostDeviceNetwork
+	// +kubebuilder:validation:Enum={"notReady", "ready", "error"}
+	State State `json:"state"`
+	// Network attachment definition generated from HostDeviceNetworkSpec
+	HostDeviceNetworkAttachmentDef string `json:"hostDeviceNetworkAttachmentDef,omitempty"`
+	// Informative string in case the observed state is error
+	Reason string `json:"reason,omitempty"`
+	// AppliedStates provide a finer view of the observed state
+	AppliedStates []AppliedState `json:"appliedStates,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:object:generate=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster
+// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0
+// +kubebuilder:printcolumn:name="Age",type=string,JSONPath=`.metadata.creationTimestamp`,priority=0
+
+// HostDeviceNetwork is the Schema for the hostdevicenetworks API
+type HostDeviceNetwork struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   HostDeviceNetworkSpec   `json:"spec,omitempty"`
+	Status HostDeviceNetworkStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:object:generate=true
+
+// HostDeviceNetworkList contains a list of HostDeviceNetwork
+type HostDeviceNetworkList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HostDeviceNetwork `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&HostDeviceNetwork{}, &HostDeviceNetworkList{})
+}

api/v1alpha1/nicclusterpolicy_types.go

@@ -95,6 +95,7 @@ type NicClusterPolicySpec struct {
 	OFEDDriver             *OFEDDriverSpec       `json:"ofedDriver,omitempty"`
 	NVPeerDriver           *NVPeerDriverSpec     `json:"nvPeerDriver,omitempty"`
 	RdmaSharedDevicePlugin *DevicePluginSpec     `json:"rdmaSharedDevicePlugin,omitempty"`
+	SriovDevicePlugin      *DevicePluginSpec     `json:"sriovDevicePlugin,omitempty"`
 	SecondaryNetwork       *SecondaryNetworkSpec `json:"secondaryNetwork,omitempty"`
 }
 

config/crd/bases/mellanox.com_hostdevicenetworks.yaml

@@ -0,0 +1,106 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: hostdevicenetworks.mellanox.com
+spec:
+  group: mellanox.com
+  names:
+    kind: HostDeviceNetwork
+    listKind: HostDeviceNetworkList
+    plural: hostdevicenetworks
+    singular: hostdevicenetwork
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.state
+      name: Status
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: string
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: HostDeviceNetwork is the Schema for the hostdevicenetworks API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork
+            properties:
+              ipam:
+                description: IPAM configuration to be used for this network
+                type: string
+              networkNamespace:
+                description: Namespace of the NetworkAttachmentDefinition custom resource
+                type: string
+              resourceName:
+                description: Host device resource pool name
+                type: string
+            type: object
+          status:
+            description: HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork
+            properties:
+              appliedStates:
+                description: AppliedStates provide a finer view of the observed state
+                items:
+                  description: AppliedState defines a finer-grained view of the observed
+                    state of NicClusterPolicy
+                  properties:
+                    name:
+                      type: string
+                    state:
+                      description: Represents reconcile state of the system
+                      enum:
+                      - ready
+                      - notReady
+                      - ignore
+                      - error
+                      type: string
+                  required:
+                  - name
+                  - state
+                  type: object
+                type: array
+              hostDeviceNetworkAttachmentDef:
+                description: Network attachment definition generated from HostDeviceNetworkSpec
+                type: string
+              reason:
+                description: Informative string in case the observed state is error
+                type: string
+              state:
+                description: Reflects the state of the HostDeviceNetwork
+                enum:
+                - notReady
+                - ready
+                - error
+                type: string
+            required:
+            - state
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []