Configure SR-IOV device plugin in NicClusterPolicy

Signed-off-by: Ivan Kolodiazhnyi <ikolodiazhny@nvidia.com>
Mellanox · Mar 9, 2021 · e0ad19b · e0ad19b
1 parent 882eb46
commit e0ad19b
Show file tree

Hide file tree

Showing 16 changed files with 170 additions and 168 deletions.
diff --git a/api/v1alpha1/hostdevicenetwork_types.go b/api/v1alpha1/hostdevicenetwork_types.go
@@ -27,8 +27,7 @@ const (
 // HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork
 type HostDeviceNetworkSpec struct {
 	// Namespace of the NetworkAttachmentDefinition custom resource
-	NetworkNamespace string            `json:"networkNamespace,omitempty"`
-	DevicePlugin     *DevicePluginSpec `json:"devicePlugin,omitempty"`
+	NetworkNamespace string `json:"networkNamespace,omitempty"`
 	//  configuration to be used for this network.
 	IPAM string `json:"ipam,omitempty"`
 }
@@ -50,6 +49,8 @@ type HostDeviceNetworkStatus struct {
 // kubebuilder:object:generate
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster
+// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0
+// +kubebuilder:printcolumn:name="Age",type=string,JSONPath=`.metadata.creationTimestamp`,priority=0
 
 // MacvlanNetwork is the Schema for the macvlannetworks API
 type HostDeviceNetwork struct {

diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -92,10 +92,11 @@ type NicClusterPolicySpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	OFEDDriver       *OFEDDriverSpec       `json:"ofedDriver,omitempty"`
-	NVPeerDriver     *NVPeerDriverSpec     `json:"nvPeerDriver,omitempty"`
-	DevicePlugin     *DevicePluginSpec     `json:"devicePlugin,omitempty"`
-	SecondaryNetwork *SecondaryNetworkSpec `json:"secondaryNetwork,omitempty"`
+	OFEDDriver             *OFEDDriverSpec       `json:"ofedDriver,omitempty"`
+	NVPeerDriver           *NVPeerDriverSpec     `json:"nvPeerDriver,omitempty"`
+	RdmaSharedDevicePlugin *DevicePluginSpec     `json:"rdmaSharedDevicePlugin,omitempty"`
+	SriovDevicePlugin      *DevicePluginSpec     `json:"sriovDevicePlugin,omitempty"`
+	SecondaryNetwork       *SecondaryNetworkSpec `json:"secondaryNetwork,omitempty"`
 }
 
 // AppliedState defines a finer-grained view of the observed state of NicClusterPolicy

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/mellanox.com_hostdevicenetworks.yaml b/config/crd/bases/mellanox.com_hostdevicenetworks.yaml
@@ -16,107 +16,85 @@ spec:
     singular: hostdevicenetwork
   scope: Cluster
   versions:
-    - additionalPrinterColumns:
-        - jsonPath: .status.state
-          name: Status
-          type: string
-        - jsonPath: .metadata.creationTimestamp
-          name: Age
-          type: date
-      name: v1alpha1
-      schema:
-        openAPIV3Schema:
-          description: MacvlanNetwork is the Schema for the macvlannetworks API
-          properties:
-            apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation
+  - additionalPrinterColumns:
+    - jsonPath: .status.state
+      name: Status
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: string
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: MacvlanNetwork is the Schema for the macvlannetworks API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
               of an object. Servers should convert recognized schemas to the latest
               internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
-            kind:
-              description: 'Kind is a string value representing the REST resource this
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
               object represents. Servers may infer this from the endpoint the client
               submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
-            metadata:
-              type: object
-            spec:
-              description: HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork
-              properties:
-                devicePlugin:
-                  description: DevicePluginSpec describes configuration options for
-                    device plugin
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork
+            properties:
+              ipam:
+                description: ' configuration to be used for this network.'
+                type: string
+              networkNamespace:
+                description: Namespace of the NetworkAttachmentDefinition custom resource
+                type: string
+            type: object
+          status:
+            description: HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork
+            properties:
+              appliedStates:
+                description: AppliedStates provide a finer view of the observed state
+                items:
+                  description: AppliedState defines a finer-grained view of the observed
+                    state of NicClusterPolicy
                   properties:
-                    config:
-                      description: Device plugin configuration
+                    name:
                       type: string
-                    image:
-                      pattern: '[a-zA-Z0-9\-]+'
-                      type: string
-                    repository:
-                      pattern: '[a-zA-Z0-9\.\-\/]+'
-                      type: string
-                    version:
-                      pattern: '[a-zA-Z0-9\.-]+'
+                    state:
+                      description: Represents reconcile state of the system
+                      enum:
+                      - ready
+                      - notReady
+                      - ignore
+                      - error
                       type: string
                   required:
-                    - config
-                    - image
-                    - repository
-                    - version
+                  - name
+                  - state
                   type: object
-                ipam:
-                  description: ' configuration to be used for this network.'
-                  type: string
-                networkNamespace:
-                  description: Namespace of the NetworkAttachmentDefinition custom resource
-                  type: string
-              type: object
-            status:
-              description: HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork
-              properties:
-                appliedStates:
-                  description: AppliedStates provide a finer view of the observed state
-                  items:
-                    description: AppliedState defines a finer-grained view of the observed
-                      state of NicClusterPolicy
-                    properties:
-                      name:
-                        type: string
-                      state:
-                        description: Represents reconcile state of the system
-                        enum:
-                          - ready
-                          - notReady
-                          - ignore
-                          - error
-                        type: string
-                    required:
-                      - name
-                      - state
-                    type: object
-                  type: array
-                hostDeviceNetworkAttachmentDef:
-                  description: Network attachment definition generated from HostDeviceNetworkSpec
-                  type: string
-                reason:
-                  description: Informative string in case the observed state is error
-                  type: string
-                state:
-                  description: Reflects the state of the MacvlanNetwork
-                  enum:
-                    - notReady
-                    - ready
-                    - error
-                  type: string
-              required:
-                - state
-              type: object
-          type: object
-      served: true
-      storage: true
-      subresources:
-        status: {}
+                type: array
+              hostDeviceNetworkAttachmentDef:
+                description: Network attachment definition generated from HostDeviceNetworkSpec
+                type: string
+              reason:
+                description: Informative string in case the observed state is error
+                type: string
+              state:
+                description: Reflects the state of the MacvlanNetwork
+                enum:
+                - notReady
+                - ready
+                - error
+                type: string
+            required:
+            - state
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
 status:
   acceptedNames:
     kind: ""

diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
@@ -40,12 +40,12 @@ spec:
           spec:
             description: NicClusterPolicySpec defines the desired state of NicClusterPolicy
             properties:
-              devicePlugin:
-                description: DevicePluginSpec describes configuration options for
-                  device plugin
+              nvPeerDriver:
+                description: NVPeerDriverSpec describes configuration options for
+                  NV Peer Memory driver
                 properties:
-                  config:
-                    description: Device plugin configuration
+                  gpuDriverSourcePath:
+                    description: GPU driver sources path - Optional
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -57,18 +57,14 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
-                - config
                 - image
                 - repository
                 - version
                 type: object
-              nvPeerDriver:
-                description: NVPeerDriverSpec describes configuration options for
-                  NV Peer Memory driver
+              ofedDriver:
+                description: OFEDDriverSpec describes configuration options for OFED
+                  driver
                 properties:
-                  gpuDriverSourcePath:
-                    description: GPU driver sources path - Optional
-                    type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
                     type: string
@@ -83,10 +79,13 @@ spec:
                 - repository
                 - version
                 type: object
-              ofedDriver:
-                description: OFEDDriverSpec describes configuration options for OFED
-                  driver
+              rdmaSharedDevicePlugin:
+                description: DevicePluginSpec describes configuration options for
+                  device plugin
                 properties:
+                  config:
+                    description: Device plugin configuration
+                    type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
                     type: string
@@ -97,6 +96,7 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
+                - config
                 - image
                 - repository
                 - version
@@ -163,6 +163,28 @@ spec:
                     - version
                     type: object
                 type: object
+              sriovDevicePlugin:
+                description: DevicePluginSpec describes configuration options for
+                  device plugin
+                properties:
+                  config:
+                    description: Device plugin configuration
+                    type: string
+                  image:
+                    pattern: '[a-zA-Z0-9\-]+'
+                    type: string
+                  repository:
+                    pattern: '[a-zA-Z0-9\.\-\/]+'
+                    type: string
+                  version:
+                    pattern: '[a-zA-Z0-9\.-]+'
+                    type: string
+                required:
+                - config
+                - image
+                - repository
+                - version
+                type: object
             type: object
           status:
             description: NicClusterPolicyStatus defines the observed state of NicClusterPolicy

diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
@@ -36,7 +36,7 @@ spec:
           spec:
             description: NicClusterPolicySpec defines the desired state of NicClusterPolicy
             properties:
-              devicePlugin:
+              rdmaSharedDevicePlugin:
                 description: DevicePluginSpec describes configuration options for
                   device plugin
                 properties:

diff --git a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml
@@ -32,19 +32,19 @@ spec:
     version: {{ .Values.nvPeerDriver.version }}
     gpuDriverSourcePath: {{ .Values.nvPeerDriver.gpuDriverSourcePath }}
   {{- end }}
-  {{- if .Values.devicePlugin.deploy }}
-  devicePlugin:
-    # {{ required "A valid value for .Values.devicePlugin.resources is required" .Values.devicePlugin.resources }}
-    image: {{ .Values.devicePlugin.image }}
-    repository: {{ .Values.devicePlugin.repository }}
-    version: {{ .Values.devicePlugin.version }}
+  {{- if .Values.rdmaSharedDevicePlugin.deploy }}
+  rdmaSharedDevicePlugin:
+    # {{ required "A valid value for .Values.rdmaSharedDevicePlugin.resources is required" .Values.rdmaSharedDevicePlugin.resources }}
+    image: {{ .Values.rdmaSharedDevicePlugin.image }}
+    repository: {{ .Values.rdmaSharedDevicePlugin.repository }}
+    version: {{ .Values.rdmaSharedDevicePlugin.version }}
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |
       {
         "configList": [
-          {{- $length := len .Values.devicePlugin.resources }}
-          {{- range $index, $element := .Values.devicePlugin.resources }}
+          {{- $length := len .Values.rdmaSharedDevicePlugin.resources }}
+          {{- range $index, $element := .Values.rdmaSharedDevicePlugin.resources }}
           {
             "resourceName": {{ $element.name | quote }},
             "rdmaHcaMax": 1000,

diff --git a/deployment/network-operator/templates/tests/test-dev-plugin.yaml b/deployment/network-operator/templates/tests/test-dev-plugin.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.deployCR .Values.devicePlugin.deploy  }}
+{{- if and .Values.deployCR .Values.rdmaSharedDevicePlugin.deploy  }}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -16,9 +16,9 @@ spec:
           add: [ "IPC_LOCK" ]
       resources:
         requests:
-          rdma/{{ (index .Values.devicePlugin.resources 0).name }}: '1'
+          rdma/{{ (index .Values.rdmaSharedDevicePlugin.resources 0).name }}: '1'
         limits:
-          rdma/{{ (index .Values.devicePlugin.resources 0).name }}: '1'
+          rdma/{{ (index .Values.rdmaSharedDevicePlugin.resources 0).name }}: '1'
       command:
         - sh
         - -c