From d81d36a21dcbb03f335b8a7fc684c6c7bf6fbbc0 Mon Sep 17 00:00:00 2001 From: Ivan Kolodiazhnyi Date: Tue, 23 Feb 2021 12:23:07 +0200 Subject: [PATCH] Add HostDeviceNetwork basic support This patch adds HostDeviceNetwork CRD support and basic transformation to NetworkAttachmentDefinition to allow Multus create secondary network. SR-IOV Network Device Plugin is deployed using Network Operator Helm chart with default configuration. NOTE: temporary added "//nolint:dupl" before code will be refactored. Closes: #44 Signed-off-by: Ivan Kolodiazhnyi --- api/v1alpha1/hostdevicenetwork_types.go | 76 ++++++++++ api/v1alpha1/nicclusterpolicy_types.go | 3 +- api/v1alpha1/zz_generated.deepcopy.go | 99 ++++++++++++ .../mellanox.com_hostdevicenetworks.yaml | 103 +++++++++++++ .../mellanox.com_nicclusterpolicies.yaml | 22 +++ config/rbac/role.yaml | 20 +++ controllers/hostdevicenetwork_controller.go | 141 ++++++++++++++++++ controllers/macvlannetwork_controller.go | 3 +- controllers/nicclusterpolicy_controller.go | 1 + .../crds/mellanox.com_hostdevicenetworks.yaml | 125 ++++++++++++++++ ...anox.com_v1alpha1_nicclusterpolicy_cr.yaml | 25 ++++ deployment/network-operator/values.yaml | 9 ++ ...nox.com_v1alpha1_hostdevicenetwork_cr.yaml | 34 +++++ ...anox.com_v1alpha1_nicclusterpolicy_cr.yaml | 17 +++ example/hostdevice-network-pod.yml | 18 +++ main.go | 8 + .../0010-hostdevice-net-cr.yml | 14 ++ .../0010-sriov-dp-configmap.yml | 20 +++ .../0020-sriov-dp-service-acocunt.yml | 18 +++ .../0030-sriov-dp-daemonset.yml | 83 +++++++++++ pkg/state/factory.go | 22 +++ pkg/state/state_hostdevice_network.go | 141 ++++++++++++++++++ pkg/state/state_macvlan_network.go | 2 +- pkg/state/state_multus_cni.go | 2 +- pkg/state/state_shared_dp.go | 3 +- pkg/state/state_sriov_dp.go | 125 ++++++++++++++++ pkg/state/state_whereabouts_cni.go | 10 +- scripts/deploy-operator.sh | 1 + 28 files changed, 1132 insertions(+), 13 deletions(-) create mode 100644 api/v1alpha1/hostdevicenetwork_types.go create mode 100644 config/crd/bases/mellanox.com_hostdevicenetworks.yaml create mode 100644 controllers/hostdevicenetwork_controller.go create mode 100644 deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml create mode 100644 example/crs/mellanox.com_v1alpha1_hostdevicenetwork_cr.yaml create mode 100644 example/hostdevice-network-pod.yml create mode 100644 manifests/stage-hostdevice-network/0010-hostdevice-net-cr.yml create mode 100644 manifests/stage-sriov-device-plugin/0010-sriov-dp-configmap.yml create mode 100644 manifests/stage-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml create mode 100644 manifests/stage-sriov-device-plugin/0030-sriov-dp-daemonset.yml create mode 100644 pkg/state/state_hostdevice_network.go create mode 100644 pkg/state/state_sriov_dp.go diff --git a/api/v1alpha1/hostdevicenetwork_types.go b/api/v1alpha1/hostdevicenetwork_types.go new file mode 100644 index 000000000..6cf752046 --- /dev/null +++ b/api/v1alpha1/hostdevicenetwork_types.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + HostDeviceNetworkCRDName = "HostDeviceNetwork" +) + +// HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork +type HostDeviceNetworkSpec struct { + // Namespace of the NetworkAttachmentDefinition custom resource + NetworkNamespace string `json:"networkNamespace,omitempty"` + // configuration to be used for this network. + IPAM string `json:"ipam,omitempty"` +} + +// HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork +type HostDeviceNetworkStatus struct { + // Reflects the state of the MacvlanNetwork + // +kubebuilder:validation:Enum={"notReady", "ready", "error"} + State State `json:"state"` + // Network attachment definition generated from HostDeviceNetworkSpec + HostDeviceNetworkAttachmentDef string `json:"hostDeviceNetworkAttachmentDef,omitempty"` + // Informative string in case the observed state is error + Reason string `json:"reason,omitempty"` + // AppliedStates provide a finer view of the observed state + AppliedStates []AppliedState `json:"appliedStates,omitempty"` +} + +// +kubebuilder:object:root=true +// + =true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0 +// +kubebuilder:printcolumn:name="Age",type=string,JSONPath=`.metadata.creationTimestamp`,priority=0 + +// HostDeviceNetwork is the Schema for the hostdevicenetworks API +type HostDeviceNetwork struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec HostDeviceNetworkSpec `json:"spec,omitempty"` + Status HostDeviceNetworkStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:object:generate=true + +// HostDeviceNetworkList contains a list of HostDeviceNetwork +type HostDeviceNetworkList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []HostDeviceNetwork `json:"items"` +} + +func init() { + SchemeBuilder.Register(&HostDeviceNetwork{}, &HostDeviceNetworkList{}) +} diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go index 58113a9f9..622e5757b 100644 --- a/api/v1alpha1/nicclusterpolicy_types.go +++ b/api/v1alpha1/nicclusterpolicy_types.go @@ -95,6 +95,7 @@ type NicClusterPolicySpec struct { OFEDDriver *OFEDDriverSpec `json:"ofedDriver,omitempty"` NVPeerDriver *NVPeerDriverSpec `json:"nvPeerDriver,omitempty"` RdmaSharedDevicePlugin *DevicePluginSpec `json:"rdmaSharedDevicePlugin,omitempty"` + SriovDevicePlugin *DevicePluginSpec `json:"sriovDevicePlugin,omitempty"` SecondaryNetwork *SecondaryNetworkSpec `json:"secondaryNetwork,omitempty"` } @@ -120,7 +121,7 @@ type NicClusterPolicyStatus struct { } // +kubebuilder:object:root=true -// kubebuilder:object:generate +// +kubebuilder:object:generate=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 20974d1bc..f2886714a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -55,6 +55,100 @@ func (in *DevicePluginSpec) DeepCopy() *DevicePluginSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostDeviceNetwork) DeepCopyInto(out *HostDeviceNetwork) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostDeviceNetwork. +func (in *HostDeviceNetwork) DeepCopy() *HostDeviceNetwork { + if in == nil { + return nil + } + out := new(HostDeviceNetwork) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HostDeviceNetwork) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostDeviceNetworkList) DeepCopyInto(out *HostDeviceNetworkList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]HostDeviceNetwork, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostDeviceNetworkList. +func (in *HostDeviceNetworkList) DeepCopy() *HostDeviceNetworkList { + if in == nil { + return nil + } + out := new(HostDeviceNetworkList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HostDeviceNetworkList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostDeviceNetworkSpec) DeepCopyInto(out *HostDeviceNetworkSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostDeviceNetworkSpec. +func (in *HostDeviceNetworkSpec) DeepCopy() *HostDeviceNetworkSpec { + if in == nil { + return nil + } + out := new(HostDeviceNetworkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostDeviceNetworkStatus) DeepCopyInto(out *HostDeviceNetworkStatus) { + *out = *in + if in.AppliedStates != nil { + in, out := &in.AppliedStates, &out.AppliedStates + *out = make([]AppliedState, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostDeviceNetworkStatus. +func (in *HostDeviceNetworkStatus) DeepCopy() *HostDeviceNetworkStatus { + if in == nil { + return nil + } + out := new(HostDeviceNetworkStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { *out = *in @@ -268,6 +362,11 @@ func (in *NicClusterPolicySpec) DeepCopyInto(out *NicClusterPolicySpec) { *out = new(DevicePluginSpec) **out = **in } + if in.SriovDevicePlugin != nil { + in, out := &in.SriovDevicePlugin, &out.SriovDevicePlugin + *out = new(DevicePluginSpec) + **out = **in + } if in.SecondaryNetwork != nil { in, out := &in.SecondaryNetwork, &out.SecondaryNetwork *out = new(SecondaryNetworkSpec) diff --git a/config/crd/bases/mellanox.com_hostdevicenetworks.yaml b/config/crd/bases/mellanox.com_hostdevicenetworks.yaml new file mode 100644 index 000000000..6b2721e7a --- /dev/null +++ b/config/crd/bases/mellanox.com_hostdevicenetworks.yaml @@ -0,0 +1,103 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: hostdevicenetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: HostDeviceNetwork + listKind: HostDeviceNetworkList + plural: hostdevicenetworks + singular: hostdevicenetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: MacvlanNetwork is the Schema for the macvlannetworks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork + properties: + ipam: + description: ' configuration to be used for this network.' + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + name: + type: string + state: + description: Represents reconcile state of the system + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + hostDeviceNetworkAttachmentDef: + description: Network attachment definition generated from HostDeviceNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the MacvlanNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml index 502497280..1834af8e3 100644 --- a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml +++ b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml @@ -163,6 +163,28 @@ spec: - version type: object type: object + sriovDevicePlugin: + description: DevicePluginSpec describes configuration options for + device plugin + properties: + config: + description: Device plugin configuration + type: string + image: + pattern: '[a-zA-Z0-9\-]+' + type: string + repository: + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + pattern: '[a-zA-Z0-9\.-]+' + type: string + required: + - config + - image + - repository + - version + type: object type: object status: description: NicClusterPolicyStatus defines the observed state of NicClusterPolicy diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 77a3f17eb..c976b5645 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -6,6 +6,26 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks/status + verbs: + - get + - patch + - update - apiGroups: - mellanox.com resources: diff --git a/controllers/hostdevicenetwork_controller.go b/controllers/hostdevicenetwork_controller.go new file mode 100644 index 000000000..c5407b610 --- /dev/null +++ b/controllers/hostdevicenetwork_controller.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package controllers //nolint:dupl + +import ( + "context" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + mellanoxcomv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1" + "github.com/Mellanox/network-operator/pkg/config" + "github.com/Mellanox/network-operator/pkg/consts" + "github.com/Mellanox/network-operator/pkg/state" +) + +// HostDeviceNetworkReconciler reconciles a HostDeviceNetwork object +type HostDeviceNetworkReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + + stateManager state.Manager +} + +// +kubebuilder:rbac:groups=mellanox.com,resources=hostdevicenetworks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mellanox.com,resources=hostdevicenetworks/status,verbs=get;update;patch + +//nolint:dupl +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *HostDeviceNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reqLogger := r.Log.WithValues("hostdevicenetwork", req.NamespacedName) + reqLogger.Info("Reconciling HostDeviceNetwork") + + // Fetch the HostDeviceNetwork instance + instance := &mellanoxcomv1alpha1.HostDeviceNetwork{} + err := r.Get(context.TODO(), req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + managerStatus, err := r.stateManager.SyncState(instance, nil) + r.updateCrStatus(instance, managerStatus) + if err != nil { + return reconcile.Result{}, err + } + + if managerStatus.Status != state.SyncStateReady { + return reconcile.Result{ + RequeueAfter: time.Duration(config.FromEnv().Controller.RequeueTimeSeconds) * time.Second, + }, nil + } + + return ctrl.Result{}, nil +} + +//nolint:dupl +func (r *HostDeviceNetworkReconciler) updateCrStatus(cr *mellanoxcomv1alpha1.HostDeviceNetwork, status state.Results) { +NextResult: + for _, stateStatus := range status.StatesStatus { + // basically iterate over results and add/update crStatus.AppliedStates + for i := range cr.Status.AppliedStates { + if cr.Status.AppliedStates[i].Name == stateStatus.StateName { + cr.Status.AppliedStates[i].State = mellanoxcomv1alpha1.State(stateStatus.Status) + continue NextResult + } + } + cr.Status.AppliedStates = append(cr.Status.AppliedStates, mellanoxcomv1alpha1.AppliedState{ + Name: stateStatus.StateName, + State: mellanoxcomv1alpha1.State(stateStatus.Status), + }) + } + // Update global State + cr.Status.State = mellanoxcomv1alpha1.State(status.Status) + + // send status update request to k8s API + r.Log.V(consts.LogLevelInfo).Info( + "Updating status", "Custom resource name", cr.Name, "namespace", cr.Namespace, "Result:", cr.Status) + err := r.Status().Update(context.TODO(), cr) + if err != nil { + r.Log.V(consts.LogLevelError).Info("Failed to update CR status", "error:", err) + } +} + +//nolint:dupl +// SetupWithManager sets up the controller with the Manager. +func (r *HostDeviceNetworkReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Create state manager + stateManager, err := state.NewManager(mellanoxcomv1alpha1.HostDeviceNetworkCRDName, mgr.GetClient(), mgr.GetScheme()) + if err != nil { + // Error creating stateManager + r.Log.V(consts.LogLevelError).Info("Error creating state manager.", "error:", err) + panic("Failed to create State manager") + } + r.stateManager = stateManager + + builder := ctrl.NewControllerManagedBy(mgr). + For(&mellanoxcomv1alpha1.HostDeviceNetwork{}). + // Watch for changes to primary resource HostDeviceNetwork + Watches(&source.Kind{Type: &mellanoxcomv1alpha1.HostDeviceNetwork{}}, &handler.EnqueueRequestForObject{}) + + // Watch for changes to secondary resource DaemonSet and requeue the owner HostDeviceNetwork + ws := stateManager.GetWatchSources() + r.Log.V(consts.LogLevelInfo).Info("Watch Sources", "Kind:", ws) + for i := range ws { + builder = builder.Watches(ws[i], &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &mellanoxcomv1alpha1.HostDeviceNetwork{}, + }) + } + + return builder.Complete(r) +} diff --git a/controllers/macvlannetwork_controller.go b/controllers/macvlannetwork_controller.go index bfc5ed32d..493a4c505 100644 --- a/controllers/macvlannetwork_controller.go +++ b/controllers/macvlannetwork_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package controllers //nolint:dupl import ( "context" @@ -47,6 +47,7 @@ type MacvlanNetworkReconciler struct { // +kubebuilder:rbac:groups=mellanox.com,resources=macvlannetworks,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mellanox.com,resources=macvlannetworks/status,verbs=get;update;patch +//nolint:dupl // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *MacvlanNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/controllers/nicclusterpolicy_controller.go b/controllers/nicclusterpolicy_controller.go index 9d4766bbb..52837e41a 100644 --- a/controllers/nicclusterpolicy_controller.go +++ b/controllers/nicclusterpolicy_controller.go @@ -114,6 +114,7 @@ func (r *NicClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } +//nolint:dupl func (r *NicClusterPolicyReconciler) updateCrStatus(cr *mellanoxv1alpha1.NicClusterPolicy, status state.Results) { NextResult: for _, stateStatus := range status.StatesStatus { diff --git a/deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml b/deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml new file mode 100644 index 000000000..9fa52117f --- /dev/null +++ b/deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml @@ -0,0 +1,125 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: hostdevicenetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: HostDeviceNetwork + listKind: HostDeviceNetworkList + plural: hostdevicenetworks + singular: hostdevicenetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: MacvlanNetwork is the Schema for the macvlannetworks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostDeviceNetworkSpec defines the desired state of HostDeviceNetwork + properties: + devicePlugin: + description: DevicePluginSpec describes configuration options for + device plugin + properties: + config: + description: Device plugin configuration + type: string + image: + pattern: '[a-zA-Z0-9\-]+' + type: string + repository: + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + pattern: '[a-zA-Z0-9\.-]+' + type: string + required: + - config + - image + - repository + - version + type: object + ipam: + description: ' configuration to be used for this network.' + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: HostDeviceNetworkStatus defines the observed state of HostDeviceNetwork + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + name: + type: string + state: + description: Represents reconcile state of the system + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + hostDeviceNetworkAttachmentDef: + description: Network attachment definition generated from HostDeviceNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the MacvlanNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml index 714325fa0..978eb71d9 100644 --- a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml +++ b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml @@ -60,6 +60,31 @@ spec: ] } {{- end }} + { { - if .Values.sriovDevicePlugin.deploy } } + sriovDevicePlugin: + # {{ required "A valid value for .Values.rdmaSharedDevicePlugin.resources is required" .Values.rdmaSharedDevicePlugin.resources }} + image: { { .Values.sriovDevicePlugin.image } } + repository: { { .Values.sriovDevicePlugin.repository } } + version: { { .Values.sriovDevicePlugin.version } } + # The config below directly propagates to sriov-network-device-plugin configuration. + # Replace 'devices' with your (RDMA capable) netdevice name. + config: | + { + "configList": [ + {{- $length := len .Values.sriovDevicePlugin.resources }} + {{- range $index, $element := .Values.sriovDevicePlugin.resources }} + { + "resourcePrefix": {{ $element.prefix | quote }}, + "resourceName": {{ $element.name | quote }}, + "selectors": { + "vendors": {{ $element.vendors | default list | toJson }}, + "isRdma": true + } + } {{- if ne $length (add1 $index) }},{{ end }} + {{- end }} + ] + } + { { - end } } {{- if .Values.secondaryNetwork.deploy }} secondaryNetwork: {{- if .Values.secondaryNetwork.cniPlugins.deploy }} diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml index 316ee17af..cce56a037 100644 --- a/deployment/network-operator/values.yaml +++ b/deployment/network-operator/values.yaml @@ -107,6 +107,15 @@ rdmaSharedDevicePlugin: - name: rdma_shared_device_a vendors: [15b3] +sriovDevicePlugin: + deploy: true + image: sriov-device-plugin + repository: docker.io/nfvpe + version: v3.3 + resources: + - name: mlnx_hostdev + - vendors: [15b3] + secondaryNetwork: deploy: true cniPlugins: diff --git a/example/crs/mellanox.com_v1alpha1_hostdevicenetwork_cr.yaml b/example/crs/mellanox.com_v1alpha1_hostdevicenetwork_cr.yaml new file mode 100644 index 000000000..c706cc416 --- /dev/null +++ b/example/crs/mellanox.com_v1alpha1_hostdevicenetwork_cr.yaml @@ -0,0 +1,34 @@ +# Copyright 2021 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: mellanox.com/v1alpha1 +kind: HostDeviceNetwork +metadata: + name: example-hostdevice-network +spec: + networkNamespace: "default" + ipam: | + { + "type": "whereabouts", + "datastore": "kubernetes", + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" + }, + "range": "192.168.3.225/28", + "exclude": [ + "192.168.3.229/30", + "192.168.3.236/32" + ], + "log_file" : "/var/log/whereabouts.log", + "log_level" : "info" + } diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml index 2b994d5d8..ea554ae14 100644 --- a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml +++ b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml @@ -38,6 +38,23 @@ spec: } ] } + sriovDevicePlugin: + image: sriov-device-plugin + repository: docker.io/nfvpe + version: v3.3 + config: | + { + "resourceList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "mlnx_hostdev", + "selectors": { + "vendors": ["15b3"], + "isRdma": true + } + } + ] + } secondaryNetwork: cniPlugins: image: containernetworking-plugins diff --git a/example/hostdevice-network-pod.yml b/example/hostdevice-network-pod.yml new file mode 100644 index 000000000..3069bbaf8 --- /dev/null +++ b/example/hostdevice-network-pod.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-hostdev-pod + annotations: + k8s.v1.cni.cncf.io/networks: example-hostdevice-network +spec: + containers: + - name: test-hostdev-pod + image: centos/tools + imagePullPolicy: IfNotPresent + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 300; done;" ] + resources: + requests: + nvidia.com/mlnx_hostdev: '1' + limits: + nvidia.com/mlnx_hostdev: '1' diff --git a/main.go b/main.go index e03df65a6..e69fbbb65 100644 --- a/main.go +++ b/main.go @@ -96,6 +96,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "MacvlanNetwork") os.Exit(1) } + if err = (&controllers.HostDeviceNetworkReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HostDeviceNetwork"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HostDeviceNetwork") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil { diff --git a/manifests/stage-hostdevice-network/0010-hostdevice-net-cr.yml b/manifests/stage-hostdevice-network/0010-hostdevice-net-cr.yml new file mode 100644 index 000000000..e6e3267be --- /dev/null +++ b/manifests/stage-hostdevice-network/0010-hostdevice-net-cr.yml @@ -0,0 +1,14 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: {{.HostDeviceNetworkName}} + namespace: {{.CrSpec.NetworkNamespace}} + annotations: + k8s.v1.cni.cncf.io/resourceName: nvidia.com/mlnx_hostdev +spec: + config: '{ + "cniVersion":"0.3.1", + "name":"{{.HostDeviceNetworkName}}", + "type":"host-device", + "ipam": {{.CrSpec.IPAM}} +}' diff --git a/manifests/stage-sriov-device-plugin/0010-sriov-dp-configmap.yml b/manifests/stage-sriov-device-plugin/0010-sriov-dp-configmap.yml new file mode 100644 index 000000000..f048e627c --- /dev/null +++ b/manifests/stage-sriov-device-plugin/0010-sriov-dp-configmap.yml @@ -0,0 +1,20 @@ +# Copyright 2021 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: {{ .RuntimeSpec.Namespace }} +data: + config.json: '{{ .CrSpec.Config }}' diff --git a/manifests/stage-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml b/manifests/stage-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml new file mode 100644 index 000000000..354b71de2 --- /dev/null +++ b/manifests/stage-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml @@ -0,0 +1,18 @@ +# Copyright 2021 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-device-plugin + namespace: {{ .RuntimeSpec.Namespace }} diff --git a/manifests/stage-sriov-device-plugin/0030-sriov-dp-daemonset.yml b/manifests/stage-sriov-device-plugin/0030-sriov-dp-daemonset.yml new file mode 100644 index 000000000..ecbd7ceb4 --- /dev/null +++ b/manifests/stage-sriov-device-plugin/0030-sriov-dp-daemonset.yml @@ -0,0 +1,83 @@ +# Copyright 2021 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sriov-device-plugin + namespace: {{ .RuntimeSpec.Namespace }} + labels: + tier: node + app: sriovdp +spec: + selector: + matchLabels: + name: sriov-device-plugin + template: + metadata: + labels: + name: sriov-device-plugin + tier: node + app: sriovdp + spec: + hostNetwork: true + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + tolerations: + # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode. + # This, along with the annotation above marks this pod as a critical add-on. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + serviceAccountName: sriov-device-plugin + containers: + - name: kube-sriovdp + image: {{ .CrSpec.ImageSpec.Repository }}/{{ .CrSpec.ImageSpec.Image }}:{{ .CrSpec.ImageSpec.Version }} + imagePullPolicy: IfNotPresent + args: + - --log-dir=sriovdp + - --log-level=10 + securityContext: + privileged: true + volumeMounts: + - name: devicesock + mountPath: /var/lib/kubelet/ + readOnly: false + - name: log + mountPath: /var/log + - name: config-volume + mountPath: /etc/pcidp + - name: device-info + mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp + volumes: + - name: devicesock + hostPath: + path: /var/lib/kubelet/ + - name: log + hostPath: + path: /var/log + - name: device-info + hostPath: + path: /var/run/k8s.cni.cncf.io/devinfo/dp + type: DirectoryOrCreate + - name: config-volume + configMap: + name: sriovdp-config + items: + - key: config.json + path: config.json diff --git a/pkg/state/factory.go b/pkg/state/factory.go index 7a82a0503..6e62306e4 100644 --- a/pkg/state/factory.go +++ b/pkg/state/factory.go @@ -59,6 +59,8 @@ func newStates(crdKind string, k8sAPIClient client.Client, scheme *runtime.Schem return newNicClusterPolicyStates(k8sAPIClient, scheme) case mellanoxv1alpha1.MacvlanNetworkCRDName: return newMacvlanNetworkStates(k8sAPIClient, scheme) + case mellanoxv1alpha1.HostDeviceNetworkCRDName: + return newHostDeviceNetworkStates(k8sAPIClient, scheme) default: break } @@ -76,6 +78,11 @@ func newNicClusterPolicyStates(k8sAPIClient client.Client, scheme *runtime.Schem sharedDpState, err := NewStateSharedDp( k8sAPIClient, scheme, filepath.Join(manifestBaseDir, "stage-rdma-device-plugin")) + if err != nil { + return nil, errors.Wrapf(err, "failed to create Shared Device plugin State") + } + sriovDpState, err := NewStateSriovDp( + k8sAPIClient, scheme, filepath.Join(manifestBaseDir, "stage-sriov-device-plugin")) if err != nil { return nil, errors.Wrapf(err, "failed to create Device plugin State") } @@ -102,6 +109,7 @@ func newNicClusterPolicyStates(k8sAPIClient client.Client, scheme *runtime.Schem return []Group{ NewStateGroup([]State{ofedState}), + NewStateGroup([]State{sriovDpState}), NewStateGroup([]State{sharedDpState, nvPeerMemState}), NewStateGroup([]State{multusState, cniPluginsState, whereaboutState}), }, nil @@ -120,3 +128,17 @@ func newMacvlanNetworkStates(k8sAPIClient client.Client, scheme *runtime.Scheme) NewStateGroup([]State{macvlanNetworkState}), }, nil } + +// newHostDeviceNetworkStates creates states that reconcile HostDeviceNetwork CRD +func newHostDeviceNetworkStates(k8sAPIClient client.Client, scheme *runtime.Scheme) ([]Group, error) { + manifestBaseDir := config.FromEnv().State.ManifestBaseDir + + hostdeviceNetworkState, err := NewStateHostDeviceNetwork( + k8sAPIClient, scheme, filepath.Join(manifestBaseDir, "stage-hostdevice-network")) + if err != nil { + return nil, errors.Wrapf(err, "failed to create HostDeviceNetwork CRD State") + } + return []Group{ + NewStateGroup([]State{hostdeviceNetworkState}), + }, nil +} diff --git a/pkg/state/state_hostdevice_network.go b/pkg/state/state_hostdevice_network.go new file mode 100644 index 000000000..e25d3039b --- /dev/null +++ b/pkg/state/state_hostdevice_network.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state //nolint:dupl + +import ( + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/source" + + mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1" + "github.com/Mellanox/network-operator/pkg/consts" + "github.com/Mellanox/network-operator/pkg/render" + "github.com/Mellanox/network-operator/pkg/utils" +) + +const ( + stateHostDeviceNetworkName = "state-host-device-network" + stateHostDeviceNetworkDescription = "Host Device net-attach-def CR deployed in cluster" +) + +// NewStateHostDeviceNetwork creates a new state for HostDeviceNetwork CR +func NewStateHostDeviceNetwork(k8sAPIClient client.Client, scheme *runtime.Scheme, manifestDir string) (State, error) { + files, err := utils.GetFilesWithSuffix(manifestDir, render.ManifestFileSuffix...) + if err != nil { + return nil, errors.Wrap(err, "failed to get files from manifest dir") + } + + renderer := render.NewRenderer(files) + return &stateHostDeviceNetwork{ + stateSkel: stateSkel{ + name: stateHostDeviceNetworkName, + description: stateHostDeviceNetworkDescription, + client: k8sAPIClient, + scheme: scheme, + renderer: renderer, + }}, nil +} + +type stateHostDeviceNetwork struct { + stateSkel +} + +type HostDeviceManifestRenderData struct { + HostDeviceNetworkName string + CrSpec mellanoxv1alpha1.HostDeviceNetworkSpec + RuntimeSpec *runtimeSpec +} + +// Sync attempt to get the system to match the desired state which State represent. +// a sync operation must be relatively short and must not block the execution thread. +func (s *stateHostDeviceNetwork) Sync(customResource interface{}, _ InfoCatalog) (SyncState, error) { + cr := customResource.(*mellanoxv1alpha1.HostDeviceNetwork) + log.V(consts.LogLevelInfo).Info( + "Sync Custom resource", "State:", s.name, "Name:", cr.Name, "Namespace:", cr.Namespace) + + objs, err := s.getManifestObjects(cr) + if err != nil { + return SyncStateError, errors.Wrap(err, "failed to render HostDeviceNetwork") + } + + if len(objs) == 0 { + return SyncStateError, errors.Wrap(err, "no rendered objects found") + } + + netAttDef := objs[0] + if netAttDef.GetKind() != "NetworkAttachmentDefinition" { + return SyncStateError, errors.Wrap(err, "no NetworkAttachmentDefinition object found") + } + + err = s.createOrUpdateObjs(func(obj *unstructured.Unstructured) error { + if err := controllerutil.SetControllerReference(cr, obj, s.scheme); err != nil { + return errors.Wrap(err, "failed to set controller reference for object") + } + return nil + }, objs) + + if err != nil { + return SyncStateNotReady, errors.Wrap(err, "failed to create/update objects") + } + + // Check objects status + syncState, err := s.getSyncState(objs) + if err != nil { + return SyncStateNotReady, errors.Wrap(err, "failed to get sync state") + } + + // Get NetworkAttachmentDefinition SelfLink + if err := s.getObj(netAttDef); err != nil { + return SyncStateError, errors.Wrap(err, "failed to get NetworkAttachmentDefinition") + } + // TODO handling HostDeviceNetwork CR Status should be done in the controller + cr.Status.HostDeviceNetworkAttachmentDef = netAttDef.GetSelfLink() + + return syncState, nil +} + +// Get a map of source kinds that should be watched for the state keyed by the source kind name +func (s *stateHostDeviceNetwork) GetWatchSources() map[string]*source.Kind { + wr := make(map[string]*source.Kind) + wr["HostDeviceNetwork"] = &source.Kind{Type: &mellanoxv1alpha1.HostDeviceNetwork{}} + wr["NetworkAttachmentDefinition"] = &source.Kind{Type: &netattdefv1.NetworkAttachmentDefinition{}} + return wr +} + +func (s *stateHostDeviceNetwork) getManifestObjects( + cr *mellanoxv1alpha1.HostDeviceNetwork) ([]*unstructured.Unstructured, error) { + renderData := &HostDeviceManifestRenderData{ + HostDeviceNetworkName: cr.Name, + CrSpec: cr.Spec, + RuntimeSpec: &runtimeSpec{ + Namespace: consts.NetworkOperatorResourceNamespace, + }, + } + + // render objects + log.V(consts.LogLevelDebug).Info("Rendering objects", "data:", renderData) + objs, err := s.renderer.RenderObjects(&render.TemplatingData{Data: renderData}) + if err != nil { + return nil, errors.Wrap(err, "failed to render objects") + } + log.V(consts.LogLevelDebug).Info("Rendered", "objects:", objs) + return objs, nil +} diff --git a/pkg/state/state_macvlan_network.go b/pkg/state/state_macvlan_network.go index d74ab212b..4a17aab1a 100644 --- a/pkg/state/state_macvlan_network.go +++ b/pkg/state/state_macvlan_network.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package state +package state //nolint:dupl import ( "context" diff --git a/pkg/state/state_multus_cni.go b/pkg/state/state_multus_cni.go index 99999de3e..7c76cb9c0 100644 --- a/pkg/state/state_multus_cni.go +++ b/pkg/state/state_multus_cni.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package state +package state //nolint:dupl import ( "github.com/pkg/errors" diff --git a/pkg/state/state_shared_dp.go b/pkg/state/state_shared_dp.go index bf6d768f1..947419aec 100644 --- a/pkg/state/state_shared_dp.go +++ b/pkg/state/state_shared_dp.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package state +package state //nolint:dupl import ( "github.com/pkg/errors" @@ -68,7 +68,6 @@ func (s *stateSharedDp) Sync(customResource interface{}, infoCatalog InfoCatalog if cr.Spec.RdmaSharedDevicePlugin == nil { // Either this state was not required to run or an update occurred and we need to remove // the resources that where created. - // TODO: Support the latter case log.V(consts.LogLevelInfo).Info("Device plugin spec in CR is nil, no action required") return SyncStateIgnore, nil } diff --git a/pkg/state/state_sriov_dp.go b/pkg/state/state_sriov_dp.go new file mode 100644 index 000000000..f095a238a --- /dev/null +++ b/pkg/state/state_sriov_dp.go @@ -0,0 +1,125 @@ +/* +Copyright 2020 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state //nolint:dupl + +import ( + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/source" + + mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1" + "github.com/Mellanox/network-operator/pkg/consts" + "github.com/Mellanox/network-operator/pkg/render" + "github.com/Mellanox/network-operator/pkg/utils" +) + +// NewStateSriovDp creates a new shared device plugin state +func NewStateSriovDp(k8sAPIClient client.Client, scheme *runtime.Scheme, manifestDir string) (State, error) { + files, err := utils.GetFilesWithSuffix(manifestDir, render.ManifestFileSuffix...) + if err != nil { + return nil, errors.Wrap(err, "failed to get files from manifest dir") + } + + renderer := render.NewRenderer(files) + return &stateSriovDp{ + stateSkel: stateSkel{ + name: "state-SRIOV-device-plugin", + description: "SR-IOV device plugin deployed in the cluster", + client: k8sAPIClient, + scheme: scheme, + renderer: renderer, + }}, nil +} + +type stateSriovDp struct { + stateSkel +} + +type sriovDpManifestRenderData struct { + CrSpec *mellanoxv1alpha1.DevicePluginSpec + RuntimeSpec *runtimeSpec +} + +// Sync attempt to get the system to match the desired state which State represent. +// a sync operation must be relatively short and must not block the execution thread. +func (s *stateSriovDp) Sync(customResource interface{}, infoCatalog InfoCatalog) (SyncState, error) { + cr := customResource.(*mellanoxv1alpha1.NicClusterPolicy) + log.V(consts.LogLevelInfo).Info( + "Sync Custom resource", "State:", s.name, "Name:", cr.Name, "Namespace:", cr.Namespace) + + if cr.Spec.SriovDevicePlugin == nil { + // Either this state was not required to run or an update occurred and we need to remove + // the resources that where created. + // TODO: Support the latter case + log.V(consts.LogLevelInfo).Info("Device plugin spec in CR is nil, no action required") + return SyncStateIgnore, nil + } + // Fill ManifestRenderData and render objects + objs, err := s.getManifestObjects(cr) + if err != nil { + return SyncStateNotReady, errors.Wrap(err, "failed to create k8s objects from manifest") + } + if len(objs) == 0 { + return SyncStateNotReady, nil + } + + // Create objects if they dont exist, Update objects if they do exist + err = s.createOrUpdateObjs(func(obj *unstructured.Unstructured) error { + if err := controllerutil.SetControllerReference(cr, obj, s.scheme); err != nil { + return errors.Wrap(err, "failed to set controller reference for object") + } + return nil + }, objs) + if err != nil { + return SyncStateNotReady, errors.Wrap(err, "failed to create/update objects") + } + // Check objects status + syncState, err := s.getSyncState(objs) + if err != nil { + return SyncStateNotReady, errors.Wrap(err, "failed to get sync state") + } + return syncState, nil +} + +// Get a map of source kinds that should be watched for the state keyed by the source kind name +func (s *stateSriovDp) GetWatchSources() map[string]*source.Kind { + wr := make(map[string]*source.Kind) + wr["DaemonSet"] = &source.Kind{Type: &appsv1.DaemonSet{}} + return wr +} + +func (s *stateSriovDp) getManifestObjects( + cr *mellanoxv1alpha1.NicClusterPolicy) ([]*unstructured.Unstructured, error) { + renderData := &sriovDpManifestRenderData{ + CrSpec: cr.Spec.SriovDevicePlugin, + RuntimeSpec: &runtimeSpec{ + Namespace: consts.NetworkOperatorResourceNamespace, + }, + } + // render objects + log.V(consts.LogLevelDebug).Info("Rendering objects", "data:", renderData) + objs, err := s.renderer.RenderObjects(&render.TemplatingData{Data: renderData}) + if err != nil { + return nil, errors.Wrap(err, "failed to render objects") + } + log.V(consts.LogLevelDebug).Info("Rendered", "objects:", objs) + return objs, nil +} diff --git a/pkg/state/state_whereabouts_cni.go b/pkg/state/state_whereabouts_cni.go index 914e81912..a4e59931c 100644 --- a/pkg/state/state_whereabouts_cni.go +++ b/pkg/state/state_whereabouts_cni.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package state +package state //nolint:dupl import ( "github.com/pkg/errors" @@ -53,13 +53,9 @@ type stateWhereaboutsCNI struct { stateSkel } -type WhereaboutsRuntimeSpec struct { - Namespace string -} - type WhereaboutsManifestRenderData struct { CrSpec *mellanoxv1alpha1.ImageSpec - RuntimeSpec *WhereaboutsRuntimeSpec + RuntimeSpec *runtimeSpec } // Sync attempt to get the system to match the desired state which State represent. @@ -115,7 +111,7 @@ func (s *stateWhereaboutsCNI) getManifestObjects( cr *mellanoxv1alpha1.NicClusterPolicy) ([]*unstructured.Unstructured, error) { renderData := &WhereaboutsManifestRenderData{ CrSpec: cr.Spec.SecondaryNetwork.IpamPlugin, - RuntimeSpec: &WhereaboutsRuntimeSpec{ + RuntimeSpec: &runtimeSpec{ Namespace: consts.NetworkOperatorResourceNamespace, }, } diff --git a/scripts/deploy-operator.sh b/scripts/deploy-operator.sh index fe7f9f868..d772f9540 100755 --- a/scripts/deploy-operator.sh +++ b/scripts/deploy-operator.sh @@ -19,6 +19,7 @@ echo "Deploying Network Operator:" echo "###########################" kubectl apply -f $repo_dir/deploy/operator-ns.yaml kubectl apply -f $repo_dir/deploy/operator-resources-ns.yaml +kubectl apply -f $repo_dir/config/crd/bases//mellanox.com_hostdevicenetworks.yaml kubectl apply -f $repo_dir/config/crd/bases/mellanox.com_nicclusterpolicies.yaml kubectl apply -f $repo_dir/config/crd/bases/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml kubectl apply -f $repo_dir/config/crd/bases/mellanox.com_macvlannetworks.yaml