From 6fdefc1205dfadc8d9241edc42564342b33aa15c Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 30 Mar 2021 15:08:32 +0300 Subject: [PATCH 1/2] Autoselect right path of OFED src for nv-peer-mem POD Based on the fact that Mellanox OFED driver POD is deployed or not, network-operator will select the right OFED src path for nv-peer-mem POD. If OFED driver POD doesn't exist, OFED from the host will be used. Signed-off-by: Yury Kulazhenkov --- .../0010_nv-peer-mem-driver-ds.yaml | 6 +++++- pkg/state/state_nv_peer.go | 14 ++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml b/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml index 2ea45d69..bff3ba61 100644 --- a/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml +++ b/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml @@ -108,9 +108,13 @@ spec: initialDelaySeconds: 10 failureThreshold: 1 volumes: + {{ $ofedPath := "/run/mellanox/drivers" }} + {{ if .RuntimeSpec.UseHostOFED }} + {{ $ofedPath = "/" }} + {{end}} - name: run-mlnx-ofed hostPath: - path: /run/mellanox/drivers + path: {{ $ofedPath }} - name: run-nvidia hostPath: path: {{ .CrSpec.GPUDriverSourcePath }} diff --git a/pkg/state/state_nv_peer.go b/pkg/state/state_nv_peer.go index be37eb74..41c5f1ca 100644 --- a/pkg/state/state_nv_peer.go +++ b/pkg/state/state_nv_peer.go @@ -47,12 +47,13 @@ type stateNVPeer struct { type nvPeerRuntimeSpec struct { runtimeSpec - CPUArch string - OSName string - OSVer string - HTTPProxy string - HTTPSProxy string - NoProxy string + CPUArch string + OSName string + OSVer string + HTTPProxy string + HTTPSProxy string + NoProxy string + UseHostOFED bool } type nvPeerManifestRenderData struct { @@ -146,6 +147,7 @@ func (s *stateNVPeer) getManifestObjects( HTTPProxy: os.Getenv(consts.HTTPProxy), HTTPSProxy: os.Getenv(consts.HTTPSProxy), NoProxy: os.Getenv(consts.NoProxy), + UseHostOFED: cr.Spec.OFEDDriver == nil, }, } // render objects From 4a55f2bbf68f5c71e374764bac27bf396553cb9e Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 30 Mar 2021 16:14:26 +0300 Subject: [PATCH 2/2] Use more specific OFED mounts in nv-peer-mem POD Replace full rootfs mount in nv-peer-mem POD with more specific mounts for OFED modules. Signed-off-by: Yury Kulazhenkov --- .../0010_nv-peer-mem-driver-ds.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml b/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml index bff3ba61..92c2fe9d 100644 --- a/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml +++ b/manifests/stage-nv-peer-mem-driver/0010_nv-peer-mem-driver-ds.yaml @@ -78,8 +78,11 @@ spec: - name: NO_PROXY value: {{ .RuntimeSpec.NoProxy }} volumeMounts: - - name: run-mlnx-ofed - mountPath: /run/mellanox/drivers + - name: mlnx-ofed-usr-src + mountPath: /run/mellanox/drivers/usr/src + mountPropagation: HostToContainer + - name: mlnx-ofed-lib-modules + mountPath: /run/mellanox/drivers/lib/modules mountPropagation: HostToContainer - name: run-nvidia mountPath: /run/nvidia/drivers @@ -110,11 +113,14 @@ spec: volumes: {{ $ofedPath := "/run/mellanox/drivers" }} {{ if .RuntimeSpec.UseHostOFED }} - {{ $ofedPath = "/" }} + {{ $ofedPath = "" }} {{end}} - - name: run-mlnx-ofed + - name: mlnx-ofed-usr-src + hostPath: + path: {{ $ofedPath }}/usr/src + - name: mlnx-ofed-lib-modules hostPath: - path: {{ $ofedPath }} + path: {{ $ofedPath }}/lib/modules - name: run-nvidia hostPath: path: {{ .CrSpec.GPUDriverSourcePath }}