From 555288a16aa2c6c78018d0c4b930cd9bdb1a69c8 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Mon, 21 Jun 2021 11:25:35 +0300 Subject: [PATCH] Fix sriov functional tests for secure boot env This commit fix the functional tests when running on a secure boot environment Signed-off-by: Sebastian Sch --- test/util/cluster/cluster.go | 93 ++++++++++++++++++++++++++++++++---- test/util/pod/pod.go | 20 +++++--- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/test/util/cluster/cluster.go b/test/util/cluster/cluster.go index fa32f1d356..e8fcb9e253 100644 --- a/test/util/cluster/cluster.go +++ b/test/util/cluster/cluster.go @@ -5,36 +5,45 @@ import ( "fmt" "io" "strings" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/nodes" - runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod" ) // EnabledNodes provides info on sriov enabled nodes of the cluster. type EnabledNodes struct { - Nodes []string - States map[string]sriovv1.SriovNetworkNodeState + Nodes []string + States map[string]sriovv1.SriovNetworkNodeState + IsSecureBootEnabled map[string]bool } var ( supportedPFDrivers = []string{"mlx5_core", "i40e", "ixgbe", "ice"} supportedVFDrivers = []string{"iavf", "vfio-pci", "mlx5_core"} + mlxVendorID = "15b3" + intelVendorID = "8086" ) // DiscoverSriov retrieves Sriov related information of a given cluster. func DiscoverSriov(clients *testclient.ClientSet, operatorNamespace string) (*EnabledNodes, error) { nodeStates, err := clients.SriovNetworkNodeStates(operatorNamespace).List(context.Background(), metav1.ListOptions{}) - res := &EnabledNodes{} - res.States = make(map[string]sriovv1.SriovNetworkNodeState) - res.Nodes = make([]string, 0) if err != nil { return nil, fmt.Errorf("Failed to retrieve note states %v", err) } + res := &EnabledNodes{} + res.States = make(map[string]sriovv1.SriovNetworkNodeState) + res.Nodes = make([]string, 0) + res.IsSecureBootEnabled = make(map[string]bool) + ss, err := nodes.MatchingOptionalSelectorState(clients, nodeStates.Items) if err != nil { return nil, fmt.Errorf("Failed to find matching node states %v", err) @@ -59,6 +68,15 @@ func DiscoverSriov(clients *testclient.ClientSet, operatorNamespace string) (*En } } + for _, node := range res.Nodes { + isSecureBootEnabled, err := GetNodeSecureBootState(clients, node) + if err != nil { + return nil, err + } + + res.IsSecureBootEnabled[node] = isSecureBootEnabled + } + if len(res.Nodes) == 0 { return nil, fmt.Errorf("No sriov enabled node found") } @@ -73,6 +91,13 @@ func (n *EnabledNodes) FindOneSriovDevice(node string) (*sriovv1.InterfaceExt, e } for _, itf := range s.Status.Interfaces { if IsPFDriverSupported(itf.Driver) && sriovv1.IsSupportedDevice(itf.DeviceID) { + + // Skip mlx interfaces if secure boot is enabled + // TODO: remove this when mlx support secure boot/lockdown mode + if itf.Vendor == mlxVendorID && n.IsSecureBootEnabled[node] { + continue + } + return &itf, nil } } @@ -89,6 +114,12 @@ func (n *EnabledNodes) FindSriovDevices(node string) ([]*sriovv1.InterfaceExt, e for i, itf := range s.Status.Interfaces { if IsPFDriverSupported(itf.Driver) && sriovv1.IsSupportedDevice(itf.DeviceID) { + // Skip mlx interfaces if secure boot is enabled + // TODO: remove this when mlx support secure boot/lockdown mode + if itf.Vendor == mlxVendorID && n.IsSecureBootEnabled[node] { + continue + } + devices = append(devices, &s.Status.Interfaces[i]) } } @@ -99,7 +130,7 @@ func (n *EnabledNodes) FindSriovDevices(node string) ([]*sriovv1.InterfaceExt, e func (n *EnabledNodes) FindOneVfioSriovDevice() (string, sriovv1.InterfaceExt) { for _, node := range n.Nodes { for _, nic := range n.States[node].Status.Interfaces { - if nic.Vendor == "8086" { + if nic.Vendor == intelVendorID { return node, nic } } @@ -113,8 +144,15 @@ func (n *EnabledNodes) FindOneMellanoxSriovDevice(node string) (*sriovv1.Interfa if !ok { return nil, fmt.Errorf("Node %s not found", node) } + + // return error here as mlx interfaces are not supported when secure boot is enabled + // TODO: remove this when mlx support secure boot/lockdown mode + if n.IsSecureBootEnabled[node] { + return nil, fmt.Errorf("Secure boot is enabled on the node mellanox cards are not supported") + } + for _, itf := range s.Status.Interfaces { - if itf.Driver == "mlx5_core" { + if itf.Vendor == mlxVendorID { return &itf, nil } } @@ -223,3 +261,42 @@ func SetDisableNodeDrainState(clients *testclient.ClientSet, operatorNamespace s } return nil } + +func GetNodeSecureBootState(clients *testclient.ClientSet, nodeName string) (bool, error) { + podDefinition := pod.GetDefinition() + podDefinition = pod.RedefineWithNodeSelector(podDefinition, nodeName) + podDefinition = pod.RedefineAsPrivileged(podDefinition) + + volume := corev1.Volume{Name: "host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/"}}} + mount := corev1.VolumeMount{Name: "host", MountPath: "/host"} + podDefinition = pod.RedefineWithMount(podDefinition, volume, mount) + created, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + var runningPod *corev1.Pod + for retry := 0; retry < 180; retry++ { + runningPod, err = clients.Pods(namespaces.Test).Get(context.Background(), created.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + if runningPod.Status.Phase == corev1.PodRunning { + break + } + + time.Sleep(time.Second) + } + + stdout, stderr, err := pod.ExecCommand(clients, runningPod, "cat", "/host/sys/kernel/security/lockdown") + if err != nil { + return false, err + } + + if stderr != "" { + return false, fmt.Errorf("command return non 0 code: %s", stderr) + } + + return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]"), nil +} diff --git a/test/util/pod/pod.go b/test/util/pod/pod.go index de7265f48c..256f02d7ce 100644 --- a/test/util/pod/pod.go +++ b/test/util/pod/pod.go @@ -21,7 +21,7 @@ import ( const hostnameLabel = "kubernetes.io/hostname" -func getDefinition() *corev1.Pod { +func GetDefinition() *corev1.Pod { podObject := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "testpod-", @@ -36,14 +36,14 @@ func getDefinition() *corev1.Pod { } func DefineWithNetworks(networks []string) *corev1.Pod { - podObject := getDefinition() + podObject := GetDefinition() podObject.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": strings.Join(networks, ",")} return podObject } func DefineWithHostNetwork(nodeName string) *corev1.Pod { - podObject := getDefinition() + podObject := GetDefinition() podObject.Spec.HostNetwork = true podObject.Spec.NodeSelector = map[string]string{ "kubernetes.io/hostname": nodeName, @@ -52,7 +52,7 @@ func DefineWithHostNetwork(nodeName string) *corev1.Pod { return podObject } -// RedefineAsPrivileged uppdates the pod to be privileged +// RedefineAsPrivileged updates the pod to be privileged func RedefineAsPrivileged(pod *corev1.Pod) *corev1.Pod { pod.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{} b := true @@ -60,13 +60,13 @@ func RedefineAsPrivileged(pod *corev1.Pod) *corev1.Pod { return pod } -// RedefineWithHostNetwork uppdates the pod definition Spec.HostNetwork to true +// RedefineWithHostNetwork updates the pod definition Spec.HostNetwork to true func RedefineWithHostNetwork(pod *corev1.Pod) *corev1.Pod { pod.Spec.HostNetwork = true return pod } -// RedefineWithNodeSelector uppdates the pod definition with a node selector +// RedefineWithNodeSelector updates the pod definition with a node selector func RedefineWithNodeSelector(pod *corev1.Pod, node string) *corev1.Pod { pod.Spec.NodeSelector = map[string]string{ hostnameLabel: node, @@ -74,6 +74,14 @@ func RedefineWithNodeSelector(pod *corev1.Pod, node string) *corev1.Pod { return pod } +// RedefineWithMount updates the pod definition with a volume and volume mount +func RedefineWithMount(pod *corev1.Pod, volume corev1.Volume, mount corev1.VolumeMount) *corev1.Pod { + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{mount} + pod.Spec.Volumes = []corev1.Volume{volume} + + return pod +} + // RedefineWithCommand updates the pod defintion with a different command func RedefineWithCommand(pod *corev1.Pod, command []string, args []string) *corev1.Pod { pod.Spec.Containers[0].Command = command