Skip to content

Commit

Permalink
Fix sriov functional tests for secure boot env
Browse files Browse the repository at this point in the history 
This commit fix the functional tests when running on a secure boot environment

Signed-off-by: Sebastian Sch <sebassch@gmail.com>
  • Loading branch information
@SchSeba
SchSeba committed Jun 21, 2021
1 parent fc360d9 commit 555288a
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 14 deletions.
93 changes: 85 additions & 8 deletions test/util/cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,45 @@ import (
"fmt"
"io"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces"
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/nodes"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod"
)

// EnabledNodes provides info on sriov enabled nodes of the cluster.
type EnabledNodes struct {
Nodes []string
States map[string]sriovv1.SriovNetworkNodeState
Nodes []string
States map[string]sriovv1.SriovNetworkNodeState
IsSecureBootEnabled map[string]bool
}

var (
supportedPFDrivers = []string{"mlx5_core", "i40e", "ixgbe", "ice"}
supportedVFDrivers = []string{"iavf", "vfio-pci", "mlx5_core"}
mlxVendorID = "15b3"
intelVendorID = "8086"
)

// DiscoverSriov retrieves Sriov related information of a given cluster.
func DiscoverSriov(clients *testclient.ClientSet, operatorNamespace string) (*EnabledNodes, error) {
nodeStates, err := clients.SriovNetworkNodeStates(operatorNamespace).List(context.Background(), metav1.ListOptions{})
res := &EnabledNodes{}
res.States = make(map[string]sriovv1.SriovNetworkNodeState)
res.Nodes = make([]string, 0)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve note states %v", err)
}

res := &EnabledNodes{}
res.States = make(map[string]sriovv1.SriovNetworkNodeState)
res.Nodes = make([]string, 0)
res.IsSecureBootEnabled = make(map[string]bool)

ss, err := nodes.MatchingOptionalSelectorState(clients, nodeStates.Items)
if err != nil {
return nil, fmt.Errorf("Failed to find matching node states %v", err)
Expand All @@ -59,6 +68,15 @@ func DiscoverSriov(clients *testclient.ClientSet, operatorNamespace string) (*En
}
}

for _, node := range res.Nodes {
isSecureBootEnabled, err := GetNodeSecureBootState(clients, node)
if err != nil {
return nil, err
}

res.IsSecureBootEnabled[node] = isSecureBootEnabled
}

if len(res.Nodes) == 0 {
return nil, fmt.Errorf("No sriov enabled node found")
}
Expand All @@ -73,6 +91,13 @@ func (n *EnabledNodes) FindOneSriovDevice(node string) (*sriovv1.InterfaceExt, e
}
for _, itf := range s.Status.Interfaces {
if IsPFDriverSupported(itf.Driver) && sriovv1.IsSupportedDevice(itf.DeviceID) {

// Skip mlx interfaces if secure boot is enabled
// TODO: remove this when mlx support secure boot/lockdown mode
if itf.Vendor == mlxVendorID && n.IsSecureBootEnabled[node] {
continue
}

return &itf, nil
}
}
Expand All @@ -89,6 +114,12 @@ func (n *EnabledNodes) FindSriovDevices(node string) ([]*sriovv1.InterfaceExt, e

for i, itf := range s.Status.Interfaces {
if IsPFDriverSupported(itf.Driver) && sriovv1.IsSupportedDevice(itf.DeviceID) {
// Skip mlx interfaces if secure boot is enabled
// TODO: remove this when mlx support secure boot/lockdown mode
if itf.Vendor == mlxVendorID && n.IsSecureBootEnabled[node] {
continue
}

devices = append(devices, &s.Status.Interfaces[i])
}
}
Expand All @@ -99,7 +130,7 @@ func (n *EnabledNodes) FindSriovDevices(node string) ([]*sriovv1.InterfaceExt, e
func (n *EnabledNodes) FindOneVfioSriovDevice() (string, sriovv1.InterfaceExt) {
for _, node := range n.Nodes {
for _, nic := range n.States[node].Status.Interfaces {
if nic.Vendor == "8086" {
if nic.Vendor == intelVendorID {
return node, nic
}
}
Expand All @@ -113,8 +144,15 @@ func (n *EnabledNodes) FindOneMellanoxSriovDevice(node string) (*sriovv1.Interfa
if !ok {
return nil, fmt.Errorf("Node %s not found", node)
}

// return error here as mlx interfaces are not supported when secure boot is enabled
// TODO: remove this when mlx support secure boot/lockdown mode
if n.IsSecureBootEnabled[node] {
return nil, fmt.Errorf("Secure boot is enabled on the node mellanox cards are not supported")
}

for _, itf := range s.Status.Interfaces {
if itf.Driver == "mlx5_core" {
if itf.Vendor == mlxVendorID {
return &itf, nil
}
}
Expand Down Expand Up @@ -223,3 +261,42 @@ func SetDisableNodeDrainState(clients *testclient.ClientSet, operatorNamespace s
}
return nil
}

func GetNodeSecureBootState(clients *testclient.ClientSet, nodeName string) (bool, error) {
podDefinition := pod.GetDefinition()
podDefinition = pod.RedefineWithNodeSelector(podDefinition, nodeName)
podDefinition = pod.RedefineAsPrivileged(podDefinition)

volume := corev1.Volume{Name: "host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/"}}}
mount := corev1.VolumeMount{Name: "host", MountPath: "/host"}
podDefinition = pod.RedefineWithMount(podDefinition, volume, mount)
created, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{})
if err != nil {
return false, err
}

var runningPod *corev1.Pod
for retry := 0; retry < 180; retry++ {
runningPod, err = clients.Pods(namespaces.Test).Get(context.Background(), created.Name, metav1.GetOptions{})
if err != nil {
return false, err
}

if runningPod.Status.Phase == corev1.PodRunning {
break
}

time.Sleep(time.Second)
}

stdout, stderr, err := pod.ExecCommand(clients, runningPod, "cat", "/host/sys/kernel/security/lockdown")
if err != nil {
return false, err
}

if stderr != "" {
return false, fmt.Errorf("command return non 0 code: %s", stderr)
}

return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]"), nil
}
20 changes: 14 additions & 6 deletions test/util/pod/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

const hostnameLabel = "kubernetes.io/hostname"

func getDefinition() *corev1.Pod {
func GetDefinition() *corev1.Pod {
podObject := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "testpod-",
Expand All @@ -36,14 +36,14 @@ func getDefinition() *corev1.Pod {
}

func DefineWithNetworks(networks []string) *corev1.Pod {
podObject := getDefinition()
podObject := GetDefinition()
podObject.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": strings.Join(networks, ",")}

return podObject
}

func DefineWithHostNetwork(nodeName string) *corev1.Pod {
podObject := getDefinition()
podObject := GetDefinition()
podObject.Spec.HostNetwork = true
podObject.Spec.NodeSelector = map[string]string{
"kubernetes.io/hostname": nodeName,
Expand All @@ -52,28 +52,36 @@ func DefineWithHostNetwork(nodeName string) *corev1.Pod {
return podObject
}

// RedefineAsPrivileged uppdates the pod to be privileged
// RedefineAsPrivileged updates the pod to be privileged
func RedefineAsPrivileged(pod *corev1.Pod) *corev1.Pod {
pod.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{}
b := true
pod.Spec.Containers[0].SecurityContext.Privileged = &b
return pod
}

// RedefineWithHostNetwork uppdates the pod definition Spec.HostNetwork to true
// RedefineWithHostNetwork updates the pod definition Spec.HostNetwork to true
func RedefineWithHostNetwork(pod *corev1.Pod) *corev1.Pod {
pod.Spec.HostNetwork = true
return pod
}

// RedefineWithNodeSelector uppdates the pod definition with a node selector
// RedefineWithNodeSelector updates the pod definition with a node selector
func RedefineWithNodeSelector(pod *corev1.Pod, node string) *corev1.Pod {
pod.Spec.NodeSelector = map[string]string{
hostnameLabel: node,
}
return pod
}

// RedefineWithMount updates the pod definition with a volume and volume mount
func RedefineWithMount(pod *corev1.Pod, volume corev1.Volume, mount corev1.VolumeMount) *corev1.Pod {
pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{mount}
pod.Spec.Volumes = []corev1.Volume{volume}

return pod
}

// RedefineWithCommand updates the pod defintion with a different command
func RedefineWithCommand(pod *corev1.Pod, command []string, args []string) *corev1.Pod {
pod.Spec.Containers[0].Command = command
Expand Down

0 comments on commit 555288a

Please sign in to comment.