test(scorecard): log workload conditions/events upon failure

Makefile

@@ -138,7 +138,7 @@ ifneq ($(SKIP_TESTS), true)
 	operator-sdk run bundle -n $(SCORECARD_NAMESPACE) $(BUNDLE_IMG)
 	$(call scorecard-cleanup); \
 	trap cleanup EXIT; \
-	operator-sdk scorecard -n $(SCORECARD_NAMESPACE) -s cryostat-scorecard -w 10m $(BUNDLE_IMG) --pod-security=restricted
+	operator-sdk scorecard -n $(SCORECARD_NAMESPACE) -s cryostat-scorecard -w 20m $(BUNDLE_IMG) --pod-security=restricted
 endif
 
 .PHONY: clean-scorecard

bundle/manifests/cryostat-operator.clusterserviceversion.yaml

@@ -54,7 +54,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:2.4.0-dev
-    createdAt: "2023-04-25T14:37:03Z"
+    createdAt: "2023-04-26T20:43:23Z"
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {

bundle/tests/scorecard/config.yaml

@@ -69,7 +69,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230425143552
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230426204317
     labels:
       suite: cryostat
       test: operator-install
@@ -79,7 +79,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230425143552
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230426204317
     labels:
       suite: cryostat
       test: cryostat-cr

config/scorecard/patches/custom.config.yaml

@@ -8,7 +8,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230425143552"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230426204317"
     labels:
       suite: cryostat
       test: operator-install
@@ -18,7 +18,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230425143552"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.4.0-20230426204317"
     labels:
       suite: cryostat
       test: cryostat-cr

internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml

@@ -41,12 +41,22 @@ metadata:
   name: cryostat-scorecard
   namespace: placeholder
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - events
+  verbs:
+  - get
+  - list
 - apiGroups:
   - apps
   resources:
   - deployments
+  - replicasets
   verbs:
   - get
+  - list
 - apiGroups:
   - operator.cryostat.io
   resources:

internal/test/scorecard/tests.go

@@ -49,7 +49,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes/scheme"
 )
 
 const (
@@ -152,7 +154,7 @@ func CryostatCRTest(bundle *apimanifests.Bundle, namespace string, openShiftCert
 
 func waitForDeploymentAvailability(ctx context.Context, client *CryostatClientset, namespace string,
 	name string, r *scapiv1alpha3.TestResult) error {
-	return wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) {
+	err := wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) {
 		deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
 		if err != nil {
 			if kerrors.IsNotFound(err) {
@@ -168,10 +170,22 @@ func waitForDeploymentAvailability(ctx context.Context, client *CryostatClientse
 				r.Log += fmt.Sprintf("deployment %s is available\n", deploy.Name)
 				return true, nil
 			}
+			if condition.Type == appsv1.DeploymentReplicaFailure &&
+				condition.Status == corev1.ConditionTrue {
+				r.Log += fmt.Sprintf("deployment %s is failing, %s: %s\n", deploy.Name,
+					condition.Reason, condition.Message)
+			}
 		}
 		r.Log += fmt.Sprintf("deployment %s is not yet available\n", deploy.Name)
 		return false, nil
 	})
+	if err != nil {
+		logErr := logErrors(r, client, namespace, name)
+		if logErr != nil {
+			r.Log += fmt.Sprintf("failed to look up deployment errors: %s\n", logErr.Error())
+		}
+	}
+	return err
 }
 
 func fail(r scapiv1alpha3.TestResult, message string) scapiv1alpha3.TestResult {
@@ -194,3 +208,83 @@ func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, nam
 		r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error())
 	}
 }
+
+func logErrors(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string, name string) error {
+	ctx := context.Background()
+	deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	// Log deployment conditions and events
+	r.Log += fmt.Sprintf("deployment %s conditions:\n", deploy.Name)
+	for _, condition := range deploy.Status.Conditions {
+		r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type,
+			condition.Status, condition.Reason, condition.Message)
+	}
+
+	r.Log += fmt.Sprintf("deployment %s warning events:\n", deploy.Name)
+	err = logEvents(r, client, namespace, scheme.Scheme, deploy)
+	if err != nil {
+		return err
+	}
+
+	// Look up replica sets for deployment and log conditions and events
+	selector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector)
+	if err != nil {
+		return err
+	}
+	replicaSets, err := client.AppsV1().ReplicaSets(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: selector.String(),
+	})
+	if err != nil {
+		return err
+	}
+	for _, rs := range replicaSets.Items {
+		r.Log += fmt.Sprintf("replica set %s conditions:\n", rs.Name)
+		for _, condition := range rs.Status.Conditions {
+			r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status,
+				condition.Reason, condition.Message)
+		}
+		r.Log += fmt.Sprintf("replica set %s warning events:\n", rs.Name)
+		err = logEvents(r, client, namespace, scheme.Scheme, &rs)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Look up pods for deployment and log conditions and events
+	pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: selector.String(),
+	})
+	if err != nil {
+		return err
+	}
+	for _, pod := range pods.Items {
+		r.Log += fmt.Sprintf("pod %s phase: %s\n", pod.Name, pod.Status.Phase)
+		r.Log += fmt.Sprintf("pod %s conditions:\n", pod.Name)
+		for _, condition := range pod.Status.Conditions {
+			r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status,
+				condition.Reason, condition.Message)
+		}
+		r.Log += fmt.Sprintf("pod %s warning events:\n", pod.Name)
+		err = logEvents(r, client, namespace, scheme.Scheme, &pod)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func logEvents(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string,
+	scheme *runtime.Scheme, obj runtime.Object) error {
+	events, err := client.CoreV1().Events(namespace).Search(scheme, obj)
+	if err != nil {
+		return err
+	}
+	for _, event := range events.Items {
+		if event.Type == corev1.EventTypeWarning {
+			r.Log += fmt.Sprintf("\t%s: %s\n", event.Reason, event.Message)
+		}
+	}
+	return nil
+}