test(scorecard): retry on cryostat update conflict (#774)

* retry on cryostat update conflict * review * review
cryostatio · Apr 16, 2024 · d7b0379 · d7b0379
1 parent 4b8ad76
commit d7b0379
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 43 deletions.
diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml
@@ -54,7 +54,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev
-    createdAt: "2024-04-05T22:43:31Z"
+    createdAt: "2024-04-16T20:46:24Z"
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {

diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml
@@ -70,7 +70,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: operator-install
@@ -80,7 +80,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -90,7 +90,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-multi-namespace
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: cryostat-multi-namespace
@@ -100,7 +100,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: cryostat-recording
@@ -110,7 +110,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-config-change
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: cryostat-config-change
@@ -120,7 +120,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-report
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604
     labels:
       suite: cryostat
       test: cryostat-report

diff --git a/config/scorecard/patches/custom.config.yaml b/config/scorecard/patches/custom.config.yaml
@@ -8,7 +8,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: operator-install
@@ -18,7 +18,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -28,7 +28,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-multi-namespace
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: cryostat-multi-namespace
@@ -38,7 +38,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: cryostat-recording
@@ -48,7 +48,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-config-change
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: cryostat-config-change
@@ -58,7 +58,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-report
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240405224154"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240416204604"
     labels:
       suite: cryostat
       test: cryostat-report
diff --git a/internal/test/scorecard/common_utils.go b/internal/test/scorecard/common_utils.go
@@ -31,11 +31,13 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/retry"
 )
 
 const (
@@ -511,11 +513,34 @@ func (r *TestResources) sendHealthRequest(base *url.URL, healthCheck func(resp *
 	return err
 }
 
-func (r *TestResources) updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat) error {
-	cr, err := r.Client.OperatorCRDs().Cryostats(cr.Namespace).Update(context.Background(), cr)
-	if err != nil {
-		r.Log += fmt.Sprintf("failed to update Cryostat CR: %s", err.Error())
+func (r *TestResources) updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat) (*operatorv1beta1.Cryostat, error) {
+	ctx := context.Background()
+
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		var err error
+		cr, err = r.Client.OperatorCRDs().Cryostats(cr.Namespace).Get(ctx, cr.Name)
+		if err != nil {
+			return fmt.Errorf("failed to get Cryostat CR: %s", err.Error())
+		}
+
+		cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{
+			PVC: &operatorv1beta1.PersistentVolumeClaimConfig{
+				Spec: &corev1.PersistentVolumeClaimSpec{
+					StorageClassName: nil,
+					Resources: corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceStorage: resource.MustParse("1Gi"),
+						},
+					},
+				},
+			},
+		}
+
+		cr, err = r.Client.OperatorCRDs().Cryostats(cr.Namespace).Update(context.Background(), cr)
 		return err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to update Cryostat CR: %s", err.Error())
 	}
 
 	// Poll the deployment until it becomes available or we timeout
@@ -569,9 +594,9 @@ func (r *TestResources) updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.C
 		return false, nil
 	})
 	if err != nil {
-		return fmt.Errorf("failed to look up deployment errors: %s", err.Error())
+		return nil, fmt.Errorf("failed to look up deployment errors: %s", err.Error())
 	}
-	return err
+	return cr, err
 }
 
 func (r *TestResources) cleanupAndLogs(name string, namespace string) {

diff --git a/internal/test/scorecard/tests.go b/internal/test/scorecard/tests.go
@@ -23,8 +23,6 @@ import (
 	operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1"
 	scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
 	apimanifests "github.com/operator-framework/api/pkg/manifests"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -128,34 +126,14 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope
 	}
 
 	// Switch Cryostat CR to PVC for redeployment
-	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
-	defer cancel()
-
-	cr, err = r.Client.OperatorCRDs().Cryostats(namespace).Get(ctx, CryostatConfigChangeTestName)
-	if err != nil {
-		return r.fail(fmt.Sprintf("failed to get Cryostat CR: %s", err.Error()))
-	}
-	cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{
-		PVC: &operatorv1beta1.PersistentVolumeClaimConfig{
-			Spec: &corev1.PersistentVolumeClaimSpec{
-				StorageClassName: nil,
-				Resources: corev1.ResourceRequirements{
-					Requests: corev1.ResourceList{
-						corev1.ResourceStorage: resource.MustParse("1Gi"),
-					},
-				},
-			},
-		},
-	}
-
-	// Wait for redeployment of Cryostat CR
-	err = r.updateAndWaitTillCryostatAvailable(cr)
+	cr, err = r.updateAndWaitTillCryostatAvailable(cr)
 	if err != nil {
 		return r.fail(fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error()))
 	}
 	r.Log += "Cryostat deployment has successfully updated with new spec template\n"
 
 	base, err := url.Parse(cr.Status.ApplicationURL)
+	r.Log += fmt.Sprintf("base url: %s\n", base)
 	if err != nil {
 		return r.fail(fmt.Sprintf("application URL is invalid: %s", err.Error()))
 	}