From 8f807ef91fdc0098a905ed1b8f452c76941b2a2b Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:51:51 -0400 Subject: [PATCH] test(scorecard): scorecard test for report generator (backport #753) (#767) * test(scorecard): scorecard test for report generator (#753) * deploy reports sidecar * report scorecard test * update * rebase fix * query health (cherry picked from commit 96ea4cb7ae2c219eb37cf5583a2607c31d7889a7) # Conflicts: # bundle/manifests/cryostat-operator.clusterserviceversion.yaml # bundle/tests/scorecard/config.yaml * Fix conflicts --------- Co-authored-by: Ming Yu Wang <90855268+mwangggg@users.noreply.github.com> Co-authored-by: Elliott Baron --- ...yostat-operator.clusterserviceversion.yaml | 2 +- bundle/tests/scorecard/config.yaml | 18 +++-- config/scorecard/patches/custom.config.yaml | 18 +++-- hack/custom.config.yaml.in | 10 +++ .../images/custom-scorecard-tests/main.go | 4 ++ internal/test/scorecard/clients.go | 2 +- internal/test/scorecard/common_utils.go | 65 ++++++++++++++----- internal/test/scorecard/tests.go | 41 +++++++++++- 8 files changed, 132 insertions(+), 28 deletions(-) diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index 9c732145..a7af2fe5 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -54,7 +54,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev - createdAt: "2024-03-12T21:00:17Z" + createdAt: "2024-03-15T21:38:16Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index 7b8e524f..8822c74a 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -70,7 +70,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - operator-install - image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322 + image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212 labels: suite: cryostat test: operator-install @@ -80,7 +80,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-cr - image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322 + image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212 labels: suite: cryostat test: cryostat-cr @@ -90,7 +90,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-recording - image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322 + image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212 labels: suite: cryostat test: cryostat-recording @@ -100,13 +100,23 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-config-change - image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322 + image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212 labels: suite: cryostat test: cryostat-config-change storage: spec: mountPath: {} + - entrypoint: + - cryostat-scorecard-tests + - cryostat-report + image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212 + labels: + suite: cryostat + test: cryostat-report + storage: + spec: + mountPath: {} storage: spec: mountPath: {} diff --git a/config/scorecard/patches/custom.config.yaml b/config/scorecard/patches/custom.config.yaml index 59b597ef..f6e806ec 100644 --- a/config/scorecard/patches/custom.config.yaml +++ b/config/scorecard/patches/custom.config.yaml @@ -8,7 +8,7 @@ entrypoint: - cryostat-scorecard-tests - operator-install - image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322" + image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212" labels: suite: cryostat test: operator-install @@ -18,7 +18,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-cr - image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322" + image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212" labels: suite: cryostat test: cryostat-cr @@ -28,7 +28,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-recording - image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322" + image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212" labels: suite: cryostat test: cryostat-recording @@ -38,7 +38,17 @@ entrypoint: - cryostat-scorecard-tests - cryostat-config-change - image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322" + image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212" labels: suite: cryostat test: cryostat-config-change +- op: add + path: /stages/1/tests/- + value: + entrypoint: + - cryostat-scorecard-tests + - cryostat-report + image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212" + labels: + suite: cryostat + test: cryostat-report diff --git a/hack/custom.config.yaml.in b/hack/custom.config.yaml.in index b707766a..4336abbe 100644 --- a/hack/custom.config.yaml.in +++ b/hack/custom.config.yaml.in @@ -41,3 +41,13 @@ labels: suite: cryostat test: cryostat-config-change +- op: add + path: /stages/1/tests/- + value: + entrypoint: + - cryostat-scorecard-tests + - cryostat-report + image: "${CUSTOM_SCORECARD_IMG}" + labels: + suite: cryostat + test: cryostat-report diff --git a/internal/images/custom-scorecard-tests/main.go b/internal/images/custom-scorecard-tests/main.go index 6faed656..b3041281 100644 --- a/internal/images/custom-scorecard-tests/main.go +++ b/internal/images/custom-scorecard-tests/main.go @@ -81,6 +81,7 @@ func printValidTests() []scapiv1alpha3.TestResult { tests.CryostatCRTestName, tests.CryostatRecordingTestName, tests.CryostatConfigChangeTestName, + tests.CryostatReportTestName, }, ",")) result.Errors = append(result.Errors, str) @@ -94,6 +95,7 @@ func validateTests(testNames []string) bool { case tests.CryostatCRTestName: case tests.CryostatRecordingTestName: case tests.CryostatConfigChangeTestName: + case tests.CryostatReportTestName: default: return false } @@ -116,6 +118,8 @@ func runTests(testNames []string, bundle *apimanifests.Bundle, namespace string, results = append(results, tests.CryostatRecordingTest(bundle, namespace, openShiftCertManager)) case tests.CryostatConfigChangeTestName: results = append(results, tests.CryostatConfigChangeTest(bundle, namespace, openShiftCertManager)) + case tests.CryostatReportTestName: + results = append(results, tests.CryostatReportTest(bundle, namespace, openShiftCertManager)) default: log.Fatalf("unknown test found: %s", testName) } diff --git a/internal/test/scorecard/clients.go b/internal/test/scorecard/clients.go index 8f78d7a0..2c831e72 100644 --- a/internal/test/scorecard/clients.go +++ b/internal/test/scorecard/clients.go @@ -587,7 +587,7 @@ func SendRequest(ctx context.Context, httpClient *http.Client, method string, ur // Create a new request req, err := NewHttpRequest(ctx, method, url, body, header) if err != nil { - return false, fmt.Errorf("failed to create a Cryostat REST request: %s", err.Error()) + return false, fmt.Errorf("failed to create an http request: %s", err.Error()) } resp, err := httpClient.Do(req) diff --git a/internal/test/scorecard/common_utils.go b/internal/test/scorecard/common_utils.go index c36dc720..00933fdf 100644 --- a/internal/test/scorecard/common_utils.go +++ b/internal/test/scorecard/common_utils.go @@ -17,7 +17,9 @@ package scorecard import ( "context" + "errors" "fmt" + "io" "net/http" "net/url" "time" @@ -338,6 +340,48 @@ func createAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, resources } func waitTillCryostatReady(base *url.URL, resources *TestResources) error { + return sendHealthRequest(base, resources, func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error) { + health := &HealthResponse{} + err = ReadJSON(resp, health) + if err != nil { + return false, fmt.Errorf("failed to read response body: %s", err.Error()) + } + + if err = health.Ready(); err != nil { + r.Log += fmt.Sprintf("application is not yet ready: %s\n", err.Error()) + return false, nil // Try again + } + + r.Log += fmt.Sprintf("application is ready at %s\n", base.String()) + return true, nil + }) +} + +func waitTillReportReady(name string, namespace string, port int32, resources *TestResources) error { + client := resources.Client + r := resources.TestResult + + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + err := waitForDeploymentAvailability(ctx, client, namespace, name, r) + if err != nil { + return fmt.Errorf("report sidecar deployment did not become available: %s", err.Error()) + } + + reportsUrl := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", name, namespace, port) + base, err := url.Parse(reportsUrl) + if err != nil { + return fmt.Errorf("application URL is invalid: %s", err.Error()) + } + + return sendHealthRequest(base, resources, func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error) { + r.Log += fmt.Sprintf("reports sidecar is ready at %s\n", base.String()) + return true, nil + }) +} + +func sendHealthRequest(base *url.URL, resources *TestResources, healthCheck func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error)) error { client := NewHttpClient() r := resources.TestResult @@ -348,12 +392,15 @@ func waitTillCryostatReady(base *url.URL, resources *TestResources) error { url := base.JoinPath("/health") req, err := NewHttpRequest(ctx, http.MethodGet, url.String(), nil, make(http.Header)) if err != nil { - return false, fmt.Errorf("failed to create a Cryostat REST request: %s", err.Error()) + return false, fmt.Errorf("failed to create a an http request: %s", err.Error()) } req.Header.Add("Accept", "*/*") resp, err := client.Do(req) if err != nil { + if errors.Is(err, io.EOF) { + return false, nil // Retry + } return false, err } defer resp.Body.Close() @@ -365,22 +412,8 @@ func waitTillCryostatReady(base *url.URL, resources *TestResources) error { } return false, fmt.Errorf("API request failed with status code %d: %s", resp.StatusCode, ReadError(resp)) } - - health := &HealthResponse{} - err = ReadJSON(resp, health) - if err != nil { - return false, fmt.Errorf("failed to read response body: %s", err.Error()) - } - - if err = health.Ready(); err != nil { - r.Log += fmt.Sprintf("application is not yet ready: %s\n", err.Error()) - return false, nil // Try again - } - - r.Log += fmt.Sprintf("application is ready at %s\n", base.String()) - return true, nil + return healthCheck(resp, r) }) - return err } diff --git a/internal/test/scorecard/tests.go b/internal/test/scorecard/tests.go index 72bb48f1..fed488d7 100644 --- a/internal/test/scorecard/tests.go +++ b/internal/test/scorecard/tests.go @@ -33,6 +33,7 @@ const ( CryostatCRTestName string = "cryostat-cr" CryostatRecordingTestName string = "cryostat-recording" CryostatConfigChangeTestName string = "cryostat-config-change" + CryostatReportTestName string = "cryostat-report" ) // OperatorInstallTest checks that the operator installed correctly @@ -97,7 +98,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope if err != nil { return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error())) } - defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace) + defer cleanupCryostat(r, tr.Client, CryostatConfigChangeTestName, namespace) // Switch Cryostat CR to PVC for redeployment ctx, cancel := context.WithTimeout(context.Background(), testTimeout) @@ -126,7 +127,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope if err != nil { return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error())) } - r.Log += "Cryostat deployment has successfully updated with new spec template" + r.Log += "Cryostat deployment has successfully updated with new spec template\n" base, err := url.Parse(cr.Status.ApplicationURL) if err != nil { @@ -278,3 +279,39 @@ func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openSh return *r } + +func CryostatReportTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult { + tr := newTestResources(CryostatReportTestName) + r := tr.TestResult + + err := setupCRTestResources(tr, openShiftCertManager) + if err != nil { + return fail(*r, fmt.Sprintf("failed to set up %s test: %s", CryostatReportTestName, err.Error())) + } + + port := int32(10000) + cr := newCryostatCR(CryostatReportTestName, namespace, !tr.OpenShift) + cr.Spec.ReportOptions = &operatorv1beta1.ReportConfiguration{ + Replicas: 1, + } + cr.Spec.ServiceOptions = &operatorv1beta1.ServiceConfigList{ + ReportsConfig: &operatorv1beta1.ReportsServiceConfig{ + HTTPPort: &port, + }, + } + + // Create a default Cryostat CR + cr, err = createAndWaitTillCryostatAvailable(cr, tr) + if err != nil { + return fail(*r, fmt.Sprintf("%s test failed: %s", CryostatReportTestName, err.Error())) + } + defer cleanupCryostat(r, tr.Client, CryostatReportTestName, namespace) + + // Query health of report sidecar + err = waitTillReportReady(cr.Name+"-reports", cr.Namespace, port, tr) + if err != nil { + return fail(*r, fmt.Sprintf("failed to reach the application: %s", err.Error())) + } + + return *r +}