[7.17][BK] Migrate the kibana-on-merge job (elastic#184497)

## Summary Backport of elastic#184391 to 7.17 Migration staging run: - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/106 - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/121
Makila-AI · Aug 5, 2024 · 93aa02e · 93aa02e
1 parent e7bd125
commit 93aa02e
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 23 deletions.
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -6,7 +6,10 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     retry:
       automatic:
         - exit_status: '*'
@@ -17,7 +20,10 @@ steps:
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: c2-standard-16
     key: build
     timeout_in_minutes: 60
     retry:
@@ -29,7 +35,12 @@ steps:
     label: Build TS Refs and Check Public API Docs
     key: public-api-docs
     agents:
-      queue: c2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-4
     timeout_in_minutes: 80
     retry:
       automatic:
@@ -39,7 +50,10 @@ steps:
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -53,7 +67,12 @@ steps:
     label: 'Default CI Group'
     parallelism: 27
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -65,7 +84,12 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -78,7 +102,11 @@ steps:
     label: 'OSS CI Group'
     parallelism: 11
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     key: oss-cigroup
@@ -90,7 +118,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
     label: 'OSS Accessibility Tests'
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -101,7 +133,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -112,7 +149,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_firefox.sh
     label: 'OSS Firefox Tests'
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -123,7 +164,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -134,7 +180,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -145,7 +196,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -157,43 +213,71 @@ steps:
     label: 'Jest Tests'
     parallelism: 8
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 90
     key: jest
 
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     parallelism: 3
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 120
     key: jest-integration
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 120
     key: api-integration
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     key: linting
     timeout_in_minutes: 90
 
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: c2-standard-16
     key: linting_with_types
     timeout_in_minutes: 90
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -203,20 +287,35 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: c2-8
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-8
     timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:
-      queue: c2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-4
     key: storybooks
     timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
     label: 'Populate local dev bazel cache (Linux)'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 15
     retry:
       automatic:
@@ -230,4 +329,7 @@ steps:
     label: Post-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh
@@ -28,6 +28,7 @@ fi
 if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
   echo "--- Upload ts-refs-cache"
   cd "$KIBANA_DIR/target/ts_refs_cache"
+  "$KIBANA_DIR/.buildkite/scripts/common/activate_service_account.sh" gs://kibana-ci-ts-refs-cache
   gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
   cd "$KIBANA_DIR"
 fi
diff --git a/.buildkite/scripts/common/activate_service_account.sh b/.buildkite/scripts/common/activate_service_account.sh
@@ -76,6 +76,11 @@ if [[ -z "$EMAIL" ]]; then
     "ci-artifacts.kibana.dev")
       EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
       ;;
+    "kibana-ci-ts-refs-cache")
+    # This bucket is used for ts-refs-cache in 7.17 only,
+    # set up with the kibana-ci-access-artifacts SA
+      EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
+      ;;
     *)
       EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
       ;;