From d5362fdaf7da5dd60f26da2ce4c64313c5930317 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 13 May 2024 16:06:55 +0200 Subject: [PATCH] [BK] Migrate batch 1 (Artifact builds) (#182582) ## Summary Migrates batch 1 - artifact builds. The upload aspect wasn't tested, because it's programmed only to run from `main`, and we didn't want to interfere with the ongoing releases. This can be tested after the merge. Verification: - [x] RREs tested locally - [x] kibana / artifacts trigger (https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/87) - [x] kibana / artifacts container image (https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/86) - [x] kibana / artifacts snapshot (https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/88) - [x] kibana / artifacts staging (https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/88) - [x] 8.13 / 8.14 / 7.17 verification (only a few jobs need to work here) Originals: - kibana / artifacts trigger [kibana-artifacts-trigger.yml](https://buildkite.com/elastic/kibana-artifacts-trigger) - kibana / artifacts container image [kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-container-image) - kibana / artifacts snapshot [kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-snapshot) - kibana / artifacts staging [kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-staging) Backports: - https://github.com/elastic/kibana/pull/182781 - https://github.com/elastic/kibana/pull/182780 The backports don't need to have the pipeline resource definition files, however, we forked 8.14 off from main, where we already had the resources. I'll remove all the unnecessary resource defs from the legacy branches, once we finalize the state (simply to save a little inconvenience on future backports.) --- .../kibana-artifacts-container-image.yml | 44 +++++++++++++ .../kibana-artifacts-snapshot.yml | 62 ++++++++++++++++++ .../kibana-artifacts-staging.yml | 58 +++++++++++++++++ .../kibana-artifacts-trigger.yml | 55 ++++++++++++++++ .../locations.yml | 4 ++ .buildkite/pipelines/artifacts.yml | 64 ++++++++++++++++--- .../pipelines/artifacts_container_image.yml | 5 +- .buildkite/pipelines/artifacts_trigger.yml | 5 +- 8 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 .buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml create mode 100644 .buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml create mode 100644 .buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml create mode 100644 .buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml new file mode 100644 index 00000000000000..eff970c69af6be --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml @@ -0,0 +1,44 @@ +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-artifacts-container-image + description: Kibana container image artifact builds + links: + - url: https://buildkite.com/elastic/kibana-artifacts-container-image + title: Pipeline link +spec: + type: buildkite-pipeline + owner: group:kibana-operations + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / artifacts container image + description: Kibana container image artifact builds + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-serverless-test-alerts' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/artifacts_container_image.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml new file mode 100644 index 00000000000000..99e6f79ba31019 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml @@ -0,0 +1,62 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-artifacts-snapshot + description: Kibana snapshot artifact builds + links: + - url: https://buildkite.com/elastic/kibana-artifacts-snapshot + title: Pipeline link +spec: + type: buildkite-pipeline + owner: group:kibana-operations + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / artifacts snapshot + description: Kibana snapshot artifact builds + spec: + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main 8.14 8.13 7.17 + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/artifacts.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build (main): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: main + Daily build (8.13): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '8.13' + Daily build (8.14): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '8.14' + Daily build (7.17): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '7.17' diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml new file mode 100644 index 00000000000000..e00e0ee9633627 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml @@ -0,0 +1,58 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-artifacts-staging + description: Kibana staging artifact builds + links: + - url: https://buildkite.com/elastic/kibana-artifacts-staging + title: Pipeline link +spec: + type: buildkite-pipeline + owner: group:kibana-operations + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / artifacts staging + description: Kibana staging artifact builds + spec: + env: + RELEASE_BUILD: 'true' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: 7.17 8.13 8.14 + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/artifacts.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build (8.13): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '8.13' + Daily build (8.14): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '8.14' + Daily build (7.17): + cronline: 0 7 * * * America/New_York + message: Daily build + branch: '7.17' diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml new file mode 100644 index 00000000000000..01a791dda269a1 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-artifacts-trigger + description: Kibana artifact trigger + links: + - url: 'https://buildkite.com/elastic/kibana-artifacts-trigger' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / artifacts trigger + description: Kibana artifact trigger + spec: + env: + RELEASE_BUILD: 'true' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: '8.13 8.14' + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/artifacts_trigger.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build (8.13): + cronline: 0 */2 * * * America/New_York + message: Daily build + branch: '8.13' + Daily build (8.14): + cronline: 0 */2 * * * America/New_York + message: Daily build + branch: '8.14' diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index 7e1d136589fc13..bd28f45d799acf 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -9,6 +9,10 @@ spec: targets: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-apis-capacity-testing-daily.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 6f5d989f1a3a54..7d3f794fb95109 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -2,7 +2,10 @@ steps: - command: .buildkite/scripts/steps/artifacts/build.sh label: Build Kibana Artifacts agents: - queue: c2-16 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: c2-standard-16 timeout_in_minutes: 60 retry: automatic: @@ -14,7 +17,13 @@ steps: - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh label: Artifact Testing agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 timeout_in_minutes: 30 retry: automatic: @@ -24,7 +33,13 @@ steps: - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh label: Artifact Testing agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 timeout_in_minutes: 30 retry: automatic: @@ -34,7 +49,13 @@ steps: - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh label: Artifact Testing agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 timeout_in_minutes: 30 retry: automatic: @@ -44,7 +65,12 @@ steps: - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh label: 'Docker Context Verification' agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-2 timeout_in_minutes: 30 retry: automatic: @@ -54,7 +80,12 @@ steps: - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh label: 'Docker Context Verification' agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-2 timeout_in_minutes: 30 retry: automatic: @@ -64,7 +95,12 @@ steps: - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh label: 'Docker Context Verification' agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-2 timeout_in_minutes: 30 retry: automatic: @@ -77,7 +113,12 @@ steps: - exit_status: 255 - exit_status: -1 agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-2 timeout_in_minutes: 60 if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'" retry: @@ -99,5 +140,10 @@ steps: - command: .buildkite/scripts/steps/artifacts/publish.sh label: 'Publish Kibana Artifacts' agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-2 timeout_in_minutes: 30 diff --git a/.buildkite/pipelines/artifacts_container_image.yml b/.buildkite/pipelines/artifacts_container_image.yml index 63744d64aba92f..8f4436fb7db9e1 100644 --- a/.buildkite/pipelines/artifacts_container_image.yml +++ b/.buildkite/pipelines/artifacts_container_image.yml @@ -2,5 +2,8 @@ steps: - command: .buildkite/scripts/steps/artifacts/docker_image.sh label: Build serverless container images agents: - queue: c2-16 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-16 timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/artifacts_trigger.yml b/.buildkite/pipelines/artifacts_trigger.yml index 1e289f4116abcd..98851ddea31add 100644 --- a/.buildkite/pipelines/artifacts_trigger.yml +++ b/.buildkite/pipelines/artifacts_trigger.yml @@ -2,5 +2,8 @@ steps: - command: .buildkite/scripts/steps/artifacts/trigger.sh label: Trigger artifacts build agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10