diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 00000000000..50b8104cb6d --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,30 @@ +# Buildkite + +This README provides an overview of the Buildkite pipeline used to automate the build and publish process for APM Server artifacts. + +## APM Server Package pipeline + +This is the Buildkite pipeline used to run the APM Server package generation and the DRA automation. + +### Artifacts + +The pipeline generates the following artifacts: + +- **dependencies-APM-SERVER_VERSION-WORKFLOW.csv**: This CSV file contains a list of dependencies for the specific APM-Server version being built. It helps track build dependencies. + +- **apm-server-APM-SERVER_VERSION-WORKFLOW-SUFFIX.EXT**: This file includes the APM Server binary. + +### Triggering the Pipeline + +The pipeline is triggered in the following scenarios: + +- **Snapshot Builds**: A snapshot build is triggered when a pull request (PR) is merged into the 'main' branch or a version-specific branch. + +- **Staging Builds**: A staging build is triggered when a PR is merged into a version-specific branch. Staging builds are typically used for a release build candidate. + +After a successful build, the pipeline publishes the generated artifacts to the Google Cloud Storage (GCS) bucket named [elastic-artifacts-snapshot/apm-server](https://console.cloud.google.com/storage/browser/elastic-artifacts-snapshot/apm-server). You can access the published artifacts in this bucket. + +### Pipeline Configuration + +To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/apm-server-package) or +go to the [catalog-info.yaml](../catalog-info.yaml) file. diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100755 index 00000000000..15126bcfadf --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +## +## This script prepares the git context so it works with GitHub Pull Requests. +## +## Originally implemented in https://github.com/elastic/elastic-agent/blob/main/.buildkite/hooks/post-checkout +## + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "--- Commit information" +git log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 00000000000..427b4a0f1ad --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +## +## This script prepares the Vault context and required tooling +## for the package pipelines. +## +## NOTE: *_SECRET or *_TOKEN env variables are masked, hence if you'd like to avoid any +## surprises please use the suffix _SECRET or _TOKEN for those values that contain +## any sensitive data. Buildkite can mask those values automatically +## + +set -eo pipefail + +retry() { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +get_os_details() { + case $(uname | tr '[:upper:]' '[:lower:]') in + linux*) + export OS_NAME=linux + ;; + darwin*) + export OS_NAME=osx + ;; + msys*) + export OS_NAME=windows + ;; + *) + export OS_NAME=notset + ;; + esac + case $(uname -m | tr '[:upper:]' '[:lower:]') in + aarch64*) + export OS_ARCH=arm64 + ;; + arm64*) + export OS_ARCH=arm64 + ;; + amd64*) + export OS_ARCH=amd64 + ;; + x86_64*) + export OS_ARCH=amd64 + ;; + *) + export OS_ARCH=notset + ;; + esac +} + +if command -v docker &>/dev/null; then + echo "--- Prepare docker context :docker:" + DOCKER_REGISTRY_SECRET_PATH="secret/ci/elastic-apm-server/docker-elastic-observability" + DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field username "${DOCKER_REGISTRY_SECRET_PATH}") + DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + DOCKER_REGISTRY_SECRET=$(retry 5 vault kv get -field registry "${DOCKER_REGISTRY_SECRET_PATH}") + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY_SECRET}" + unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET + export DOCKER_REGISTRY_SECRET + retry 4 docker pull --quiet docker.elastic.co/infra/release-manager:latest +fi + +echo "--- Setting up the :golang: environment..." +GO_VERSION=$(cat .go-version) +WORKSPACE=$(git rev-parse --show-toplevel) +export GO_VERSION WORKSPACE +get_os_details +retry 5 curl -sL -o ${WORKSPACE}/gvm "https://github.com/andrewkroh/gvm/releases/download/v0.5.1/gvm-${OS_NAME}-${OS_ARCH}" +chmod +x ${WORKSPACE}/gvm +eval "$(${WORKSPACE}/gvm $GO_VERSION)" +echo "Golang version:" +go version +GOPATH=$(command go env GOPATH) +PATH="${PATH}:$GOPATH:$GOPATH/bin" +export PATH + +echo "--- Prepare vault context :vault:" +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" +DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) +VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') +VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') +VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') +export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100755 index 00000000000..8368ff3d62c --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +## +## Cleanup the created env variables on the fly. +## + +unset VAULT_ROLE_ID_SECRET +unset VAULT_ADDR_SECRET +unset VAULT_SECRET + +if command -v docker &>/dev/null; then + docker logout $DOCKER_REGISTRY_SECRET +fi diff --git a/.buildkite/package.yml b/.buildkite/package.yml index 34321b61161..e0405b2ef7f 100644 --- a/.buildkite/package.yml +++ b/.buildkite/package.yml @@ -1,5 +1,53 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +notify: + - slack: "#apm-server" + if: build.state == "failed" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" steps: - - label: "Example test" - command: echo "Hello!" + - group: "Package" + key: "package" + steps: + - label: "Package Ubuntu-20 x86_64" + key: "package-x86-64" + command: ".buildkite/scripts/package.sh {{matrix.type}}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + matrix: + setup: + type: + - "snapshot" + - "staging" + artifact_paths: + - "build/distributions/**/*" + - "build/dependencies*.csv" + + - label: "Package Ubuntu-20 aarch64" + key: "package-arm" + command: ".buildkite/scripts/package.sh {{matrix.type}}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.2xlarge" + matrix: + setup: + type: + - "snapshot" + - "staging" + artifact_paths: + - "build/distributions/**/*" + + - label: "DRA" + key: "dra" + command: ".buildkite/scripts/dra.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + depends_on: + - step: "package" + allow_failure: false diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100644 index 00000000000..69481c31716 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +## +## It relies on the .buildkite/hooks/pre-command so the Vault and other tooling +## are prepared automatically by buildkite. +## +## It downloads the generated artifacts and run the DRA only if the branch is an active +## branch, based on the Unified Release policy. Otherwise, it won't run the DRA but print +## some traces. +## + +set -eo pipefail + +## Read current version. +VERSION=$(make get-version) + +echo "--- Restoring Artifacts" +buildkite-agent artifact download "build/**/*" . +buildkite-agent artifact download "build/dependencies*.csv" . +# The dependencies file needs to be saved in the build/distributions folder +cp build/dependencies*.csv build/distributions/ + +echo "--- Changing permissions for the release manager" +sudo chown -R :1000 build/ +ls -l build/distributions/ + +if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then + echo "--- :arrow_right: Release Manager does not run on PRs, skipping" + exit 0 +fi + +curl -s https://storage.googleapis.com/artifacts-api/snapshots/branches.json > active-branches.json +if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then + echo "--- :arrow_right: Release Manager only supports the current active branches, skipping" + echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH" + echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT" + echo "VERSION=$VERSION" + echo "Supported branches:" + cat active-branches.json + exit 0 +fi + +dra() { + local workflow=$1 + echo "--- Run release manager $workflow" + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src=$(pwd),target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project apm-server \ + --branch $BUILDKITE_BRANCH \ + --commit $BUILDKITE_COMMIT \ + --workflow $workflow \ + --artifact-set main \ + --version $VERSION +} + +dra "snapshot" +if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then + dra "staging" +fi diff --git a/.buildkite/scripts/package.sh b/.buildkite/scripts/package.sh new file mode 100755 index 00000000000..26c021445e6 --- /dev/null +++ b/.buildkite/scripts/package.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +## +## Builds the distribution packages: tarballs, RPMs, Docker images, etc. +## It runs for both linux/amd64 and linux/arm64. On linux/amd64 it builds all packages; +## on linux/arm64 it builds only Docker images. +## + +set -eo pipefail + +TYPE="$1" +PLATFORM_TYPE=$(uname -m) + +MAKE_GOAL=package +if [[ ${PLATFORM_TYPE} == "arm" || ${PLATFORM_TYPE} == "aarch64" ]]; then + MAKE_GOAL=package-docker +fi + +if [[ ${TYPE} == "snapshot" ]]; then + MAKE_GOAL="${MAKE_GOAL}-snapshot" +fi + +make $MAKE_GOAL diff --git a/catalog-info.yaml b/catalog-info.yaml index a672d94ee4d..a95c5d941a2 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -36,12 +36,6 @@ spec: spec: repository: elastic/apm-server pipeline_file: ".buildkite/package.yml" - provider_settings: - # TODO: delete this section once the BK pipeline works as expected. - build_branches: false # Disable builds from branches. - # Will not create any builds based on GitHub activity but it will be - # triggered using the GitHub Buildkite composite action. - #trigger_mode: none cancel_intermediate_builds: false skip_intermediate_builds: false teams: