From 4ae0028e34283eb68ec1697ebe6ba9d0cc3cdc5c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:37:29 +0200 Subject: [PATCH] [7.17](backport #38492) Add testing for the buildkite dynamic pipeline generator (#38544) This PR changes the main dynamic pipeline to detect if there are changes in the Python or yaml scripts and runs some Python tests to make sure that everything works before moving into triggering the pipeline. This will ensure that we don't accidentally break the dynamic pipeline generator. Signed-off-by: Alexandros Sapranidis Co-authored-by: Alexandros Sapranidis Co-authored-by: Dimitrios Liappis --- .buildkite/pipeline.yml | 3 - .buildkite/pytest.ini | 11 +++ .buildkite/scripts/generate_pipeline.sh | 7 +- .../scripts/run_dynamic_pipeline_tests.sh | 43 +++++++++++ .buildkite/test_pipeline.py | 71 +++++++++++++++++++ 5 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pytest.ini create mode 100755 .buildkite/scripts/run_dynamic_pipeline_tests.sh create mode 100644 .buildkite/test_pipeline.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8f124fea4ae..84f725981fe 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -5,6 +5,3 @@ env: steps: - label: "Generate dynamic pipeline" command: ".buildkite/scripts/generate_pipeline.sh" - agents: - provider: "gcp" - image: "family/platform-ingest-beats-ubuntu-2204" diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini new file mode 100644 index 00000000000..3eff7473d9f --- /dev/null +++ b/.buildkite/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +junit_family=xunit1 + +addopts = --strict-markers +markers = + load: Load tests + tag(name): Tag tests with Go-like semantics + +# Ignore setup and teardown for the timeout +#timeout_func_only = True + diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 0d9703e9660..c5627aaf007 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -2,7 +2,12 @@ set -euo pipefail echo "~~~ Install dependencies" -pip3 install --quiet "ruamel.yaml<0.18.0" +python3 -mpip install --quiet "ruamel.yaml<0.18.0" +# temporary solution until we have this into a base container +curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 +chmod a+x /usr/bin/yq + +.buildkite/scripts/run_dynamic_pipeline_tests.sh echo "+++ Run pipeline generator in dry-run mode" python3 .buildkite/pipeline.py | yq . diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh new file mode 100755 index 00000000000..8eb72d7a96b --- /dev/null +++ b/.buildkite/scripts/run_dynamic_pipeline_tests.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed +# this will allow us to fail fast, if e.g. a PR has broken the generator + +set -euo pipefail + +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +pipeline_generator_changeset=( + "^.buildkite/pipeline.py" + "^*/buildkite.yml" +) + +if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then + echo "~~~ Skipping pipeline generator tests" + exit +fi + +echo "~~~ Running pipeline generator tests" + +python3 -mpip install --quiet "pytest" +pushd .buildkite +pytest . +popd diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py new file mode 100644 index 00000000000..75fd949ccc8 --- /dev/null +++ b/.buildkite/test_pipeline.py @@ -0,0 +1,71 @@ +import os + +import pytest +import pipeline + + +@pytest.fixture +def ubuntu2204_aws_agent(): + return { + "command": "fake-cmd", + "platform": "platform-ingest-beats-ubuntu-2204-aarch64", + "provider": "aws" + } + + +@pytest.fixture() +def fake_simple_group(): + return { + "unitTest": { + "command": "fake-cmd", + "platform": "family/platform-ingest-beats-ubuntu-2204", + }, + "integrationTest": { + "command": "fake-integration", + "platform": "family/platform-ingest-beats-ubuntu-2204", + "env": { + "FOO": "BAR", + }, + }, + } + + +def test_fetch_stage(ubuntu2204_aws_agent): + step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") + assert step.create_entity() == { + "label": "fake test", + "command": ["cd fake", "fake-cmd"], + "notify": [ + { + "github_commit_status": { + "context": "Fake: test", + } + } + ], + "agents": { + "provider": "aws", + "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", + "instanceType": "t4g.large", + }, + "artifact_paths": [ + "fake/build/*.xml", + "fake/build/*.json", + ], + } + + +def test_fetch_group(fake_simple_group): + group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") + assert len(group.steps) == 2 + for step in group.steps: + assert "testing" == step.category + assert "gcp" == step.agent.provider + + assert group.steps[1].env.get("FOO") == "BAR" + + +def test_is_pr(): + os.environ["BUILDKITE_PULL_REQUEST"] = "1234" + assert pipeline.is_pr() is True + os.environ["BUILDKITE_PULL_REQUEST"] = "false" + assert pipeline.is_pr() is False