-
Notifications
You must be signed in to change notification settings - Fork 4.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
CI Split integration tests to run nightly and every PR #18518
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,314 @@ | ||
# Copyright (c) HashiCorp, Inc. | ||
# SPDX-License-Identifier: MPL-2.0 | ||
|
||
name: Nightly test-integrations | ||
|
||
on: | ||
schedule: | ||
# Run nightly at 12AM UTC/8PM EST/5PM PST | ||
- cron: '* 0 * * *' | ||
workflow_dispatch: {} | ||
|
||
env: | ||
TEST_RESULTS_DIR: /tmp/test-results | ||
TEST_RESULTS_ARTIFACT_NAME: test-results | ||
CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} | ||
GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} | ||
GOTESTSUM_VERSION: "1.10.1" | ||
CONSUL_BINARY_UPLOAD_NAME: consul-bin | ||
# strip the hashicorp/ off the front of github.repository for consul | ||
CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }} | ||
GOPRIVATE: github.com/hashicorp # Required for enterprise deps | ||
|
||
jobs: | ||
setup: | ||
runs-on: ubuntu-latest | ||
name: Setup | ||
outputs: | ||
compute-small: ${{ steps.runners.outputs.compute-small }} | ||
compute-medium: ${{ steps.runners.outputs.compute-medium }} | ||
compute-large: ${{ steps.runners.outputs.compute-large }} | ||
compute-xl: ${{ steps.runners.outputs.compute-xl }} | ||
enterprise: ${{ steps.runners.outputs.enterprise }} | ||
steps: | ||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
- id: runners | ||
run: .github/scripts/get_runner_classes.sh | ||
|
||
dev-build: | ||
needs: [setup] | ||
uses: ./.github/workflows/reusable-dev-build.yml | ||
with: | ||
runs-on: ${{ needs.setup.outputs.compute-xl }} | ||
repository-name: ${{ github.repository }} | ||
uploaded-binary-name: 'consul-bin' | ||
secrets: | ||
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} | ||
|
||
generate-envoy-job-matrices: | ||
needs: [setup] | ||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} | ||
name: Generate Envoy Job Matrices | ||
outputs: | ||
envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} | ||
steps: | ||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
- name: Generate Envoy Job Matrix | ||
id: set-matrix | ||
env: | ||
# this is further going to multiplied in envoy-integration tests by the | ||
# other dimensions in the matrix. Currently TOTAL_RUNNERS would be | ||
# multiplied by 8 based on these values: | ||
# envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"] | ||
# xds-target: ["server", "client"] | ||
TOTAL_RUNNERS: 4 | ||
JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' | ||
run: | | ||
NUM_RUNNERS=$TOTAL_RUNNERS | ||
NUM_DIRS=$(find ./test/integration/connect/envoy -mindepth 1 -maxdepth 1 -type d | wc -l) | ||
|
||
if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then | ||
echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." | ||
NUM_RUNNERS=$((NUM_DIRS-1)) | ||
fi | ||
# fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. | ||
NUM_RUNNERS=$((NUM_RUNNERS-1)) | ||
{ | ||
echo -n "envoy-matrix=" | ||
find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ | ||
| xargs -0 -n 1 basename \ | ||
| jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ | ||
| jq --compact-output 'map(join("|"))' | ||
} >> "$GITHUB_OUTPUT" | ||
|
||
envoy-integration-test: | ||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} | ||
needs: | ||
- setup | ||
- generate-envoy-job-matrices | ||
- dev-build | ||
permissions: | ||
id-token: write # NOTE: this permission is explicitly required for Vault auth. | ||
contents: read | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
envoy-version: ["1.24.10", "1.25.9", "1.26.4"] | ||
xds-target: ["server", "client"] | ||
test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} | ||
env: | ||
ENVOY_VERSION: ${{ matrix.envoy-version }} | ||
XDS_TARGET: ${{ matrix.xds-target }} | ||
AWS_LAMBDA_REGION: us-west-2 | ||
steps: | ||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 | ||
with: | ||
go-version-file: 'go.mod' | ||
|
||
- name: fetch binary | ||
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | ||
with: | ||
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' | ||
path: ./bin | ||
- name: restore mode+x | ||
run: chmod +x ./bin/consul | ||
|
||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@2a1a44ac4aa01993040736bd95bb470da1a38365 # v2.9.0 | ||
|
||
- name: Docker build | ||
run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin | ||
|
||
- name: Envoy Integration Tests | ||
env: | ||
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml | ||
GOTESTSUM_FORMAT: standard-verbose | ||
COMPOSE_INTERACTIVE_NO_CLI: 1 | ||
LAMBDA_TESTS_ENABLED: "true" | ||
# tput complains if this isn't set to something. | ||
TERM: ansi | ||
run: | | ||
# shellcheck disable=SC2001 | ||
echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" | ||
# shellcheck disable=SC2001 | ||
sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" | ||
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ | ||
--debug \ | ||
--rerun-fails \ | ||
--rerun-fails-report=/tmp/gotestsum-rerun-fails \ | ||
--jsonfile /tmp/jsonfile/go-test.log \ | ||
--packages=./test/integration/connect/envoy \ | ||
-- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" | ||
|
||
# NOTE: ENT specific step as we store secrets in Vault. | ||
- name: Authenticate to Vault | ||
if: ${{ endsWith(github.repository, '-enterprise') }} | ||
id: vault-auth | ||
run: vault-auth | ||
|
||
# NOTE: ENT specific step as we store secrets in Vault. | ||
- name: Fetch Secrets | ||
if: ${{ endsWith(github.repository, '-enterprise') }} | ||
id: secrets | ||
uses: hashicorp/vault-action@v2.5.0 | ||
with: | ||
url: ${{ steps.vault-auth.outputs.addr }} | ||
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} | ||
token: ${{ steps.vault-auth.outputs.token }} | ||
secrets: | | ||
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; | ||
|
||
- name: prepare datadog-ci | ||
if: ${{ !endsWith(github.repository, '-enterprise') }} | ||
run: | | ||
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" | ||
chmod +x /usr/local/bin/datadog-ci | ||
|
||
- name: upload coverage | ||
# do not run on forks | ||
if: github.event.pull_request.head.repo.full_name == github.repository | ||
env: | ||
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" | ||
DD_ENV: ci | ||
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml | ||
|
||
upgrade-integration-test: | ||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} | ||
needs: | ||
- setup | ||
- dev-build | ||
permissions: | ||
id-token: write # NOTE: this permission is explicitly required for Vault auth. | ||
contents: read | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
consul-version: [ "1.15", "1.16"] | ||
env: | ||
CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} | ||
ENVOY_VERSION: "1.24.6" | ||
steps: | ||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. | ||
- name: Setup Git | ||
if: ${{ endsWith(github.repository, '-enterprise') }} | ||
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" | ||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 | ||
with: | ||
go-version-file: 'go.mod' | ||
- run: go env | ||
|
||
# Get go binary from workspace | ||
- name: fetch binary | ||
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | ||
with: | ||
name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' | ||
path: . | ||
- name: restore mode+x | ||
run: chmod +x consul | ||
- name: Build consul:local image | ||
run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . | ||
- name: Build consul-envoy:latest-version image | ||
id: buildConsulEnvoyLatestImage | ||
continue-on-error: true | ||
run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets | ||
- name: Retry Build consul-envoy:latest-version image | ||
if: steps.buildConsulEnvoyLatestImage.outcome == 'failure' | ||
run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets | ||
- name: Build consul-envoy:target-version image | ||
id: buildConsulEnvoyTargetImage | ||
continue-on-error: true | ||
run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets | ||
- name: Retry Build consul-envoy:target-version image | ||
if: steps.buildConsulEnvoyTargetImage.outcome == 'failure' | ||
run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets | ||
- name: Build sds image | ||
run: docker build -t consul-sds-server ./test/integration/connect/envoy/test-sds-server/ | ||
- name: Configure GH workaround for ipv6 loopback | ||
if: ${{ !endsWith(github.repository, '-enterprise') }} | ||
run: | | ||
cat /etc/hosts && echo "-----------" | ||
sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts | ||
cat /etc/hosts | ||
- name: Upgrade Integration Tests | ||
run: | | ||
mkdir -p "${{ env.TEST_RESULTS_DIR }}" | ||
cd ./test/integration/consul-container/test/upgrade | ||
docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version | ||
go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ | ||
--raw-command \ | ||
--format=short-verbose \ | ||
--debug \ | ||
--rerun-fails=2 \ | ||
--packages="./..." \ | ||
-- \ | ||
go test \ | ||
-p=4 \ | ||
-tags "${{ env.GOTAGS }}" \ | ||
-timeout=30m \ | ||
-json \ | ||
./... \ | ||
--follow-log=false \ | ||
--target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ | ||
--target-version local \ | ||
--latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ | ||
--latest-version "${{ env.CONSUL_LATEST_VERSION }}" | ||
ls -lrt | ||
env: | ||
# this is needed because of incompatibility between RYUK container and GHA | ||
GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml | ||
GOTESTSUM_FORMAT: standard-verbose | ||
COMPOSE_INTERACTIVE_NO_CLI: 1 | ||
# tput complains if this isn't set to something. | ||
TERM: ansi | ||
# NOTE: ENT specific step as we store secrets in Vault. | ||
- name: Authenticate to Vault | ||
if: ${{ endsWith(github.repository, '-enterprise') }} | ||
id: vault-auth | ||
run: vault-auth | ||
|
||
# NOTE: ENT specific step as we store secrets in Vault. | ||
- name: Fetch Secrets | ||
if: ${{ endsWith(github.repository, '-enterprise') }} | ||
id: secrets | ||
uses: hashicorp/vault-action@v2.5.0 | ||
with: | ||
url: ${{ steps.vault-auth.outputs.addr }} | ||
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} | ||
token: ${{ steps.vault-auth.outputs.token }} | ||
secrets: | | ||
kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; | ||
|
||
- name: prepare datadog-ci | ||
if: ${{ !endsWith(github.repository, '-enterprise') }} | ||
run: | | ||
curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" | ||
chmod +x /usr/local/bin/datadog-ci | ||
|
||
- name: upload coverage | ||
# do not run on forks | ||
if: github.event.pull_request.head.repo.full_name == github.repository | ||
env: | ||
DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" | ||
DD_ENV: ci | ||
run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml | ||
|
||
|
||
test-integrations-success: | ||
needs: | ||
- setup | ||
- dev-build | ||
- generate-envoy-job-matrices | ||
- envoy-integration-test | ||
- upgrade-integration-test | ||
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} | ||
if: ${{ always() }} | ||
steps: | ||
- name: evaluate upstream job results | ||
run: | | ||
# exit 1 if failure or cancelled result for any upstream job | ||
if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion: On failure, you should probably send a slack message to the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @curtbushko , I will have a follow-up PR to setup the slack message job. |
||
printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}" | ||
exit 1 | ||
fi |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
thought: If we want nightly test runs for the release branches also, then you will need separate cron workflows similar to this and will need to set the release branch on checkout.
This is because Github Actions run all of the workflows off of main. You do this in the code checkout step.