feat: initial merge of ci.yml and cd.yml workflows to begin testing (no removing live CI workflows yet) #474
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: earthly ci | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, closed] | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| upload: | |
| description: "Upload the container to our registry" | |
| default: false | |
| type: boolean | |
| repository: | |
| description: "GitHub repository name (e.g., username/repo)" | |
| required: true | |
| default: 'input-output-hk/partner-chains' | |
| branch: | |
| description: "Branch name to build" | |
| required: true | |
| default: 'master' | |
| env: | |
| AWS_REGION: "eu-central-1" | |
| SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
| jobs: | |
| build-and-push: | |
| runs-on: ubuntu-latest | |
| if: github.event.action != 'closed' || github.event.pull_request.merged == true | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repository }} | |
| ref: ${{ github.event.inputs.branch }} | |
| - name: Setup Earthly | |
| uses: ./.github/actions/earthly-setup | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }} | |
| config_tar: ${{ secrets.EARTHLY_TAR }} | |
| - name: Build With Benchmarking Features Enabled | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| env: | |
| EARTHLY_CI: true | |
| run: | | |
| export EARTHLY_OUTPUT=true | |
| earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks | |
| - name: Generate and Extract Weights | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| continue-on-error: true | |
| run: | | |
| repository_name="${GITHUB_REPOSITORY##*/}" | |
| echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:" | |
| ls -la /home/runner/work/${repository_name}/${repository_name} | |
| echo "Pulling Docker image..." | |
| docker pull ubuntu:22.04 | |
| mkdir -p weights | |
| echo "Running Docker container..." | |
| docker run -d --name weight_generation \ | |
| --memory=4096m \ | |
| --cpus=1 \ | |
| -v /home/runner/work/${repository_name}/${repository_name}:/workspace \ | |
| ubuntu:22.04 \ | |
| /bin/bash -c "sleep infinity" | |
| echo "Installing necessary packages inside the container..." | |
| docker exec weight_generation bash -c "\ | |
| apt-get update && \ | |
| apt-get install -y jq curl build-essential && \ | |
| echo 'Checking files in workspace...' && \ | |
| ls -la /workspace && \ | |
| mkdir -p /workspace/target/production && \ | |
| cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \ | |
| echo 'Verifying the binary is in the expected path...' && \ | |
| ls -la /workspace/target/production && \ | |
| cd /workspace && \ | |
| echo 'Setting the current working directory to /workspace...' && \ | |
| chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \ | |
| chmod +x scripts/run_storage_benchmarks.sh && \ | |
| source .envrc || true && \ | |
| ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \ | |
| ./scripts/run_storage_benchmarks.sh -b || true" | |
| echo "Finding and copying weight files..." | |
| weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs') | |
| echo "$weight_files" | while read weight_file; do | |
| weight_file_name=$(basename "$weight_file") | |
| echo "Copying ${weight_file_name}" | |
| docker cp "weight_generation:$weight_file" "weights/${weight_file_name}" | |
| done | |
| docker stop weight_generation | |
| docker rm weight_generation | |
| - name: Upload Weights | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: weights | |
| path: weights/ | |
| - name: Overwrite Weights in Runtime Directory | |
| continue-on-error: true | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| run: | | |
| pwd | |
| ls -la | |
| sudo chmod -R a+rwx ./runtime/src/weights | |
| for weight_file in weights/*.rs | |
| do | |
| cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")" | |
| done | |
| - name: Acquire AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Login to container registry | |
| uses: docker/login-action@v3 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| registry: ${{ secrets.ECR_REGISTRY_SECRET }} | |
| - name: Main Build | |
| env: | |
| EARTHLY_CI: true | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| run: | | |
| ref=${{ github.ref_name }} | |
| type=${{ github.ref_type }} | |
| args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node") | |
| event_name="${{ github.event_name }}" | |
| if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then | |
| export EARTHLY_PUSH=true | |
| args+=("--tags=$ref") | |
| elif [[ \ | |
| "${{ github.event.pull_request.merged }}" == 'true' \ | |
| && "$ref" == 'master' \ | |
| || "${{ inputs.upload }}" == 'true' \ | |
| ]]; then | |
| export EARTHLY_PUSH=true | |
| export EARTHLY_OUTPUT=true | |
| fi | |
| if [[ "$EARTHLY_PUSH" == true ]]; then | |
| args+=(--PROFILE=production) | |
| fi | |
| earthly -P +ci ${args[@]} | |
| - name: Upload chain spec artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }} | |
| with: | |
| name: chain-specs | |
| path: | | |
| ./devnet_chain_spec.json | |
| ./staging_chain_spec.json | |
| deploy-argocd: | |
| needs: build-and-push | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| steps: | |
| - name: Deploy ArgoCD Node | |
| uses: ./.github/actions/deploy/argocd/deploy-argocd | |
| with: | |
| sha: ${{ github.sha }} | |
| env: | |
| ACTIONS_PAT: ${{ secrets.ACTIONS_PAT }} | |
| argocd-tests: | |
| needs: deploy-argocd | |
| runs-on: [self-hosted, eks] | |
| steps: | |
| - name: Run Tests | |
| uses: ./.github/actions/tests/run-k8-tests | |
| with: | |
| node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local | |
| node-port: 9933 | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
| AWS_ROLE_ARN_: ${{ secrets.AWS_ROLE_ARN_ }} | |
| SSH_KEY_BINARY_HOST: ${{ secrets.SSH_KEY_BINARY_HOST }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| JIRA_URL: ${{ secrets.JIRA_URL }} | |
| ACTIONS_PAT: ${{ secrets.ACTIONS_PAT }} | |
| teardown-argocd: | |
| needs: argocd-tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Teardown ArgoCD Environment | |
| uses: ./.github/actions/deploy/argocd/teardown-argocd | |
| with: | |
| sha: ${{ github.sha }} | |
| env: | |
| ACTIONS_PAT: ${{ secrets.ACTIONS_PAT }} | |
| chain-specs: | |
| runs-on: [self-hosted, eks] | |
| needs: [build-and-push] | |
| if: github.ref_name == 'master' || inputs.upload == true | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Install kubectl and awscli | |
| run: | | |
| curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" | |
| chmod +x ./kubectl | |
| sudo mv ./kubectl /usr/local/bin/kubectl | |
| sudo apt update && sudo apt install -y awscli | |
| - name: Configure kubectl | |
| run: | | |
| echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml | |
| kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true | |
| kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} | |
| kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default | |
| kubectl config use-context my-context | |
| - name: Download chain spec artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: chain-specs | |
| path: ./artifacts | |
| - name: Update Kubernetes secret for devnet chain spec | |
| run: | | |
| TIMESTAMP=$(date +%Y%m%d%H%M) | |
| SHA=${{ github.sha }} | |
| kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc | |
| - name: Update Kubernetes secret for staging chain spec | |
| run: | | |
| TIMESTAMP=$(date +%Y%m%d%H%M) | |
| SHA=${{ github.sha }} | |
| kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging |