feat: keep live workflows

.github/workflows/earthly.yml

@@ -0,0 +1,281 @@
+name: earthly ci
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
+    branches:
+      - master
+  workflow_dispatch:
+    inputs:
+      upload:
+        description: "Upload the container to our registry"
+        default: false
+        type: boolean
+      repository:
+        description: "GitHub repository name (e.g., username/repo)"
+        required: true
+        default: 'input-output-hk/partner-chains'
+      branch:
+        description: "Branch name to build"
+        required: true
+        default: 'master'
+env:
+  AWS_REGION: "eu-central-1"
+  SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.event.action != 'closed' || github.event.pull_request.merged == true
+    concurrency:
+      group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }}
+      cancel-in-progress: true
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.event.inputs.repository }}
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Setup Earthly
+        uses: ./.github/earthly-setup
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
+        with:
+          ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }}
+          config_tar: ${{ secrets.EARTHLY_TAR }}
+
+      - name: Build With Benchmarking Features Enabled
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
+        env:
+          EARTHLY_CI: true
+        run: |
+          export EARTHLY_OUTPUT=true
+          earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks
+
+      - name: Generate and Extract Weights
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
+        continue-on-error: true
+        run: |
+          repository_name="${GITHUB_REPOSITORY##*/}"
+          echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:"
+          ls -la /home/runner/work/${repository_name}/${repository_name}
+          echo "Pulling Docker image..."
+          docker pull ubuntu:22.04
+          mkdir -p weights
+          echo "Running Docker container..."
+          docker run -d --name weight_generation \
+              --memory=4096m \
+              --cpus=1 \
+              -v /home/runner/work/${repository_name}/${repository_name}:/workspace \
+              ubuntu:22.04 \
+              /bin/bash -c "sleep infinity"
+          echo "Installing necessary packages inside the container..."
+          docker exec weight_generation bash -c "\
+              apt-get update && \
+              apt-get install -y jq curl build-essential && \
+              echo 'Checking files in workspace...' && \
+              ls -la /workspace && \
+              mkdir -p /workspace/target/production && \
+              cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \
+              echo 'Verifying the binary is in the expected path...' && \
+              ls -la /workspace/target/production && \
+              cd /workspace && \
+              echo 'Setting the current working directory to /workspace...' && \
+              chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \
+              chmod +x scripts/run_storage_benchmarks.sh && \
+              source .envrc || true && \
+              ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \
+              ./scripts/run_storage_benchmarks.sh -b || true"
+          echo "Finding and copying weight files..."
+          weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs')
+          echo "$weight_files" | while read weight_file; do
+              weight_file_name=$(basename "$weight_file")
+              echo "Copying ${weight_file_name}"
+              docker cp "weight_generation:$weight_file" "weights/${weight_file_name}"
+          done
+          docker stop weight_generation
+          docker rm weight_generation
+
+      - name: Upload Weights
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
+        continue-on-error: true
+        uses: actions/upload-artifact@v4
+        with:
+          name: weights
+          path: weights/
+
+      - name: Overwrite Weights in Runtime Directory
+        continue-on-error: true
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
+        run: |
+          pwd
+          ls -la
+          sudo chmod -R a+rwx ./runtime/src/weights
+          for weight_file in weights/*.rs
+          do
+            cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")"
+          done
+
+      - name: Acquire AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to container registry
+        uses: docker/login-action@v3
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
+        with:
+          registry: ${{ secrets.ECR_REGISTRY_SECRET }}
+
+      - name: Main Build
+        env:
+          EARTHLY_CI: true
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
+        run: |
+          ref=${{ github.ref_name }}
+          type=${{ github.ref_type }}
+          args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node")
+          event_name="${{ github.event_name }}"
+          if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then
+            export EARTHLY_PUSH=true
+            args+=("--tags=$ref")
+          elif [[ \
+            "${{ github.event.pull_request.merged }}" == 'true' \
+            && "$ref" == 'master' \
+            || "${{ inputs.upload }}" == 'true' \
+          ]]; then
+            export EARTHLY_PUSH=true
+            export EARTHLY_OUTPUT=true
+          fi
+          if [[ "$EARTHLY_PUSH" == true ]]; then
+            args+=(--PROFILE=production)
+          fi
+          earthly -P +ci ${args[@]}
+
+      - name: Upload chain spec artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }}
+        with:
+          name: chain-specs
+          path: |
+            ./devnet_chain_spec.json
+            ./staging_chain_spec.json
+
+  generate-manifest:
+    needs: build-and-push
+    if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Create and Push Manifest
+        env:
+          GH_TOKEN: ${{ secrets.ACTIONS_PAT }}
+        run: |
+          cd .github/workflows/modules/deploy/argocd
+          bash generate-manifest.sh ${{ github.sha }}
+
+      - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks)
+        run: sleep 720s
+
+  run-e2e-tests:
+    needs: generate-manifest
+    uses: ./.github/workflows/modules/tests/argocd-tests.yml
+    with:
+      node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local
+      node-port: 9933
+    secrets: inherit
+
+  teardown:
+    runs-on: ubuntu-latest
+    needs: [build-and-push, generate-manifest, run-e2e-tests]
+    if: always() && needs.generate-manifest.result == 'success'
+    steps:
+      - name: Checkout ArgoCD Repository
+        uses: actions/checkout@v4
+        with:
+          repository: input-output-hk/sidechains-argocd
+          token: ${{ secrets.ACTIONS_PAT }}
+          path: sidechains-argocd
+
+      - name: Delete Ephemeral Environment Files
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.ACTIONS_PAT }}
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const directory = 'sidechains-argocd/integration-testing';
+            const files = fs.readdirSync(directory);
+
+            for (const file of files) {
+              if (file.startsWith('manifest-sha-')) {
+                console.log(`Deleting file: ${file}`);
+
+                // Fetch the SHA of the file
+                const shaResponse = await github.rest.repos.getContent({
+                  owner: 'input-output-hk',
+                  repo: 'sidechains-argocd',
+                  path: `integration-testing/${file}`,
+                });
+                const sha = shaResponse.data.sha;
+
+                // GitHub API request to delete the file
+                await github.rest.repos.deleteFile({
+                  owner: 'input-output-hk',
+                  repo: 'sidechains-argocd',
+                  path: `integration-testing/${file}`,
+                  message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`,
+                  sha: sha,
+                  branch: 'main'
+                });
+              }
+            }
+
+  chain-specs:
+    runs-on: [self-hosted, eks]
+    needs: [build-and-push]
+    if: github.ref_name == 'master' || inputs.upload == true
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - name: Install kubectl and awscli
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+          sudo apt update && sudo apt install -y awscli
+
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+
+      - name: Download chain spec artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: chain-specs
+          path: ./artifacts
+
+      - name: Update Kubernetes secret for devnet chain spec
+        run: |
+          TIMESTAMP=$(date +%Y%m%d%H%M)
+          SHA=${{ github.sha }}
+          kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc
+
+      - name: Update Kubernetes secret for staging chain spec
+        run: |
+          TIMESTAMP=$(date +%Y%m%d%H%M)
+          SHA=${{ github.sha }}
+          kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging

.github/workflows/nix.yml

@@ -0,0 +1,46 @@
+name: "Nix Devshells CI"
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+  repository_dispatch:
+    types: [test-run-pr]
+
+env:
+  AWS_DEFAULT_REGION: eu-central-1
+
+jobs:
+  build-and-test:
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        os: [nixos, macos]
+    runs-on:
+      - self-hosted
+      - ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ github.token }}
+      - name: Acquire AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+      - name: Add signing key for nix
+        run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key"
+      - name: Run nixci to build/test all outputs
+        run: |
+          nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs
+      - name: Copy nix scopes to nix cache
+        run: |
+          nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \
+          | nix copy --stdin --to \
+          "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key&region=$AWS_DEFAULT_REGION" \
+          && rm /tmp/outputs

.github/workflows/publish-doc.yml

@@ -0,0 +1,63 @@
+name: Publish Documentation
+
+on:
+
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch:
+    inputs:
+      rustdoc:
+        description: "Build and publish the rustdoc"
+        default: false
+        type: boolean
+
+env:
+  SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+jobs:
+  deploy-rustdoc:
+    if: github.event_name != 'workflow_dispatch' || inputs.rustdoc
+    name: Deploy Rust Docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install tooling
+        run: |
+          sudo apt-get install -y protobuf-compiler
+          protoc --version
+
+      - name: Checkout repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Add SSH key to read Substrate Repo
+        run: |
+          mkdir ~/.ssh
+          ssh-keyscan github.com >> ~/.ssh/known_hosts
+          ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+          ssh-add - <<< "${{ secrets.SUBSTRATE_REPO_SSH_KEY }}"
+
+      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
+      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
+      - name: Rust versions
+        run: rustup show
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # v2.6.2
+
+      - name: Build rustdocs
+        run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps
+
+      - name: Make index.html
+        run: echo "<meta http-equiv=refresh content=0;url=node_template/index.html>" > ./target/doc/index.html
+
+      - name: Deploy documentation
+        if: github.ref_name == 'master'
+        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: ./target/doc