diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000000..ba84099593b26 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,136 @@ +name: Lint +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/lint.yml + + - name: Rust related changes + id: rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + Cargo.** + crates/** + shim/** + xtask/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Formatting related changes + id: format + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/*.{yml,yaml,md,mdx,js,jsx,ts,tsx,json,toml,css} + + outputs: + rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} + format: ${{ steps.ci.outputs.diff != '' || steps.format.outputs.diff != '' }} + + rust_lint: + needs: [determine_jobs] + if: needs.determine_jobs.outputs.rust == 'true' + name: Rust lints + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo fmt check + run: | + cargo fmt --check + + - name: Check Cargo.toml formatting (taplo) + run: npx @taplo/cli@0.5.2 format --check + + - name: Check licenses + uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check licenses + + format_lint: + name: Formatting + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + needs: determine_jobs + if: needs.determine_jobs.outputs.format == 'true' + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: "Setup Node" + uses: ./.github/actions/setup-node + with: + extra-flags: --no-optional + node-version: "20" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Lint + # Filters some packages out, but not sure why + run: | + turbo run lint \ + --filter=!@vercel/devlow-bench \ + --filter=!@vercel/experimental-nft-next-plugin \ + --filter=!@vercel/experimental-nft-next-plugin \ + --filter=!turbopack-bump-action \ + --filter=!next-integration-stat \ + --env-mode=strict + + cleanup: + name: Cleanup + needs: + - rust_lint + - format_lint + if: always() + uses: ./.github/workflows/pr-clean-caches.yml + secrets: inherit diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml new file mode 100644 index 0000000000000..0f0b68dd31251 --- /dev/null +++ b/.github/workflows/test-js-packages.yml @@ -0,0 +1,103 @@ +name: JS Package Tests +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/test-js-packages.yml + + - name: /packages related changes + id: packages + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + packages/** + + - name: Docs related changes + id: docs + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + docs/** + + outputs: + ci: ${{ steps.ci.outputs.diff != ''}} + packages: ${{ steps.packages.outputs.diff != '' }} + docs: ${{ steps.docs.outputs.diff != '' }} + + js_packages: + name: JS Package Tests + timeout-minutes: 30 + if: needs.determine_jobs.outputs.ci == 'true' || needs.determine_jobs.outputs.packages == 'true' || needs.determine_jobs.outputs.docs == 'true' + needs: [determine_jobs] + runs-on: ${{ matrix.os.runner }} + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + - name: macos + runner: macos-12 + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + # on main -> current + prev commit + # pr -> pr commits + base commit + - name: Determine fetch depth + id: fetch-depth + run: | + echo "depth=$(( ${{ github.event.pull_request.commits || 1 }} + 1 ))" >> $GITHUB_OUTPUT + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + fetch-depth: ${{ steps.fetch-depth.outputs.depth }} + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + node-version: "20" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Run tests + # We manually set TURBO_API to an empty string to override Hetzner env + # We filter out turborepo-repository because it's a native package and needs + # to run when turbo core changes. This job (`js_packages`) does not run on turborpeo core + # changes, and we don't want to enable that beahvior for _all_ our JS packages. + run: | + TURBO_API= turbo run check-types test --filter=docs --filter="!turborepo-repository" --filter={./packages/*}...[${{ github.event.pull_request.base.sha || 'HEAD^1' }}] --color --env-mode=strict diff --git a/.github/workflows/test.yml b/.github/workflows/turbopack-test.yml similarity index 59% rename from .github/workflows/test.yml rename to .github/workflows/turbopack-test.yml index 2274e9a53aa1d..b5313e8d62acc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/turbopack-test.yml @@ -1,4 +1,4 @@ -name: Test +name: Turbopack Test on: push: branches: [main] @@ -55,7 +55,9 @@ jobs: with: PATTERNS: | .github/actions/** - .github/workflows/test.yml + .github/workflows/turbopack-test.yml + .github/workflows/test-turbopack-rust-bench-test.yml + .github/workflows/pr-clean-caches.yml - name: Root cargo related changes id: cargo @@ -65,13 +67,6 @@ jobs: Cargo.* rust-toolchain - - name: Turborepo version changes - id: turborepo_version - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - version.txt - - name: Rust related changes id: rust uses: technote-space/get-diff-action@v6 @@ -131,39 +126,6 @@ jobs: !**.md !**.mdx - - name: Turborepo integration tests changes - id: turborepo_integration - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - turborepo-tests/integration/** - turborepo-tests/helpers/** - - - name: Examples related changes - id: examples - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - examples/** - turborepo-tests/example-*/** - turborepo-tests/helpers/** - !**.md - !**.mdx - - - name: Turborepo JS Package related changes - id: turborepo_js - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - packages/** - - - name: Docs related changes - id: docs - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - docs/** - - name: Formatting related changes id: format uses: technote-space/get-diff-action@v6 @@ -172,227 +134,13 @@ jobs: **/*.{yml,yaml,md,mdx,js,jsx,ts,tsx,json,toml,css} outputs: + turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }} rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} - cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '' && steps.turborepo_rust.outputs.diff == '') }} + turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }} # We only test workspace dependency changes on main, not on PRs to speed up CI cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} - turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }} - turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }} - turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }} + cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '' && steps.turborepo_rust.outputs.diff == '') }} turbopack_bench: ${{ steps.ci.outputs.diff != '' || steps.turbopack_bench.outputs.diff != '' }} - turborepo_build: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != ''}} - turborepo_integration: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != '' }} - examples: ${{ steps.ci.outputs.diff != '' || steps.examples.outputs.diff != '' || steps.turborepo_version.outputs.diff != '' }} - turborepo_js: ${{ steps.ci.outputs.diff != '' || steps.turborepo_js.outputs.diff != '' }} - docs: ${{ steps.ci.outputs.diff != '' || steps.docs.outputs.diff != '' }} - format: ${{ steps.ci.outputs.diff != '' || steps.format.outputs.diff != '' }} - push: ${{ steps.ci.outputs.diff != '' || github.event_name == 'push' }} - tag: ${{ steps.ci.outputs.diff != '' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/node-file-trace')) }} - main_push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - tag_push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/node-file-trace') }} - - build_turborepo: - name: Build Turborepo - needs: determine_jobs - if: needs.determine_jobs.outputs.turborepo_build == 'true' - runs-on: ${{ matrix.os.runner }} - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - - name: macos - runner: macos-latest - - name: windows - runner: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues - - run: | - if [ -z "${RUSTC_WRAPPER}" ]; then - unset RUSTC_WRAPPER - fi - turbo run build --filter=cli --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} - shell: bash - env: - SCCACHE_BUCKET: turborepo-sccache - SCCACHE_REGION: us-east-2 - # Only use sccache if we're in the Vercel repo. - RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }} - CARGO_INCREMENTAL: 0 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Run sccache stat for check - shell: bash - run: ${SCCACHE_PATH} --show-stats - if: ${{ !github.event.pull_request.head.repo.fork }} - - turborepo_integration: - name: Turborepo Integration - needs: [determine_jobs, build_turborepo] - if: needs.determine_jobs.outputs.turborepo_integration == 'true' - runs-on: ${{ matrix.os.runner }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: - - runner: ubuntu-latest - - runner: macos-latest - - runner: windows-latest - steps: - # On Windows, set autocrlf to input so that when the repo is cloned down - # the fixtures retain their line endings and don't get updated to CRLF. - # We want this because this repo also contains the fixtures for our test cases - # and these fixtures have files that need stable file hashes. If we let git update - # the line endings on checkout, the file hashes will change. - # https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf - - name: set crlf - if: matrix.os.runner == 'windows-latest' - shell: bash - run: git config --global core.autocrlf input - - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@v1 - with: - macos-skip-brew-update: "true" - env: - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true - - - name: Cache Prysk - id: cache-prysk - uses: actions/cache@v3 - with: - path: cli/.cram_env - key: prysk-venv-${{ matrix.os.runner }} - - - name: Integration Tests - run: turbo run test --filter=turborepo-tests-integration --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} - - turborepo_examples: - name: Turborepo Examples - needs: [determine_jobs] - if: needs.determine_jobs.outputs.examples == 'true' - timeout-minutes: 40 - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - # Disable corepack. actions/setup-node invokes other package managers and - # that causes corepack to throw an error, so we disable it first. - - name: Disable corepack - shell: bash - run: corepack disable - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Check examples - shell: bash - # Concurrency being 1 is a big hammer here. - # It's a quick fix for non-deterministic behaviors we're seeing around package resolution. - # We could likely do some hacking to reparallelize this, but it's not worth it right now. - run: turbo run test --filter="@turborepo-examples-tests/*" --continue --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --env-mode=strict --concurrency=1 - - # Disable corepack again. actions/setup-node's "Post" step runs at the end of - # this job invokes other package managers, and corepack throws an error. - # (corepack was enabled from inside the tests above). - - name: Disable corepack again - shell: bash - run: corepack disable - - js_packages: - name: JS Package Tests - timeout-minutes: 30 - if: needs.determine_jobs.outputs.turborepo_js == 'true' || needs.determine_jobs.outputs.docs == 'true' - needs: [determine_jobs] - runs-on: ${{ matrix.os.runner }} - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - - name: macos - runner: macos-latest - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - - steps: - # on main -> current + prev commit - # pr -> pr commits + base commit - - name: Determine fetch depth - id: fetch-depth - run: | - echo "depth=$(( ${{ github.event.pull_request.commits || 1 }} + 1 ))" >> $GITHUB_OUTPUT - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - fetch-depth: ${{ steps.fetch-depth.outputs.depth }} - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - node-version: "20" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Run tests - # We manually set TURBO_API to an empty string to override Hetzner env - # We filter out turborepo-repository because it's a native package and needs - # to run when turbo core changes. This job (`js_packages`) does not run on turborpeo core - # changes, and we don't want to enable that beahvior for _all_ our JS packages. - run: | - TURBO_API= turbo run check-types test --filter=docs --filter="!turborepo-repository" --filter={./packages/*}...[${{ github.event.pull_request.base.sha || 'HEAD^1' }}] --color --env-mode=strict turbopack_typescript: name: Turbopack TypeScript files @@ -425,87 +173,6 @@ jobs: working-directory: crates/turbopack-ecmascript-runtime/js run: pnpm run check - rust_lint: - needs: [determine_jobs] - if: needs.determine_jobs.outputs.rust == 'true' - name: Rust lints - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Rust - uses: ./.github/actions/setup-rust - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo fmt check - run: | - cargo fmt --check - - - name: Check Cargo.toml formatting (taplo) - run: npx @taplo/cli@0.5.2 format --check - - - name: Check licenses - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check licenses - - turborepo_rust_check: - needs: [determine_jobs] - # We test dependency changes only on main - if: | - (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turborepo_rust == 'true') || - needs.determine_jobs.outputs.cargo_on_main == 'true' || - needs.determine_jobs.outputs.cargo_only == 'true' - name: Turborepo rust check - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo check - run: | - cargo groups check turborepo-libraries --features rustls-tls - - turborepo_rust_clippy: - needs: [turborepo_rust_check] - name: Turborepo rust clippy - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo clippy - run: | - cargo groups clippy turborepo-libraries --features rustls-tls -- --deny clippy::all - - - name: Run ast-grep lints - run: | - npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turborepo-libraries | awk '{ print $2 }' | tr '\n' ' ') - turbopack_rust_check: needs: [determine_jobs] # We test dependency changes only on main @@ -639,48 +306,6 @@ jobs: filePath: ./out.log comment_tag: check_next_swc_turbopack - turborepo_rust_test: - needs: [turborepo_rust_check] - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - nextest: linux - - name: macos - runner: macos-latest - nextest: mac - - name: windows - runner: windows-latest - nextest: windows-tar - runs-on: ${{ matrix.os.runner }} - name: Turborepo Rust testing on ${{ matrix.os.name }} - steps: - - name: Set git to use LF line endings - run: | - git config --global core.autocrlf false - git config --global core.eol lf - if: matrix.os.name == 'windows' - - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run tests - timeout-minutes: 120 - run: | - cargo groups test turborepo-libraries - turbopack_rust_test1: needs: [turbopack_rust_check] runs-on: @@ -844,53 +469,15 @@ jobs: os: ${{ matrix.os.name }} all: ${{ needs.determine_jobs.outputs.turbopack_bench == 'true' }} - format_lint: - name: Formatting - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - needs: determine_jobs - if: needs.determine_jobs.outputs.format == 'true' - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Lint - # Filters some workspaces out: - # - Other `@vercel/*` packages because ?? - run: turbo run lint --filter=!@vercel/devlow-bench --filter=!@vercel/experimental-nft-next-plugin --filter=!@vercel/experimental-nft-next-plugin --filter=!turbopack-bump-action --filter=!next-integration-stat --env-mode=strict - final: name: Ok needs: - determine_jobs - - turborepo_examples - - turborepo_integration - - js_packages - - rust_lint - - turborepo_rust_check - - turborepo_rust_clippy - turbopack_rust_check - turbopack_rust_clippy - turbopack_rust_test1 - turbopack_rust_test_bench1 - turbopack_typescript - - turborepo_rust_test - - format_lint if: always() permissions: contents: read @@ -914,19 +501,11 @@ jobs: fi } subjob ${{needs.determine_jobs.result}} "Determining jobs" - subjob ${{needs.turborepo_examples.result}} "Turborepo examples" - subjob ${{needs.turborepo_integration.result}} "Turborepo integration tests" - subjob ${{needs.js_packages.result}} "JS Package tests" - subjob ${{needs.rust_lint.result}} "Rust lints" - subjob ${{needs.turborepo_rust_check.result}} "Turborepo Rust checks" - subjob ${{needs.turborepo_rust_clippy.result}} "Turborepo Rust clippy" subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" - subjob ${{needs.turborepo_rust_test.result}} "Turborepo Rust tests" - subjob ${{needs.format_lint.result}} "Formatting" if [ "$cancelled" = "true" ]; then echo "cancelled=true" >> $GITHUB_OUTPUT elif [ "$failure" = "true" ]; then @@ -1026,11 +605,6 @@ jobs: needs: - final - determine_jobs - - turborepo_examples - - turborepo_integration - - rust_lint - - turborepo_rust_check - - turborepo_rust_clippy - turbopack_rust_check - turbopack_rust_clippy - turbopack_rust_test1 @@ -1038,8 +612,6 @@ jobs: - turbopack_rust_test_bench1 - turbopack_rust_test_bench2 - turbopack_typescript - - turborepo_rust_test - - format_lint if: always() permissions: contents: read @@ -1064,11 +636,6 @@ jobs: fi } subjob ${{needs.determine_jobs.result}} "Determining jobs" - subjob ${{needs.turborepo_examples.result}} "Turborepo examples" - subjob ${{needs.turborepo_integration.result}} "Turborepo integration tests" - subjob ${{needs.rust_lint.result}} "Rust lints" - subjob ${{needs.turborepo_rust_check.result}} "Turborepo Rust checks" - subjob ${{needs.turborepo_rust_clippy.result}} "Turborepo Rust clippy" subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" @@ -1076,8 +643,6 @@ jobs: subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" subjob ${{needs.turbopack_rust_test_bench2.result}} "Turbopack Rust benchmark tests (mac/win, non-blocking)" subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" - subjob ${{needs.turborepo_rust_test.result}} "TurboRepo Rust tests" - subjob ${{needs.format_lint.result}} "Formatting" if [ "$cancelled" = "true" ]; then echo "cancelled=true" >> $GITHUB_OUTPUT elif [ "$failure" = "true" ]; then diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml new file mode 100644 index 0000000000000..9317e74322e57 --- /dev/null +++ b/.github/workflows/turborepo-test.yml @@ -0,0 +1,393 @@ +name: Turborepo Test +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/turborepo-test.yml + + - name: Root cargo related changes + id: cargo + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + Cargo.* + rust-toolchain + + - name: Turborepo version changes + id: turborepo_version + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + version.txt + + - name: Rust related changes + id: rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + Cargo.** + crates/** + shim/** + xtask/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Turborepo Rust related changes + id: turborepo_rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + crates/turborepo*/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Turborepo integration tests changes + id: turborepo_integration + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + turborepo-tests/integration/** + turborepo-tests/helpers/** + + - name: Examples related changes + id: examples + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + examples/** + turborepo-tests/example-*/** + turborepo-tests/helpers/** + !**.md + !**.mdx + + outputs: + rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} + cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turborepo_rust.outputs.diff == '') }} + # We only test workspace dependency changes on main, not on PRs to speed up CI + cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} + turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }} + turborepo_build: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != ''}} + turborepo_integration: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != '' }} + examples: ${{ steps.ci.outputs.diff != '' || steps.examples.outputs.diff != '' || steps.turborepo_version.outputs.diff != '' }} + + build_turborepo: + name: Build Turborepo + needs: determine_jobs + if: needs.determine_jobs.outputs.turborepo_build == 'true' + runs-on: ${{ matrix.os.runner }} + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + - name: macos + runner: macos-12 + - name: windows + runner: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues + - run: | + if [ -z "${RUSTC_WRAPPER}" ]; then + unset RUSTC_WRAPPER + fi + turbo run build --filter=cli --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} + shell: bash + env: + SCCACHE_BUCKET: turborepo-sccache + SCCACHE_REGION: us-east-2 + # Only use sccache if we're in the Vercel repo. + RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }} + CARGO_INCREMENTAL: 0 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Run sccache stat for check + shell: bash + run: ${SCCACHE_PATH} --show-stats + if: ${{ !github.event.pull_request.head.repo.fork }} + + integration: + name: Turborepo Integration + needs: [determine_jobs, build_turborepo] + if: needs.determine_jobs.outputs.turborepo_integration == 'true' + runs-on: ${{ matrix.os.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - runner: ubuntu-latest + - runner: macos-12 + - runner: windows-latest + steps: + # On Windows, set autocrlf to input so that when the repo is cloned down + # the fixtures retain their line endings and don't get updated to CRLF. + # We want this because this repo also contains the fixtures for our test cases + # and these fixtures have files that need stable file hashes. If we let git update + # the line endings on checkout, the file hashes will change. + # https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf + - name: set crlf + if: matrix.os.runner == 'windows-latest' + shell: bash + run: git config --global core.autocrlf input + - uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v1 + with: + macos-skip-brew-update: "true" + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true + + - name: Cache Prysk + id: cache-prysk + uses: actions/cache@v3 + with: + path: cli/.cram_env + key: prysk-venv-${{ matrix.os.runner }} + + - name: Integration Tests + run: turbo run test --filter=turborepo-tests-integration --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} + + examples: + name: Turborepo Examples + needs: [determine_jobs] + if: needs.determine_jobs.outputs.examples == 'true' + timeout-minutes: 40 + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Disable corepack. actions/setup-node invokes other package managers and + # that causes corepack to throw an error, so we disable it first. + - name: Disable corepack + shell: bash + run: corepack disable + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Check examples + shell: bash + # Concurrency being 1 is a big hammer here. + # It's a quick fix for non-deterministic behaviors we're seeing around package resolution. + # We could likely do some hacking to reparallelize this, but it's not worth it right now. + run: turbo run test --filter="@turborepo-examples-tests/*" --continue --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --env-mode=strict --concurrency=1 + + # Disable corepack again. actions/setup-node's "Post" step runs at the end of + # this job invokes other package managers, and corepack throws an error. + # (corepack was enabled from inside the tests above). + - name: Disable corepack again + shell: bash + run: corepack disable + + rust_lint: + needs: [determine_jobs] + if: needs.determine_jobs.outputs.rust == 'true' + name: Rust lints + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo fmt check + run: | + cargo fmt --check + + - name: Check Cargo.toml formatting (taplo) + run: npx @taplo/cli@0.5.2 format --check + + - name: Check licenses + uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check licenses + + rust_check: + needs: [determine_jobs] + # We test dependency changes only on main + if: | + (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turborepo_rust == 'true') || + needs.determine_jobs.outputs.cargo_on_main == 'true' || + needs.determine_jobs.outputs.cargo_only == 'true' + name: Turborepo rust check + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo check + run: | + cargo groups check turborepo-libraries --features rustls-tls + + rust_clippy: + needs: [rust_check] + name: Turborepo rust clippy + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo clippy + run: | + cargo groups clippy turborepo-libraries --features rustls-tls -- --deny clippy::all + + - name: Run ast-grep lints + run: | + npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turborepo-libraries | awk '{ print $2 }' | tr '\n' ' ') + + rust_test: + needs: [rust_check] + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + nextest: linux + - name: macos + runner: macos-12 + nextest: mac + - name: windows + runner: windows-latest + nextest: windows-tar + runs-on: ${{ matrix.os.runner }} + name: Turborepo Rust testing on ${{ matrix.os.name }} + steps: + - name: Set git to use LF line endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + if: matrix.os.name == 'windows' + + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run tests + timeout-minutes: 120 + run: | + cargo groups test turborepo-libraries + + cleanup: + name: Cleanup + needs: + - determine_jobs + - build_turborepo + - integration + - examples + - rust_lint + - rust_check + - rust_clippy + - rust_test + if: always() + uses: ./.github/workflows/pr-clean-caches.yml + secrets: inherit