ci.yml

name: CI

on:
  push:
    branches:
      - main
  pull_request:
  merge_group:
    types: [checks_requested]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  fmt:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - run: python -m pip install --upgrade pip && pip install nox
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt
      - name: Check python formatting and lints (ruff)
        run: nox -s ruff
      - name: Check rust formatting (rustfmt)
        run: nox -s rustfmt

  resolve:
    runs-on: ubuntu-latest
    outputs:
      MSRV: ${{ steps.resolve-msrv.outputs.MSRV }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: resolve MSRV
        id: resolve-msrv
        run:
          echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT

  semver-checks:
    if: github.ref != 'refs/heads/main'
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: obi1kenobi/cargo-semver-checks-action@v2

  check-msrv:
    needs: [fmt, resolve]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ needs.resolve.outputs.MSRV }}
          components: rust-src
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - run: python -m pip install --upgrade pip && pip install nox
      # This is a smoke test to confirm that CI will run on MSRV (including dev dependencies)
      - name: Check with MSRV package versions
        run: |
          nox -s set-msrv-package-versions
          nox -s check-all

    env:
      CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu

  clippy:
    needs: [fmt]
    runs-on: ${{ matrix.platform.os }}
    strategy:
      # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
      fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
      matrix:
        rust: [stable]
        platform: [
          {
            os: "macos-14",  # first available arm macos runner
            python-architecture: "arm64",
            rust-target: "aarch64-apple-darwin",
          },
          {
            os: "ubuntu-latest",
            python-architecture: "x64",
            rust-target: "x86_64-unknown-linux-gnu",
          },
          {
            os: "ubuntu-latest",
            python-architecture: "x64",
            rust-target: "powerpc64le-unknown-linux-gnu",
          },
          {
            os: "ubuntu-latest",
            python-architecture: "x64",
            rust-target: "s390x-unknown-linux-gnu",
          },
          {
            os: "ubuntu-latest",
            python-architecture: "x64",
            rust-target: "wasm32-wasi",
          },
          {
            os: "windows-latest",
            python-architecture: "x64",
            rust-target: "x86_64-pc-windows-msvc",
          },
          {
            os: "windows-latest",
            python-architecture: "x86",
            rust-target: "i686-pc-windows-msvc",
          },
        ]
        include:
          # Run beta clippy as a way to detect any incoming lints which may affect downstream users
          - rust: beta
            platform:
              {
                os: "ubuntu-latest",
                python-architecture: "x64",
                rust-target: "x86_64-unknown-linux-gnu",
              }
    name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }}
    continue-on-error: ${{ matrix.rust != 'stable' }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.rust }}
          targets: ${{ matrix.platform.rust-target }}
          components: clippy,rust-src
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          architecture: ${{ matrix.platform.python-architecture }}
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - run: python -m pip install --upgrade pip && pip install nox
      - run: nox -s clippy-all
    env:
      CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}

  build-pr:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }}
    name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
    needs: [fmt, resolve]
    uses: ./.github/workflows/build.yml
    with:
      os: ${{ matrix.platform.os }}
      python-version: ${{ matrix.python-version }}
      python-architecture: ${{ matrix.platform.python-architecture }}
      rust: ${{ matrix.rust }}
      rust-target: ${{ matrix.platform.rust-target }}
      MSRV: ${{ needs.resolve.outputs.MSRV }}
    secrets: inherit
    strategy:
      # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
      fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
      matrix:
        rust: [stable]
        python-version: ["3.12"]
        platform:
          [
            {
              os: "macos-14",  # first available arm macos runner
              python-architecture: "arm64",
              rust-target: "aarch64-apple-darwin",
            },
            {
              os: "macos-13",  # last available x86_64 macos runner
              python-architecture: "x64",
              rust-target: "x86_64-apple-darwin",
            },
            {
              os: "ubuntu-latest",
              python-architecture: "x64",
              rust-target: "x86_64-unknown-linux-gnu",
            },
            {
              os: "windows-latest",
              python-architecture: "x64",
              rust-target: "x86_64-pc-windows-msvc",
            },
            {
              os: "windows-latest",
              python-architecture: "x86",
              rust-target: "i686-pc-windows-msvc",
            },
          ]
        include:
          # Test nightly Rust on PRs so that PR authors have a chance to fix nightly
          # failures, as nightly does not block merge.
          - rust: nightly
            python-version: "3.12"
            platform:
              {
                os: "ubuntu-latest",
                python-architecture: "x64",
                rust-target: "x86_64-unknown-linux-gnu",
              }
  build-full:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
    needs: [fmt, resolve]
    uses: ./.github/workflows/build.yml
    with:
      os: ${{ matrix.platform.os }}
      python-version: ${{ matrix.python-version }}
      python-architecture: ${{ matrix.platform.python-architecture }}
      rust: ${{ matrix.rust }}
      rust-target: ${{ matrix.platform.rust-target }}
      MSRV: ${{ needs.resolve.outputs.MSRV }}
    secrets: inherit
    strategy:
      # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
      fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
      matrix:
        rust: [stable]
        python-version: [
          "3.7",
          "3.8",
          "3.9",
          "3.10",
          "3.11",
          "3.12",
          "3.13-dev",
          "pypy3.7",
          "pypy3.8",
          "pypy3.9",
          "pypy3.10",
          "graalpy24.0",
        ]
        platform:
          [
            # for the full matrix, use x86_64 macos runners because not all Python versions
            # PyO3 supports are available for arm on GitHub Actions. (Availability starts
            # around Python 3.10, can switch the full matrix to arm once earlier versions
            # are dropped.)
            # NB: if this switches to arm, switch the arm job below in the `include` to x86_64
            {
              os: "macos-13",
              python-architecture: "x64",
              rust-target: "x86_64-apple-darwin",
            },
            {
              os: "ubuntu-latest",
              python-architecture: "x64",
              rust-target: "x86_64-unknown-linux-gnu",
            },
            {
              os: "windows-latest",
              python-architecture: "x64",
              rust-target: "x86_64-pc-windows-msvc",
            },
          ]
        include:
          # Test minimal supported Rust version
          - rust: ${{ needs.resolve.outputs.MSRV }}
            python-version: "3.12"
            platform:
              {
                os: "ubuntu-latest",
                python-architecture: "x64",
                rust-target: "x86_64-unknown-linux-gnu",
              }

          # Test the `nightly` feature
          - rust: nightly
            python-version: "3.12"
            platform:
              {
                os: "ubuntu-latest",
                python-architecture: "x64",
                rust-target: "x86_64-unknown-linux-gnu",
              }

          # Run rust beta to help catch toolchain regressions
          - rust: beta
            python-version: "3.12"
            platform:
              {
                os: "ubuntu-latest",
                python-architecture: "x64",
                rust-target: "x86_64-unknown-linux-gnu",
              }

          # Test 32-bit Windows only with the latest Python version
          - rust: stable
            python-version: "3.12"
            platform:
              {
                os: "windows-latest",
                python-architecture: "x86",
                rust-target: "i686-pc-windows-msvc",
              }

          # test arm macos runner with the latest Python version
          # NB: if the full matrix switchess to arm, switch to x86_64 here
          - rust: stable
            python-version: "3.12"
            platform:
              {
                os: "macos-14",
                python-architecture: "arm64",
                rust-target: "aarch64-apple-darwin",
              }

  valgrind:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          # FIXME valgrind detects an issue with Python 3.12.5, needs investigation
          # whether it's a PyO3 issue or upstream CPython.
          python-version: '3.12.4'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@stable
      - uses: taiki-e/install-action@valgrind
      - run: python -m pip install --upgrade pip && pip install nox
      - run: nox -s test-rust -- release skip-full
    env:
      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: valgrind --leak-check=no --error-exitcode=1
      RUST_BACKTRACE: 1
      TRYBUILD: overwrite

  careful:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@nightly
        with:
          components: rust-src
      - uses: taiki-e/install-action@cargo-careful
      - run: python -m pip install --upgrade pip && pip install nox
      - run: nox -s test-rust -- careful skip-full
    env:
      RUST_BACKTRACE: 1
      TRYBUILD: overwrite

  docsrs:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@nightly
        with:
          components: rust-src
      - run: cargo rustdoc --lib --no-default-features --features full -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]"

  coverage:
    if : ${{ github.event_name != 'merge_group' }}
    needs: [fmt]
    name: coverage ${{ matrix.os }}
    strategy:
      matrix:
        os: ["windows-latest", "macos-14", "ubuntu-latest"]  # first available arm macos runner
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          # Use the PR head, not the merge commit, because the head commit (and the base)
          # is what codecov uses to calculate diffs.
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: llvm-tools-preview,rust-src
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - run: python -m pip install --upgrade pip && pip install nox
      - run: nox -s coverage
      - uses: codecov/codecov-action@v4
        with:
          file: coverage.json
          name: ${{ matrix.os }}
          token: ${{ secrets.CODECOV_TOKEN }}

  emscripten:
    name: emscripten
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          # TODO bump emscripten builds to test on 3.12
          python-version: 3.11
        id: setup-python
      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: wasm32-unknown-emscripten
          components: rust-src
      - uses: actions/setup-node@v4
        with:
          node-version: 14
      - run: python -m pip install --upgrade pip && pip install nox
      - uses: actions/cache@v4
        id: cache
        with:
          path: |
            .nox/emscripten
          key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - name: Build
        if: steps.cache.outputs.cache-hit != 'true'
        run: nox -s build-emscripten
      - name: Test
        run: nox -s test-emscripten

  test-debug:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rust-src
      - name: Install python3 standalone debug build with nox
        run: |
          PBS_RELEASE="20231002"
          PBS_PYTHON_VERSION="3.12.0"
          PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst"
          wget "https://github.com/indygreg/python-build-standalone/releases/download/${PBS_RELEASE}/${PBS_ARCHIVE}"
          tar -I zstd -xf "${PBS_ARCHIVE}"
          ls -l $(pwd)/python/install/bin
          ls -l $(pwd)/python/install/lib
          echo PATH=$(pwd)/python/install/bin:$PATH >> $GITHUB_ENV
          echo LD_LIBRARY_PATH=$(pwd)/python/install/lib:$LD_LIBRARY_PATH >> $GITHUB_ENV
          echo PYTHONHOME=$(pwd)/python/install >> $GITHUB_ENV
          echo PYO3_PYTHON=$(pwd)/python/install/bin/python3 >> $GITHUB_ENV
      - run: python3 -m sysconfig
      - run: python3 -m pip install --upgrade pip && pip install nox
      - run: |
          PYO3_CONFIG_FILE=$(mktemp)
          cat > $PYO3_CONFIG_FILE << EOF
          implementation=CPython
          version=3.12
          shared=true
          abi3=false
          lib_name=python3.12d
          lib_dir=${{ github.workspace }}/python/install/lib
          executable=${{ github.workspace }}/python/install/bin/python3
          pointer_width=64
          build_flags=Py_DEBUG,Py_REF_DEBUG
          suppress_build_script_link_lines=false
          EOF
          echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV
      - run: python3 -m nox -s test

  test-free-threaded:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    needs: [fmt]
    runs-on: ubuntu-latest
    env:
      UNSAFE_PYO3_BUILD_FREE_THREADED: 1
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rust-src
      # TODO: replace with setup-python when there is support
      - uses: deadsnakes/action@v3.2.0
        with:
          python-version: '3.13-dev'
          nogil: true
      - run: python3 -m sysconfig
      - run: python3 -m pip install --upgrade pip && pip install nox
      - run: nox -s ffi-check
      - name: Run default nox sessions that should pass
        run: nox -s clippy docs rustfmt ruff
      - name: Run PyO3 tests with free-threaded Python (can fail)
        # TODO fix the test crashes so we can unset this
        continue-on-error: true
        run: nox -s test

  test-version-limits:
    needs: [fmt]
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@stable
      - run: python3 -m pip install --upgrade pip && pip install nox
      - run: python3 -m nox -s test-version-limits

  check-feature-powerset:
    needs: [fmt, resolve]
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    runs-on: ubuntu-latest
    name: check-feature-powerset ${{ matrix.rust }}
    strategy:
      # run on stable and MSRV to check that all combinations of features are expected to build fine on our supported
      # range of compilers
      matrix:
        rust: ["stable"]
        include:
          - rust: ${{ needs.resolve.outputs.MSRV }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
      - uses: taiki-e/install-action@v2
        with:
          tool: cargo-hack,cargo-minimal-versions
      - run: python3 -m pip install --upgrade pip && pip install nox
      - run: python3 -m nox -s check-feature-powerset -- ${{ matrix.rust != 'stable' && 'minimal-versions' || '' }}

  test-cross-compilation:
    needs: [fmt]
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    runs-on: ${{ matrix.os }}
    name: test-cross-compilation ${{ matrix.os }} -> ${{ matrix.target }}
    strategy:
      # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
      fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
      matrix:
        include:
          # ubuntu "cross compile" to itself
          - os: "ubuntu-latest"
            target: "x86_64-unknown-linux-gnu"
            flags: "-i python3.12"
            manylinux: auto
          # ubuntu x86_64 -> aarch64
          - os: "ubuntu-latest"
            target: "aarch64-unknown-linux-gnu"
            flags: "-i python3.12"
            manylinux: auto
          # ubuntu x86_64 -> windows x86_64
          - os: "ubuntu-latest"
            target: "x86_64-pc-windows-gnu"
            flags: "-i python3.12 --features abi3 --features generate-import-lib"
            manylinux: off
          # macos x86_64 -> aarch64
          - os: "macos-13"  # last x86_64 macos runners
            target: "aarch64-apple-darwin"
          # macos aarch64 -> x86_64
          - os: "macos-14"  # aarch64 macos runners
            target: "x86_64-apple-darwin"
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces:
            examples/maturin-starter
          save-if: ${{ github.event_name != 'merge_group' }}
          key: ${{ matrix.target }}
      - name: Setup cross-compiler
        if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }}
        run: sudo apt-get install -y mingw-w64 llvm
      - uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux }}
          args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }}

  test-cross-compilation-windows:
    needs: [fmt]
    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces:
            examples/maturin-starter
          save-if: ${{ github.event_name != 'merge_group' }}
      - uses: actions/cache/restore@v4
        with:
          # https://github.com/PyO3/maturin/discussions/1953
          path: ~/.cache/cargo-xwin
          key: cargo-xwin-cache
      - name: Test cross compile to Windows
        env:
          XWIN_ARCH: x86_64
        run: |
          set -ex
          sudo apt-get install -y mingw-w64 llvm
          rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc
          pip install cargo-xwin
          # abi3
          cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu
          cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc
          # non-abi3
          export PYO3_CROSS_PYTHON_VERSION=3.12
          cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu
          cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc
      - if: ${{ github.ref == 'refs/heads/main' }}
        uses: actions/cache/save@v4
        with:
          path: ~/.cache/cargo-xwin
          key: cargo-xwin-cache
  conclusion:
    needs:
      - fmt
      - check-msrv
      - clippy
      - build-pr
      - build-full
      - valgrind
      - careful
      - docsrs
      - coverage
      - emscripten
      - test-debug
      - test-free-threaded
      - test-version-limits
      - check-feature-powerset
      - test-cross-compilation
      - test-cross-compilation-windows
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Result
        run: |
          jq -C <<< "${needs}"
          # Check if all needs were successful or skipped.
          "$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")"
        env:
          needs: ${{ toJson(needs) }}