Build Matrix of Binaries #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build Matrix of Binaries | |
| 'on': | |
| push: | |
| tags: | |
| - "v[0-9]+.[0-9]+.[0-9]*" | |
| branches: | |
| - "build-all-*" | |
| - "build-bins-*" | |
| # also run once a month | |
| schedule: | |
| - cron: "05 00 01 * *" | |
| # Run manually | |
| workflow_dispatch: | |
| env: | |
| TS_FILENAME: "tari_pre_mine_generation" | |
| TS_BUNDLE_ID_BASE: "com.tarilabs" | |
| TS_SIG_FN: "sha256-unsigned.txt" | |
| # For debug builds | |
| # TS_BUILD: "debug" | |
| TS_BUILD: "release" | |
| toolchain: nightly-2024-07-07 | |
| matrix-json-file: ".github/workflows/build_binaries.json" | |
| CARGO_HTTP_MULTIPLEXING: false | |
| CARGO_UNSTABLE_SPARSE_REGISTRY: true | |
| CARGO: cargo | |
| # CARGO_OPTIONS: "--locked" | |
| CARGO_OPTIONS: "" | |
| CARGO_CACHE: true | |
| concurrency: | |
| # https://docs.github.com/en/actions/examples/using-concurrency-expressions-and-a-test-matrix | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' | |
| cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') || github.ref != 'refs/heads/main' || github.ref != 'refs/heads/nextnet' || github.ref != 'refs/heads/stagenet' }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| matrix-prep: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set Matrix | |
| id: set-matrix | |
| run: | | |
| # | |
| # build all targets images | |
| # matrix=$( jq -s -c .[] .github/workflows/build_binaries.json ) | |
| # | |
| # build only single target image | |
| # matrix_selection=$( jq -c '.[] | select( ."name" == "windows-x64" )' ${{ env.matrix-json-file }} ) | |
| # matrix_selection=$( jq -c '.[] | select( ."name" | contains("macos") )' ${{ env.matrix-json-file }} ) | |
| # | |
| # build select target images - build_enabled | |
| matrix_selection=$( jq -c '.[] | select( ."build_enabled" != false )' ${{ env.matrix-json-file }} ) | |
| # | |
| # Setup the json build matrix | |
| matrix=$(echo ${matrix_selection} | jq -s -c '{"builds": .}') | |
| echo $matrix | |
| echo $matrix | jq . | |
| echo "matrix=${matrix}" >> $GITHUB_OUTPUT | |
| matrix-check: | |
| # Debug matrix | |
| if: ${{ false }} | |
| runs-on: ubuntu-latest | |
| needs: matrix-prep | |
| steps: | |
| - name: Install json2yaml | |
| run: | | |
| sudo npm install -g json2yaml | |
| - name: Check matrix definition | |
| run: | | |
| matrix='${{ needs.matrix-prep.outputs.matrix }}' | |
| echo $matrix | |
| echo $matrix | jq . | |
| echo $matrix | json2yaml | |
| builds: | |
| name: Building ${{ matrix.builds.name }} on ${{ matrix.builds.runs-on }} | |
| needs: matrix-prep | |
| continue-on-error: ${{ matrix.builds.best_effort || false }} | |
| outputs: | |
| TS_VERSION: ${{ steps.set-tari-vars.outputs.TS_VERSION }} | |
| VSHA_SHORT: ${{ steps.set-tari-vars.outputs.VSHA_SHORT }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix-prep.outputs.matrix) }} | |
| runs-on: ${{ matrix.builds.runs-on }} | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Declare Global Variables 4 GHA ${{ github.event_name }} | |
| id: set-tari-vars | |
| shell: bash | |
| run: | | |
| echo "VBRANCH=${{ github.ref_name }}" >> $GITHUB_ENV | |
| VSHA_SHORT=$(git rev-parse --short HEAD) | |
| echo "VSHA_SHORT=${VSHA_SHORT}" >> $GITHUB_ENV | |
| echo "VSHA_SHORT=${VSHA_SHORT}" >> $GITHUB_OUTPUT | |
| TS_VERSION=$(awk -F ' = ' '$1 ~ /^version/ \ | |
| { gsub(/["]/, "", $2); printf("%s",$2) }' \ | |
| "$GITHUB_WORKSPACE/Cargo.toml") | |
| echo "TS_VERSION=${TARI_VERSION}" >> $GITHUB_ENV | |
| echo "TS_VERSION=${TARI_VERSION}" >> $GITHUB_OUTPUT | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| components: rustfmt, clippy | |
| toolchain: ${{ matrix.builds.rust }} | |
| targets: ${{ matrix.builds.target }} | |
| - name: Install Linux dependencies - Ubuntu | |
| if: ${{ startsWith(runner.os,'Linux') && ( ! matrix.builds.cross ) }} | |
| run: | | |
| sudo apt-get update | |
| #sudo bash scripts/install_ubuntu_dependencies.sh | |
| sudo apt-get install --no-install-recommends --assume-yes \ | |
| libprotobuf-dev \ | |
| protobuf-compiler \ | |
| libudev-dev \ | |
| libhidapi-dev | |
| - name: Install Linux dependencies - Ubuntu - cross-compiled ${{ env.TARI_BUILD_ISA_CPU }} on x86-64 | |
| if: ${{ false }} | |
| # if: ${{ startsWith(runner.os,'Linux') && ( ! matrix.builds.cross ) && matrix.builds.name != 'linux-x86_64' }} | |
| run: | | |
| sudo apt-get update | |
| sudo bash scripts/install_ubuntu_dependencies-cross_compile.sh ${{ env.TARI_BUILD_ISA_CPU }} | |
| rustup target add ${{ matrix.builds.target }} | |
| echo "PKG_CONFIG_SYSROOT_DIR=/usr/${{ env.TARI_BUILD_ISA_CPU }}-linux-gnu/" >> $GITHUB_ENV | |
| - name: Install macOS dependencies | |
| if: startsWith(runner.os,'macOS') | |
| run: | | |
| # openssl, cmake and autoconf already installed | |
| brew install zip coreutils automake protobuf | |
| rustup target add ${{ matrix.builds.target }} | |
| - name: Install macOS-14 missing dependencies - hack | |
| if: ${{ startsWith(runner.os,'macOS') && startsWith(runner.arch,'ARM64') }} | |
| run: | | |
| brew install libtool | |
| - name: Install Windows dependencies | |
| if: startsWith(runner.os,'Windows') | |
| run: | | |
| vcpkg.exe install sqlite3:x64-windows zlib:x64-windows | |
| # Bug in choco - need to install each package individually | |
| choco upgrade llvm -y | |
| # psutils is out of date | |
| # choco upgrade psutils -y | |
| choco upgrade openssl -y | |
| # Should already be installed | |
| # choco upgrade strawberryperl -y | |
| choco upgrade protoc -y | |
| - name: Set environment variables - Nix | |
| if: ${{ ! startsWith(runner.os,'Windows') }} | |
| shell: bash | |
| run: | | |
| echo "SHARUN=shasum --algorithm 256" >> $GITHUB_ENV | |
| echo "TS_EXT=" >> $GITHUB_ENV | |
| echo "SHELL_EXT=.sh" >> $GITHUB_ENV | |
| echo "PLATFORM_SPECIFIC_DIR=linux" >> $GITHUB_ENV | |
| echo "TS_DIST=/dist" >> $GITHUB_ENV | |
| - name: Set environment variables - Windows | |
| if: startsWith(runner.os,'Windows') | |
| shell: bash | |
| run: | | |
| # echo "SHARUN=pwsh C:\ProgramData\chocolatey\lib\psutils\tools\psutils-master\shasum.ps1 --algorithm 256" >> $GITHUB_ENV | |
| mkdir -p "$GITHUB_WORKSPACE\psutils" | |
| curl -v -o "$GITHUB_WORKSPACE\psutils\getopt.ps1" "https://raw.githubusercontent.com/lukesampson/psutils/master/getopt.ps1" | |
| curl -v -o "$GITHUB_WORKSPACE\psutils\shasum.ps1" "https://raw.githubusercontent.com/lukesampson/psutils/master/shasum.ps1" | |
| echo "SHARUN=pwsh $GITHUB_WORKSPACE\psutils\shasum.ps1 --algorithm 256" >> $GITHUB_ENV | |
| echo "TS_EXT=.exe" >> $GITHUB_ENV | |
| echo "LIB_EXT=.dll" >> $GITHUB_ENV | |
| echo "SHELL_EXT=.bat" >> $GITHUB_ENV | |
| echo "TS_DIST=\dist" >> $GITHUB_ENV | |
| echo "PLATFORM_SPECIFIC_DIR=windows" >> $GITHUB_ENV | |
| echo "SQLITE3_LIB_DIR=C:\vcpkg\installed\x64-windows\lib" >> $GITHUB_ENV | |
| echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >> $GITHUB_ENV | |
| echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $GITHUB_ENV | |
| echo "C:\Strawberry\perl\bin" >> $GITHUB_PATH | |
| - name: Cache cargo files and outputs | |
| if: ${{ ( ! startsWith(github.ref, 'refs/tags/v') ) && ( ! matrix.builds.cross ) && ( env.CARGO_CACHE ) }} | |
| uses: swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ matrix.builds.target }} | |
| - name: Install and setup cargo cross | |
| if: ${{ matrix.builds.cross }} | |
| shell: bash | |
| run: | | |
| #cargo install cross | |
| cargo install cross --git https://github.com/cross-rs/cross | |
| echo "CARGO=cross" >> $GITHUB_ENV | |
| - name: Install and setup cargo-auditable | |
| # if: ${{ ( startsWith(github.ref, 'refs/tags/v') ) && ( ! matrix.builds.cross ) }} | |
| if: ${{ ( ! matrix.builds.cross ) }} | |
| shell: bash | |
| run: | | |
| cargo install cargo-auditable cargo-audit | |
| echo "CARGO=${{ env.CARGO }} auditable" >> $GITHUB_ENV | |
| - name: Build release targets | |
| # if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| shell: bash | |
| run: | | |
| # echo "TS_BUILD=release" >> $GITHUB_ENV | |
| if [[ "${{ env.TS_BUILD }}" == "release" ]]; then | |
| echo "CARGO_OPTIONS=${{ env.CARGO_OPTIONS }} --${{ env.TS_BUILD }}" >> $GITHUB_ENV | |
| fi | |
| - name: Show command used for Cargo | |
| shell: bash | |
| run: | | |
| echo "cargo command is: ${{ env.CARGO }}" | |
| echo "cargo options is: ${{ env.CARGO_OPTIONS }}" | |
| echo "cross flag: ${{ matrix.builds.cross }}" | |
| - name: Build release binaries | |
| shell: bash | |
| run: | | |
| ${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \ | |
| --target ${{ matrix.builds.target }} \ | |
| ${{ matrix.builds.flags }} | |
| - name: Copy binaries to folder for archiving | |
| shell: bash | |
| run: | | |
| # set -xo pipefail | |
| mkdir -p "$GITHUB_WORKSPACE${TS_DIST}" | |
| cd "$GITHUB_WORKSPACE${TS_DIST}" | |
| BINFILE="${TS_FILENAME}-${TS_VERSION}-${VSHA_SHORT}-${{ matrix.builds.name }}${TS_EXT}" | |
| echo "BINFILE=${BINFILE}" >> $GITHUB_ENV | |
| echo "Copying files for ${BINFILE} to $(pwd)" | |
| echo "MTS_SOURCE=$(pwd)" >> $GITHUB_ENV | |
| ls -alht "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/${{ env.TS_BUILD }}/" | |
| echo "checking for file - ${FILE}${TS_EXT}" | |
| if [ -f "${GITHUB_WORKSPACE}/target/${{ matrix.builds.target }}/${{ env.TS_BUILD }}/${TS_FILENAME}${TS_EXT}" ]; then | |
| cp -vf "${GITHUB_WORKSPACE}/target/${{ matrix.builds.target }}/${{ env.TS_BUILD }}/${TS_FILENAME}${TS_EXT}" . | |
| fi | |
| ls -alhtR ${{ env.MTS_SOURCE }} | |
| - name: Audit tree and feedback for binaries | |
| # if: ${{ ( ! matrix.builds.cross ) }} | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| cd "${{ env.MTS_SOURCE }}" | |
| echo "Audit binaries ..." | |
| cargo audit bin *tari* | |
| - name: Archive and Checksum Binaries | |
| shell: bash | |
| run: | | |
| echo "Archive ${{ env.BINFILE }} too ${{ env.BINFILE }}.zip" | |
| cd "${{ env.MTS_SOURCE }}" | |
| echo "Compute files shasum" | |
| ${SHARUN} * >> "${{ env.BINFILE }}.sha256" | |
| echo "Show the shasum" | |
| cat "${{ env.BINFILE }}.sha256" | |
| echo "Checksum verification for files is " | |
| ${SHARUN} --check "${{ env.BINFILE }}.sha256" | |
| 7z a "${{ env.BINFILE }}.zip" * | |
| echo "Compute archive shasum" | |
| ${SHARUN} "${{ env.BINFILE }}.zip" >> "${{ env.BINFILE }}.zip.sha256" | |
| echo "Show the shasum" | |
| cat "${{ env.BINFILE }}.zip.sha256" | |
| echo "Checksum verification archive is " | |
| ${SHARUN} --check "${{ env.BINFILE }}.zip.sha256" | |
| - name: Artifact upload for Archive | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.TS_FILENAME }}_archive-${{ matrix.builds.name }} | |
| path: "${{ github.workspace }}${{ env.TS_DIST }}/${{ env.BINFILE }}.zip*" | |
| create-release: | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| runs-on: ubuntu-latest | |
| needs: [ builds ] | |
| env: | |
| TS_VERSION: ${{ needs.builds.outputs.TS_VERSION }} | |
| VSHA_SHORT: ${{ needs.builds.outputs.VSHA_SHORT }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Download binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ env.TS_FILENAME }} | |
| pattern: "${{ env.TS_FILENAME }}*" | |
| merge-multiple: true | |
| - name: Verify checksums and Prep Uploads | |
| shell: bash | |
| working-directory: ${{ env.TS_FILENAME }} | |
| run: | | |
| # set -xo pipefail | |
| sudo apt-get update | |
| sudo apt-get --no-install-recommends --assume-yes install dos2unix | |
| ls -alhtR | |
| if [ -f "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" ] ; then | |
| rm -fv "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" | |
| fi | |
| # Merge all sha256 files into one | |
| find . -name "*.sha256" -type f -print | xargs cat >> "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" | |
| dos2unix --quiet "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" | |
| cat "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" | |
| sha256sum --ignore-missing --check "${{ env.TS_FILENAME }}-${{ env.TS_VERSION }}-${{ env.VSHA_SHORT }}.${{ env.TS_SIG_FN }}" | |
| ls -alhtR | |
| - name: Create release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| artifacts: "${{ env.TS_FILENAME }}*/**/*" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| prerelease: true | |
| draft: true | |
| allowUpdates: true | |
| updateOnlyUnreleased: true | |
| replacesArtifacts: true |