Merge pull request #2623 from subspace/fix-farming-performance-windows #292

Workflow file for this run

.github/workflows/snapshot-build.yml at 5d1f276

	# This action enables building container images and executables for farmer and node, can be triggered manually or by
	# release creation.
	#
	# Container images are only pushed to GitHub Container Registry for releases.
	# Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets.
	name: Snapshot build

	on:
	workflow_dispatch:
	push:
	tags:
	- "snapshot-*"
	- "gemini-*"

	# Incremental compilation here isn't helpful
	env:
	CARGO_INCREMENTAL: 0

	jobs:
	container-linux:
	runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' \|\| '"ubuntu-22.04"') }}
	permissions:
	contents: write
	packages: write
	strategy:
	matrix:
	image:
	- farmer
	- node
	- bootstrap-node
	platform:
	- arch: linux/amd64
	dockerfile-suffix: ""
	suffix: ubuntu-x86_64-${{ github.ref_name }}
	image-suffix: ""
	rustflags: "-C target-cpu=skylake"
	# We build AArch64
	- arch: linux/amd64
	dockerfile-suffix: ".aarch64"
	suffix: ubuntu-aarch64-${{ github.ref_name }}
	image-suffix: "-aarch64"
	fail-fast: false

	steps:
	- name: Set up QEMU
	uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # @v2.2.0

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # @v2.2.1

	- name: Log into registry
	uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # @v2.2.0
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ github.token }}

	- name: Extract Docker metadata
	id: meta
	uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # @v4.1.1
	with:
	images: \|
	ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
	tags: \|
	type=ref,event=tag
	type=ref,event=branch
	type=sha,format=long
	flavor: \|
	latest=false
	suffix=${{ matrix.platform.image-suffix }}

	- name: Build and push ${{ matrix.image }} image
	uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # @v3.2.0
	with:
	file: Dockerfile-${{ matrix.image }}${{ matrix.platform.dockerfile-suffix }}
	platforms: ${{ matrix.platform.arch }}
	push: true
	tags: ${{ steps.meta.outputs.tags }}
	labels: ${{ steps.meta.outputs.labels }}
	build-args: \|
	SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }}
	RUSTFLAGS=${{ matrix.platform.rustflags }}

	- name: Trigger trivy-security-scan Workflow
	uses: peter-evans/repository-dispatch@a4a90276d01e3a2ae44fa10a0247287f045afd59 # @v2.1.2
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	event-type: trivy-scan-dispatch
	client-payload: '{"image": "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.sha }}"}'

	executables:
	strategy:
	matrix:
	build:
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' \|\| '"ubuntu-20.04"') }}
	target: x86_64-unknown-linux-gnu
	suffix: ubuntu-x86_64-skylake-${{ github.ref_name }}
	rustflags: "-C target-cpu=skylake"
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' \|\| '"ubuntu-20.04"') }}
	target: x86_64-unknown-linux-gnu
	suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
	rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' \|\| '"ubuntu-20.04"') }}
	target: aarch64-unknown-linux-gnu
	suffix: ubuntu-aarch64-${{ github.ref_name }}
	# TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships:
	# https://github.com/RustCrypto/block-ciphers/pull/395
	rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8"
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-14-arm64"]' \|\| '"macos-14"') }}
	target: aarch64-apple-darwin
	suffix: macos-aarch64-${{ github.ref_name }}
	# TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships:
	# https://github.com/RustCrypto/block-ciphers/pull/395
	rustflags: "--cfg aes_armv8"
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-14-arm64"]' \|\| '"macos-14"') }}
	target: x86_64-apple-darwin
	suffix: macos-x86_64-${{ github.ref_name }}
	rustflags: ""
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' \|\| '"windows-2022"') }}
	target: x86_64-pc-windows-msvc
	suffix: windows-x86_64-skylake-${{ github.ref_name }}
	rustflags: "-C target-cpu=skylake"
	- os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' \|\| '"windows-2022"') }}
	target: x86_64-pc-windows-msvc
	suffix: windows-x86_64-v2-${{ github.ref_name }}
	rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
	fail-fast: false
	runs-on: ${{ matrix.build.os }}
	env:
	PRODUCTION_TARGET: target/${{ matrix.build.target }}/production
	RUSTFLAGS: ${{ matrix.build.rustflags }}

	steps:
	- name: Checkout
	uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0

	# On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support
	- name: Install LLVM and Clang for macOS
	uses: KyleMayes/install-llvm-action@be40c5af3a4adc3e4a03199995ab73aa37536712 # v1.9.0
	with:
	# TODO: Switch to LLVM 17 on arm64 runners once https://github.com/KyleMayes/install-llvm-action/issues/61 is resolved
	version: 15.0.7
	if: runner.os == 'macOS'

	# TODO: Workaround for https://github.com/actions/runner-images/issues/9290
	- name: Install glibtoolize (macOS)
	run: brew install libtool
	if: runner.os == 'macOS'

	- name: Install Protoc
	uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9 # v2.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	# Needed for hwloc
	- name: Install automake (macOS)
	run: brew install automake
	if: runner.os == 'macOS'

	# Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
	- name: Remove msys64
	run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
	if: runner.os == 'Windows'
	# Doesn't exist on self-hosted runners
	continue-on-error: true

	- name: AArch64 cross-compile packages
	run: \|
	FLAVOR="$(lsb_release -sc)"

	sudo tee /etc/apt/sources.list.d/arm64.list <<LIST
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} main restricted
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates main restricted
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} universe
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates universe
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} multiverse
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates multiverse
	deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-backports main restricted universe multiverse
	LIST
	sudo sed -i 's/deb http/deb [arch=amd64] http/' /etc/apt/sources.list
	# GitHub runners use mirror file
	sudo sed -i 's/deb mirror/deb [arch=amd64] mirror/' /etc/apt/sources.list

	sudo dpkg --add-architecture arm64
	sudo apt-get update
	# zlib1g-dev:arm64 is only necessary because amd64 version is present on the host and cross-compilation of
	# hwlocality-sys fails otherwise
	sudo apt-get install -y --no-install-recommends \
	g++-aarch64-linux-gnu \
	gcc-aarch64-linux-gnu \
	libc6-dev-arm64-cross \
	zlib1g-dev:arm64

	echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV
	if: matrix.build.target == 'aarch64-unknown-linux-gnu'

	- name: Configure cache
	uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # @v3.3.2
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
	restore-keys: \|
	${{ runner.os }}-cargo-

	- name: Build farmer (Linux and Windows)
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer
	if: runner.os != 'macOS'

	# We build macOS without `numa` feature, primarily because of https://github.com/HadrienG2/hwlocality/issues/31
	- name: Build farmer (macOS)
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --no-default-features
	if: runner.os == 'macOS'

	- name: Build node
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node

	- name: Sign Application (macOS)
	run: \|
	echo "Importing certificate"
	echo "${{ secrets.MACOS_CERTIFICATE }}" \| base64 --decode > certificate.p12
	security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
	security default-keychain -s build.keychain
	security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
	security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign
	security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
	echo "Signing farmer"
	codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer
	echo "Signing node"
	codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node
	echo "Creating an archive"
	mkdir ${{ env.PRODUCTION_TARGET }}/macos-binaries
	cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries
	ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip
	echo "Notarizing"
	brew update
	brew install mitchellh/gon/gon
	cat << EOF > gon.hcl
	source = ["subspace-binaries.zip"]
	bundle_id = "${{ secrets.MACOS_BUNDLE_ID }}"
	sign {
	application_identity = "${{ secrets.MACOS_IDENTITY }}"
	}
	apple_id {
	username = "${{ secrets.MACOS_APPLE_ID }}"
	password = "${{ secrets.MACOS_APP_PW }}"
	}
	EOF
	gon -log-level=info -log-json gon.hcl

	# Notarize the ZIP using notarytool
	xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait

	# // todo stapling for macOS artifacts
	# Staple the zip package
	# xcrun stapler staple subspace-binaries.zip

	echo "Done!"
	# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
	continue-on-error: ${{ github.repository_owner != 'subspace' \|\| github.event_name != 'push' \|\| github.ref_type != 'tag' }}
	if: runner.os == 'macOS'

	- name: Sign Application (Windows)
	run: \|
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe"
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe"
	# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
	continue-on-error: ${{ github.repository_owner != 'subspace' \|\| github.event_name != 'push' \|\| github.ref_type != 'tag' }}
	if: runner.os == 'Windows'

	- name: Prepare executables for uploading (Ubuntu)
	run: \|
	mkdir executables
	mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }}
	mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
	if: runner.os == 'Linux'

	- name: Prepare executables for uploading (macOS)
	run: \|
	mkdir executables
	mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }}
	mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
	# Zip it so that signature is not lost
	ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip
	ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip
	rm executables/subspace-farmer-${{ matrix.build.suffix }}
	rm executables/subspace-node-${{ matrix.build.suffix }}
	if: runner.os == 'macOS'

	- name: Prepare executables for uploading (Windows)
	run: \|
	mkdir executables
	move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe
	move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe
	if: runner.os == 'Windows'

	- name: Upload node and farmer executables to artifacts
	uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # @v3.1.1
	with:
	name: executables-${{ matrix.build.suffix }}
	path: \|
	executables/*
	if-no-files-found: error

	- name: Upload node and farmer executables to assets
	uses: alexellis/upload-assets@259de5111cb56966d046ced998941e93f91d2c93 # @0.4.0
	env:
	GITHUB_TOKEN: ${{ github.token }}
	with:
	asset_paths: '["executables/*"]'
	if: github.event_name == 'push' && github.ref_type == 'tag'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #2623 from subspace/fix-farming-performance-windows #292

Workflow file

Merge pull request #2623 from subspace/fix-farming-performance-windows #292

Jobs

Run details

Workflow file for this run