diff --git a/.github/actions/install-nix/action.yml b/.github/actions/install-nix/action.yml new file mode 100644 index 00000000..ab7efd58 --- /dev/null +++ b/.github/actions/install-nix/action.yml @@ -0,0 +1,21 @@ +name: install Nix + +inputs: + cachixAuthToken: + description: auth token for https://app.cachix.org/cache/rvolosatovs + +runs: + using: composite + steps: + - uses: DeterminateSystems/nix-installer-action@v4 + with: + extra-conf: | + accept-flake-config = true + + - uses: DeterminateSystems/magic-nix-cache-action@v2 + + - uses: cachix/cachix-action@v12 + continue-on-error: true + with: + name: rvolosatovs + authToken: '${{ inputs.cachixAuthToken }}' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08262570..0d087d4e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,284 +10,236 @@ concurrency: cancel-in-progress: true jobs: - rust-hello: + rust-hello-linux: strategy: matrix: config: - # Darwin - # NOTE: Only build Wasm targets on Darwin here, since Mac machines are extremely slow on GitHub - - os: macos-12 - package: rust-hello-wasm32-unknown-unknown - test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm - - - os: macos-12 - package: rust-hello-wasm32-wasi - test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm - - - os: macos-12 - package: rust-hello-debug-wasm32-unknown-unknown - test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm - - - os: macos-12 - package: rust-hello-debug-wasm32-wasi - test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm - - # Linux - - os: ubuntu-22.04 - package: default + - package: default test-bin: ./result/bin/rust-hello - - os: ubuntu-22.04 - package: rust-hello + - package: rust-hello test-bin: ./result/bin/rust-hello - - os: ubuntu-22.04 - package: rust-hello-aarch64-apple-darwin + - package: rust-hello-aarch64-apple-darwin test-bin: file ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-aarch64-unknown-linux-musl + - package: rust-hello-aarch64-unknown-linux-musl test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-armv7-unknown-linux-musleabihf + - package: rust-hello-armv7-unknown-linux-musleabihf test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-wasm32-unknown-unknown + - package: rust-hello-wasm32-unknown-unknown test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm - - os: ubuntu-22.04 - package: rust-hello-wasm32-wasi + - package: rust-hello-wasm32-wasi test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-x86_64-apple-darwin + - package: rust-hello-x86_64-apple-darwin test-bin: file ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-x86_64-pc-windows-gnu + - package: rust-hello-x86_64-pc-windows-gnu test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/rust-hello.exe test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-x86_64-unknown-linux-musl + - package: rust-hello-x86_64-unknown-linux-musl test-bin: ./result/bin/rust-hello test-oci: | docker load < ./result docker run --rm rust-hello:$(nix eval --raw --override-input 'nixify' '.' ./examples/rust-hello#rust-hello-x86_64-unknown-linux-musl-oci.imageTag) - - os: ubuntu-22.04 - package: rust-hello-debug + - package: rust-hello-debug test-bin: ./result/bin/rust-hello - - os: ubuntu-22.04 - package: rust-hello-debug-aarch64-apple-darwin + - package: rust-hello-debug-aarch64-apple-darwin test-bin: file ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-aarch64-unknown-linux-musl + - package: rust-hello-debug-aarch64-unknown-linux-musl test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-armv7-unknown-linux-musleabihf + - package: rust-hello-debug-armv7-unknown-linux-musleabihf test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-wasm32-wasi + - package: rust-hello-debug-wasm32-wasi test-bin: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-x86_64-apple-darwin + - package: rust-hello-debug-x86_64-apple-darwin test-bin: file ./result/bin/rust-hello test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-x86_64-pc-windows-gnu + - package: rust-hello-debug-x86_64-pc-windows-gnu test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/rust-hello.exe test-oci: docker load < ./result - - os: ubuntu-22.04 - package: rust-hello-debug-x86_64-unknown-linux-musl + - package: rust-hello-debug-x86_64-unknown-linux-musl test-bin: ./result/bin/rust-hello test-oci: | docker load < ./result docker run --rm rust-hello:$(nix eval --raw --override-input 'nixify' '.' ./examples/rust-hello#rust-hello-x86_64-unknown-linux-musl-oci.imageTag) - runs-on: ${{ matrix.config.os }} + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@v4 + - uses: ./.github/actions/install-nix with: - extra-conf: | - accept-flake-config = true - - uses: DeterminateSystems/magic-nix-cache-action@v2 - - uses: cachix/cachix-action@v12 - continue-on-error: true - with: - name: rvolosatovs authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix build -L --show-trace --override-input 'nixify' '.' './examples/rust-hello#${{ matrix.config.package }}' - run: ${{ matrix.config.test-bin }} - - run: nix build -L --show-trace --override-input 'nixify' '.' './examples/rust-hello#${{ matrix.config.package }}-oci' if: ${{ matrix.config.package != 'default' && matrix.config.package != 'rust-hello' && matrix.config.package != 'rust-hello-debug' }} - run: ${{ matrix.config.test-oci }} - rust-complex: + + rust-hello-macos-wasm: + strategy: + matrix: + # NOTE: Only build Wasm targets on Darwin here, since Mac machines are extremely slow on GitHub. Other targets are built as part of `rust-complex-macos` + config: + - package: rust-hello-wasm32-unknown-unknown + - package: rust-hello-wasm32-wasi + - package: rust-hello-debug-wasm32-unknown-unknown + - package: rust-hello-debug-wasm32-wasi + + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-nix + with: + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix build -L --show-trace --override-input 'nixify' '.' './examples/rust-hello#${{ matrix.config.package }}' + - run: nix run --inputs-from . 'nixpkgs#wasmtime' ./result/bin/rust-hello.wasm + + + rust-complex-linux: strategy: matrix: config: - # Darwin - - os: macos-12 - package: default + - package: default test-bin: ./result/bin/foo - - os: macos-12 - package: rust-complex + - package: rust-complex test-bin: ./result/bin/foo - - os: macos-12 - package: rust-complex-aarch64-apple-darwin + - package: rust-complex-aarch64-apple-darwin test-bin: file ./result/bin/foo - - os: macos-12 - package: rust-complex-aarch64-unknown-linux-musl - test-bin: file ./result/bin/foo + - package: rust-complex-aarch64-unknown-linux-musl + test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/foo - - os: macos-12 - package: rust-complex-armv7-unknown-linux-musleabihf - test-bin: file ./result/bin/foo + - package: rust-complex-armv7-unknown-linux-musleabihf + test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/foo - - os: macos-12 - package: rust-complex-x86_64-apple-darwin - test-bin: ./result/bin/foo + - package: rust-complex-x86_64-apple-darwin + test-bin: file ./result/bin/foo - - os: macos-12 - package: rust-complex-x86_64-pc-windows-gnu - test-bin: file ./result/bin/foo.exe + - package: rust-complex-x86_64-pc-windows-gnu + test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/foo.exe suffix: .exe - - os: macos-12 - package: rust-complex-x86_64-unknown-linux-musl - test-bin: file ./result/bin/foo + - package: rust-complex-x86_64-unknown-linux-musl + test-bin: ./result/bin/foo - - os: macos-12 - package: rust-complex-debug + - package: rust-complex-debug test-bin: ./result/bin/foo - - os: macos-12 - package: rust-complex-debug-aarch64-apple-darwin + - package: rust-complex-debug-aarch64-apple-darwin test-bin: file ./result/bin/foo - - os: macos-12 - package: rust-complex-debug-aarch64-unknown-linux-musl - test-bin: file ./result/bin/foo + - package: rust-complex-debug-aarch64-unknown-linux-musl + test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/foo - - os: macos-12 - package: rust-complex-debug-armv7-unknown-linux-musleabihf - test-bin: file ./result/bin/foo + - package: rust-complex-debug-armv7-unknown-linux-musleabihf + test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/foo - - os: macos-12 - package: rust-complex-debug-x86_64-apple-darwin - test-bin: ./result/bin/foo + - package: rust-complex-debug-x86_64-apple-darwin + test-bin: file ./result/bin/foo - - os: macos-12 - package: rust-complex-debug-x86_64-pc-windows-gnu - test-bin: file ./result/bin/foo.exe + - package: rust-complex-debug-x86_64-pc-windows-gnu + test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/foo.exe suffix: .exe - - os: macos-12 - package: rust-complex-debug-x86_64-unknown-linux-musl - test-bin: file ./result/bin/foo + - package: rust-complex-debug-x86_64-unknown-linux-musl + test-bin: ./result/bin/foo + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-nix + with: + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - uses: ./.github/actions/build-nix + with: + package: ${{ matrix.config.package }} + flake: ./examples/rust-complex + install-path: /bin/foo${{ matrix.config.suffix }} + - run: ${{ matrix.config.test-bin }} - # Linux - - os: ubuntu-22.04 - package: default + rust-complex-macos: + strategy: + matrix: + config: + - package: default test-bin: ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex + - package: rust-complex test-bin: ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-aarch64-apple-darwin + - package: rust-complex-aarch64-apple-darwin test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-aarch64-unknown-linux-musl - test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/foo - - - os: ubuntu-22.04 - package: rust-complex-armv7-unknown-linux-musleabihf - test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/foo + - package: rust-complex-aarch64-unknown-linux-musl + test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-x86_64-apple-darwin + - package: rust-complex-armv7-unknown-linux-musleabihf test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-x86_64-pc-windows-gnu - test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/foo.exe + - package: rust-complex-x86_64-apple-darwin + test-bin: ./result/bin/foo + + - package: rust-complex-x86_64-pc-windows-gnu + test-bin: file ./result/bin/foo.exe suffix: .exe - - os: ubuntu-22.04 - package: rust-complex-x86_64-unknown-linux-musl - test-bin: ./result/bin/foo + - package: rust-complex-x86_64-unknown-linux-musl + test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-debug + - package: rust-complex-debug test-bin: ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-debug-aarch64-apple-darwin + - package: rust-complex-debug-aarch64-apple-darwin test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-debug-aarch64-unknown-linux-musl - test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-aarch64 ./result/bin/foo - - - os: ubuntu-22.04 - package: rust-complex-debug-armv7-unknown-linux-musleabihf - test-bin: nix shell --inputs-from . 'nixpkgs#qemu' -c qemu-arm ./result/bin/foo + - package: rust-complex-debug-aarch64-unknown-linux-musl + test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-debug-x86_64-apple-darwin + - package: rust-complex-debug-armv7-unknown-linux-musleabihf test-bin: file ./result/bin/foo - - os: ubuntu-22.04 - package: rust-complex-debug-x86_64-pc-windows-gnu - test-bin: nix shell --inputs-from . 'nixpkgs#wine64' -c wine64 ./result/bin/foo.exe + - package: rust-complex-debug-x86_64-apple-darwin + test-bin: ./result/bin/foo + + - package: rust-complex-debug-x86_64-pc-windows-gnu + test-bin: file ./result/bin/foo.exe suffix: .exe - - os: ubuntu-22.04 - package: rust-complex-debug-x86_64-unknown-linux-musl - test-bin: ./result/bin/foo + - package: rust-complex-debug-x86_64-unknown-linux-musl + test-bin: file ./result/bin/foo - runs-on: ${{ matrix.config.os }} + runs-on: macos-12 steps: - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@v4 + - uses: ./.github/actions/install-nix with: - extra-conf: | - accept-flake-config = true - - uses: DeterminateSystems/magic-nix-cache-action@v2 - - uses: cachix/cachix-action@v12 - continue-on-error: true - with: - name: rvolosatovs authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - uses: ./.github/actions/build-nix with: package: ${{ matrix.config.package }} @@ -296,8 +248,8 @@ jobs: - run: ${{ matrix.config.test-bin }} test-linux: - runs-on: macos-12 - needs: rust-complex + runs-on: ubuntu-22.04 + needs: rust-complex-linux steps: - uses: actions/download-artifact@v3 with: @@ -307,7 +259,7 @@ jobs: test-macos: runs-on: macos-12 - needs: rust-complex + needs: rust-complex-linux steps: - uses: actions/download-artifact@v3 with: @@ -327,20 +279,34 @@ jobs: test-windows: runs-on: windows-2022 - needs: rust-complex + needs: rust-complex-linux steps: - uses: actions/download-artifact@v3 with: name: rust-complex-x86_64-pc-windows-gnu - run: '& .\rust-complex-x86_64-pc-windows-gnu' + test-oci: + runs-on: ubuntu-22.04 + needs: + - rust-hello-linux + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-nix + with: + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix run -L --show-trace --override-input 'nixify' '.' './examples/rust-hello#build-rust-hello-oci' + test-artifacts: runs-on: ubuntu-22.04 needs: - - rust-complex - - rust-hello + - rust-complex-linux + - rust-complex-macos + - rust-hello-linux + - rust-hello-macos-wasm - test-linux - test-macos + - test-oci - test-windows steps: - run: echo "binary artifacts are valid" diff --git a/flake.nix b/flake.nix index 09c9caa6..25319a9a 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,7 @@ }: extendDerivations { buildInputs = with pkgs; [ + buildah wasmtime zig ]; diff --git a/lib/rust/mkAttrs.nix b/lib/rust/mkAttrs.nix index 6690b184..303d688e 100644 --- a/lib/rust/mkAttrs.nix +++ b/lib/rust/mkAttrs.nix @@ -278,11 +278,13 @@ with self.lib.rust.targets; # buildPackageFor builds for `target`. # `extraArgs` are passed through to `buildPackage` verbatim. # NOTE: Upstream only provides binary caches for a subset of supported systems. - buildPackageFor = target: craneArgs: let + buildPackageFor = { + craneArgs, + craneLib, + pkgsCross, + target, + }: let kebab2snake = replaceStrings ["-"] ["_"]; - rustToolchain = rustToolchainFor target; - craneLib = mkCraneLib final rustToolchain; - pkgsCross = pkgsFor final target; useRosetta = final.stdenv.buildPlatform.isDarwin && final.stdenv.buildPlatform.isAarch64 && pkgsCross.stdenv.hostPlatform.isDarwin && pkgsCross.stdenv.hostPlatform.isx86_64; useEmu = final.stdenv.buildPlatform.system != pkgsCross.stdenv.hostPlatform.system && !useRosetta && pkgsCross.stdenv.hostPlatform.system != aarch64-darwin; @@ -498,39 +500,133 @@ with self.lib.rust.targets; targetBins = let mkOutputs = target: let - release = buildPackageFor target commonReleaseArgs; - debug = buildPackageFor target commonDebugArgs; + pkgsCross = pkgsFor final target; + rustToolchain = rustToolchainFor target; + craneLib = mkCraneLib final rustToolchain; + + withPassthru = craneArgs: {passthru ? {}, ...} @ pkg: + pkg + // { + passthru = + passthru + // { + inherit + pkgsCross + rustToolchain + target + ; + } + // optionalAttrs (craneArgs ? CARGO_PROFILE) { + inherit + (craneArgs) + CARGO_PROFILE + ; + }; + }; + + buildPackageFor' = craneArgs: let + pkg = buildPackageFor { + inherit + craneArgs + craneLib + pkgsCross + target + ; + }; + in + withPassthru craneArgs pkg; in optionalAttrs targets'.${target} { - "${pname'}-${target}" = release; - "${pname'}-${target}-deps" = release.cargoArtifacts; - "${pname'}-debug-${target}" = debug; - "${pname'}-debug-${target}-deps" = debug.cargoArtifacts; + "${pname'}-${target}" = buildPackageFor' commonReleaseArgs; + "${pname'}-debug-${target}" = buildPackageFor' commonDebugArgs; }; packages = map mkOutputs (attrValues rust.targets); in foldr mergeAttrs {} packages; + targetDeps = mapAttrs' (name: bin: nameValuePair "${name}-deps" bin.cargoArtifacts) targetBins; + + # https://github.com/docker-library/official-images#architectures-other-than-amd64 + # https://go.dev/doc/install/source#environment + # https://github.com/docker-library/bashbrew/blob/7e160dca3123caecf32c33ba31821dd2aa3716cd/architecture/oci-platform.go#L14-L27 + ociArchitecture.${aarch64-apple-darwin} = "darwin-arm64v8"; + ociArchitecture.${aarch64-unknown-linux-gnu} = "arm64v8"; + ociArchitecture.${aarch64-unknown-linux-musl} = "arm64v8"; + ociArchitecture.${armv7-unknown-linux-musleabihf} = "arm32v7"; + ociArchitecture.${wasm32-unknown-unknown} = "wasm"; + ociArchitecture.${wasm32-wasi} = "wasm"; + ociArchitecture.${x86_64-apple-darwin} = "darwin-amd64"; + ociArchitecture.${x86_64-pc-windows-gnu} = "windows-amd64"; + ociArchitecture.${x86_64-unknown-linux-gnu} = "amd64"; + ociArchitecture.${x86_64-unknown-linux-musl} = "amd64"; + bins' = genAttrs bins (_: {}); - targetImages = optionalAttrs (bins' ? ${pname'}) (mapAttrs' (target: bin: - nameValuePair "${target}-oci" (final.dockerTools.buildImage { - name = pname'; - tag = version'; - copyToRoot = final.buildEnv { - name = pname'; - paths = [bin]; - }; - config.Cmd = [pname']; - config.Env = ["PATH=${bin}/bin"]; - })) - targetBins); + targetImages = optionalAttrs (bins' ? ${pname'}) ( + mapAttrs' ( + target: bin: let + img = final.dockerTools.buildImage ({ + name = pname'; + tag = version'; + copyToRoot = final.buildEnv { + name = pname'; + paths = [bin]; + }; + config.Cmd = [pname']; + config.Env = ["PATH=${bin}/bin"]; + } + // optionalAttrs (ociArchitecture ? ${bin.passthru.target}) { + architecture = ociArchitecture.${bin.passthru.target}; + }); + in + nameValuePair "${target}-oci" (img + // { + passthru = bin.passthru // img.passthru; + }) + ) + targetBins + ); + + multiArchTargets = [ + aarch64-unknown-linux-musl + armv7-unknown-linux-musleabihf + x86_64-pc-windows-gnu + x86_64-unknown-linux-musl + ]; in { "${pname'}" = hostBin; "${pname'}-debug" = hostDebugBin; } + // targetDeps // targetBins - // targetImages; + // targetImages + // optionalAttrs (any (target: targetImages ? "${pname'}-${target}-oci") multiArchTargets) + { + "build-${pname'}-oci" = let + build = final.writeShellScriptBin "build-${pname'}-oci" '' + set -xe + + build() { + ${final.buildah}/bin/buildah manifest create "localhost/''${1}" + ${concatMapStringsSep "\n" ( + target: let + name = "${pname'}-${target}-oci"; + in + optionalString (targetImages ? ${name}) '' + ${final.buildah}/bin/buildah manifest add "''${1}" docker-archive:${targetImages."${name}"} + '' + ) + multiArchTargets} + } + build "''${1:-${pname'}:${version'}}" + ''; + in (build + // { + inherit + version + ; + }); + }; packages = mkPackages final; in {