Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: improve the macOS story on CI #1597

Merged
merged 1 commit into from
May 2, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 18 additions & 39 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,6 @@ setup-env: &setup-env


jobs:
# This job is needed for macOS, which doesn't support restoring caches that
# were created on Linux. This job will run on Linux, restore the cache and
# then persist it to the workspace, so that the macOS job can then use the
# files from the workspace.
persist_parameters_to_workspace:
executor: default
environment: *setup-env
steps:
- checkout
- restore_parameter_cache
- persist_to_workspace:
root: /tmp
paths:
- filecoin-proof-parameters

ensure_groth_parameters_and_keys_linux:
executor: default
environment: *setup-env
Expand Down Expand Up @@ -296,35 +281,29 @@ jobs:
environment: *setup-env
steps:
- checkout
# In case of macOS, the workspace contains the parameter files.
- attach_workspace:
at: "/tmp"
- run:
name: Install hwloc 2.3.0
command: |
cd /tmp
curl https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz --location --output /tmp/hwloc-2.3.0.tar.gz
tar xzvf hwloc-2.3.0.tar.gz
cd hwloc-2.3.0
./configure
make
sudo make install
name: Install hwloc with Homebrew
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install hwloc
- run:
name: Install Rust
command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y
- run: rustup install $(cat rust-toolchain)
- run: rustup default $(cat rust-toolchain)
- run: cargo update
- run: cargo fetch
# On CircleCI you cannot create a cache on Linux and restore it on macOS.
# Hence we need a separate cache for macOS that we create and restore. It
# is not a separate job (like `ensure_groth_parameters_and_keys_linux`,
# as there is only this single job that runs on macOS.
- restore_parameter_cache
- ensure_filecoin_parameters
- save_parameter_cache
- run:
name: Test Darwin
command: |
sudo ulimit -n 20000
sudo ulimit -u 20000
ulimit -n 20000
cargo +$(cat rust-toolchain) test --release --verbose --workspace -- --nocapture
cargo test --release --verbose --workspace -- --nocapture
no_output_timeout: 2h

commands:
Expand All @@ -337,7 +316,7 @@ commands:
name: Build paramcache if it doesn't already exist
command: |
set -x; test -f /tmp/paramcache.awesome \
|| (cargo build --release --workspace && find . -type f -name paramcache | xargs -I {} mv {} /tmp/paramcache.awesome)
|| (cargo build --release --bin paramcache && find . -type f -name paramcache | xargs -I {} mv {} /tmp/paramcache.awesome)
- run:
name: Obtain filecoin groth parameters
command: /tmp/paramcache.awesome --sector-sizes='2048,4096,16384,32768'
Expand Down Expand Up @@ -373,26 +352,27 @@ commands:
- run: cp -R /tmp/.cargo /tmp/.rustup ~/ || true
save_parameter_cache:
steps:
# Make sure we have different cache for Linux and Darwin.
- run: uname > os.txt
- save_cache:
name: "Save parameter cache"
key: proof-params-v28-k-{{ checksum "filecoin-proofs/parameters.json" }}
key: proof-params-v28-k-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }}
paths:
- "/tmp/paramcache.awesome"
- "/tmp/filecoin-proof-parameters/"
restore_parameter_cache:
steps:
# Make sure we have different cache for Linux and Darwin.
- run: uname > os.txt
- restore_cache:
name: "Restore parameter cache"
key: proof-params-v28-k-{{ checksum "filecoin-proofs/parameters.json" }}
key: proof-params-v28-k-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }}

workflows:
version: 2.1
test_all:
jobs:
- ensure_groth_parameters_and_keys_linux
- persist_parameters_to_workspace:
requires:
- ensure_groth_parameters_and_keys_linux
- cargo_fetch
- rustfmt:
requires:
Expand Down Expand Up @@ -540,9 +520,8 @@ workflows:
requires:
- cargo_fetch
- ensure_groth_parameters_and_keys_linux
- test_darwin:
requires:
- persist_parameters_to_workspace

- test_darwin

- test:
name: test_fr32
Expand Down