From f4e2aff3b15b097659dd31d476e33680f673a50f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 15:55:36 -0800 Subject: [PATCH 01/17] .github/workflows/macos.yml: Install tox from PyPI, not homebrew --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index bfa4bc58f96..0e8a76ad13d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -77,7 +77,7 @@ jobs: - name: Install test prerequisites run: | - brew install tox + pip install tox - name: Download upstream artifact uses: actions/download-artifact@v3 with: From 15193b35c9a5bfb48b9cea9daf434bcaefb2f4fb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 16:37:45 -0800 Subject: [PATCH 02/17] .github/workflows/macos.yml: Add tests on M1 runners --- .github/workflows/macos.yml | 17 +++++++++++------ tox.ini | 3 +++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0e8a76ad13d..2cbeb765b20 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -19,17 +19,22 @@ on: type: string # System configuration osversion_xcodeversion_toxenv_tuples: + # As of 2024-02, "runs-on: macos-latest" is macos-12. + # and "runs-on: macos-14" selects the new M1 runners. + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories description: 'Stringified JSON object' default: >- - [["latest", "", "homebrew-macos-usrlocal-minimal"], - ["latest", "", "homebrew-macos-usrlocal-standard"], - ["11", "xcode_13.2.1", "homebrew-macos-usrlocal-minimal"], + [["11", "xcode_13.2.1", "homebrew-macos-usrlocal-minimal"], + ["12", "", "homebrew-macos-usrlocal-minimal"], ["12", "", "homebrew-macos-usrlocal-standard"], + ["12", "", "homebrew-macos-usrlocal-python3_xcode-standard"], + ["12", "", "homebrew-macos-usrlocal-maximal"], ["13", "xcode_15.0", "homebrew-macos-usrlocal-standard"], - ["latest", "", "homebrew-macos-usrlocal-maximal"], - ["latest", "", "homebrew-macos-usrlocal-python3_xcode-standard"], + ["14", "", "homebrew-macos-opthomebrew-standard"], ["latest", "", "conda-forge-macos-minimal"], - ["latest", "", "conda-forge-macos-standard"]] + ["latest", "", "conda-forge-macos-standard"], + ["14", "", "conda-forge-macos-minimal"], + ["14", "", "conda-forge-macos-standard"]] type: string extra_sage_packages: description: 'Extra Sage packages to install as system packages' diff --git a/tox.ini b/tox.ini index 3673833724c..8f5a381201d 100644 --- a/tox.ini +++ b/tox.ini @@ -512,6 +512,9 @@ setenv = # brew caches downloaded files in ${HOME}/Library/Caches. We share it between different toxenvs. homebrew: SYSTEM=homebrew local-homebrew: HOMEBREW={envdir}/homebrew + # /opt/homebrew is the default install location on arm64 macOS + local-homebrew-opthomebrew: HOMEBREW=/opt/homebrew + # /usr/local is the default install location on x86_64 macOS local-{homebrew-usrlocal,nohomebrew}: HOMEBREW=/usr/local # local-macos-nohomebrew: "best effort" isolation to avoid using a homebrew installation in /usr/local # We use liblzma from the macOS system - which is available but its headers are not (neither is the xz executable). From 6b7c9a6e631209ca1e12f231405e416bcc659ec1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 16:39:12 -0800 Subject: [PATCH 03/17] .github/workflows/macos.yml: Increase SAGE_NUM_THREADS to 6 --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2cbeb765b20..4a5dd8d5888 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -134,7 +134,7 @@ jobs: *) export TARGETS_PRE="${{ inputs.targets_pre }}" TARGETS="${{ inputs.targets }} TARGETS_OPTIONAL="${{ inputs.targets_optional }} ;; esac - MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS + MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME" From 7b1d6c927b3f9f4648861f117c6f3b4d6f116661 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 16:46:47 -0800 Subject: [PATCH 04/17] .github/workflows/ci-macos.yml (local-nohomebrew): Increase SAGE_NUM_THREADS to 6 --- .github/workflows/ci-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 9d2c70cbb78..19ea15aaed6 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -140,7 +140,7 @@ jobs: # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts. # For doctesting, we use a lower parallelization to avoid timeouts. run: | - MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS + MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME" From 9d924c85ac866e9c55393387825d89b620d69c7c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 16:48:33 -0800 Subject: [PATCH 05/17] .github/workflows/ci-macos.yml (local-nohomebrew): Also test on M1 --- .github/workflows/ci-macos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 19ea15aaed6..13c0a83e1a9 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -108,7 +108,7 @@ jobs: fail-fast: false max-parallel: 4 matrix: - os: [ macos-11, macos-12 ] + os: [ macos-11, macos-12, macos-14 ] tox_system_factor: [macos-nobootstrap] tox_packages_factor: [minimal] xcode_version_factor: [default] @@ -129,7 +129,7 @@ jobs: if: contains(matrix.tox_system_factor, 'nobootstrap') - name: Move homebrew away run: | - (cd /usr/local && for a in bin etc include lib opt sbin share; do sudo mv $a $a-moved; done) + (cd $(brew --prefix) && for a in bin etc include lib opt sbin share; do sudo mv $a $a-moved; done) - name: Select Xcode version run: | if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi From c885e540527e51de7fc9b5e9c45bf02bfc644ffb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 17:02:18 -0800 Subject: [PATCH 06/17] .github/workflows/ci-macos.yml (local-nohomebrew): Add self-destruct sequence --- .github/workflows/ci-macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 13c0a83e1a9..bac51f2d592 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -140,6 +140,7 @@ jobs: # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts. # For doctesting, we use a lower parallelization to avoid timeouts. run: | + (sleep 20000; pkill make) & MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | From 50d1eafe5e7f54485db7eda6ee7d4ac6fe0b7fe2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 17:24:06 -0800 Subject: [PATCH 07/17] .github/workflows/macos.yml: Add input 'timeout' --- .github/workflows/macos.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4a5dd8d5888..20f6b29cf41 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -46,6 +46,10 @@ on: free_disk_space: default: false type: boolean + timeout: + description: 'Elapsed time (seconds) at which to kill the build' + default: 20000 + type: number # # For use in upstream CIs. # @@ -134,6 +138,7 @@ jobs: *) export TARGETS_PRE="${{ inputs.targets_pre }}" TARGETS="${{ inputs.targets }} TARGETS_OPTIONAL="${{ inputs.targets_optional }} ;; esac + (sleep ${{ inputs.timeout }}; pkill make) & MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | From 52654ea586f0d30d7906cca63acf0d4e20f27ef0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 21:28:58 -0800 Subject: [PATCH 08/17] .github/workflows/macos.yml: Use pipx to install tox --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 20f6b29cf41..12d50394a39 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -86,7 +86,7 @@ jobs: - name: Install test prerequisites run: | - pip install tox + pipx install tox - name: Download upstream artifact uses: actions/download-artifact@v3 with: From 700a93973230b7a6a66a97258acb5b53ec22e91b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 22:05:20 -0800 Subject: [PATCH 09/17] .github/workflows/macos.yml: Use tox via 'pipx run' --- .github/workflows/macos.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 12d50394a39..93a8f856994 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -83,10 +83,16 @@ jobs: repository: ${{ inputs.sage_repo }} ref: ${{ inputs.sage_ref }} fetch-depth: 10000 - + - uses: actions/setup-python@v5 + # As of 2024-02-03, the macOS M1 runners do not have preinstalled python or pipx. + # Installing pipx follows the approach of https://github.com/pypa/cibuildwheel/pull/1743 + id: python + with: + python-version: "3.8 - 3.12" + update-environment: false - name: Install test prerequisites run: | - pipx install tox + "${{ steps.python.outputs.python-path }}" -m pip install pipx - name: Download upstream artifact uses: actions/download-artifact@v3 with: @@ -139,7 +145,7 @@ jobs: ;; esac (sleep ${{ inputs.timeout }}; pkill make) & - MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS + MAKE="make -j12" "${{ steps.python.outputs.python-path }}" -m pipx run tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME" From c8ee8e7bacda9d42cf9e07752a2e82591fe013c2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 3 Feb 2024 22:14:15 -0800 Subject: [PATCH 10/17] .github/workflows/ci-macos.yml: Set timeout for stage 1 to 4 hours --- .github/workflows/ci-macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index bac51f2d592..1b448c62722 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -34,6 +34,7 @@ jobs: uses: ./.github/workflows/macos.yml with: stage: "1" + timeout: 14400 stage-2: uses: ./.github/workflows/macos.yml From 49fdc95bd06330f64d8ec68743428b21bedce011 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 4 Feb 2024 17:23:21 -0800 Subject: [PATCH 11/17] tox.ini (local-conda): Use 'uname -m' to determine arch for installer --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8f5a381201d..a9ff0423488 100644 --- a/tox.ini +++ b/tox.ini @@ -541,10 +541,11 @@ setenv = local-conda: PATH={env:CONDA_PREFIX}/bin:/usr/bin:/bin:/usr/sbin:/sbin local-conda: CONDA_PKGS_DIRS={env:SHARED_CACHE_DIR}/conda_pkgs local-conda: CONDA_OS=$(uname | sed 's/^Darwin/MacOSX/;') + local-conda: CONDA_ARCH=$(uname -m) local-conda-forge: CONDA_INSTALLER_URL_BASE=https://github.com/conda-forge/miniforge/releases/latest/download/ - local-conda-forge: CONDA_INSTALLER_FILE=Miniforge3-{env:CONDA_OS}-x86_64.sh + local-conda-forge: CONDA_INSTALLER_FILE=Miniforge3-{env:CONDA_OS}-{env:CONDA_ARCH}.sh local-conda-miniconda: CONDA_INSTALLER_URL_BASE=https://repo.anaconda.com/miniconda/ - local-conda-miniconda: CONDA_INSTALLER_FILE=Miniconda3-latest-{env:CONDA_OS}-x86_64.sh + local-conda-miniconda: CONDA_INSTALLER_FILE=Miniconda3-latest-{env:CONDA_OS}-{env:CONDA_ARCH}.sh local-conda: SETENV=. {env:CONDA_PREFIX}/bin/activate base local-conda-environment: CONDA_SAGE_ENVIRONMENT=sage-build local-conda-environment: CONDA_SAGE_ENVIRONMENT_DIR= From 5f4e0fed9740e250011e98934a52825918a3a546 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 5 Feb 2024 12:37:30 -0800 Subject: [PATCH 12/17] .github/workflows/macos.yml: Remove failing test of macos-14-minimal --- .github/workflows/macos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 93a8f856994..7344d839620 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -33,7 +33,6 @@ on: ["14", "", "homebrew-macos-opthomebrew-standard"], ["latest", "", "conda-forge-macos-minimal"], ["latest", "", "conda-forge-macos-standard"], - ["14", "", "conda-forge-macos-minimal"], ["14", "", "conda-forge-macos-standard"]] type: string extra_sage_packages: From 1f2d9f9fbf26a1b4a11dd74de297c4c797b09359 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 5 Feb 2024 17:28:16 -0800 Subject: [PATCH 13/17] build/pkgs/dsdp/spkg-install.in: Increase verbosity --- build/pkgs/dsdp/spkg-install.in | 1 + 1 file changed, 1 insertion(+) diff --git a/build/pkgs/dsdp/spkg-install.in b/build/pkgs/dsdp/spkg-install.in index 8c29005b9b1..5d0ecadf030 100644 --- a/build/pkgs/dsdp/spkg-install.in +++ b/build/pkgs/dsdp/spkg-install.in @@ -1,6 +1,7 @@ cd src cp ../patches/CMakeLists.txt . sdh_cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ -DBUILD_SHARED_LIBS=ON \ -DBLA_VENDOR=OpenBLAS \ -DBLAS_LIBRARIES="$(pkg-config --libs blas)" \ From 628c21958cff440cbf5cdb5753c2d8d94d0439b6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 5 Feb 2024 21:07:50 -0800 Subject: [PATCH 14/17] src/bin/sage-env [macOS]: Detect ld-classic when full XCode is in use --- src/bin/sage-env | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/bin/sage-env b/src/bin/sage-env index 8fcfda48fb6..074392150b9 100644 --- a/src/bin/sage-env +++ b/src/bin/sage-env @@ -373,7 +373,12 @@ if [ -n "$SAGE_LOCAL" ]; then # On OS X, test whether "ld-classic" is present in the installed # version of the command-line tools. If so, we add "-ld_classic" # to LD_FLAGS. See #36599. - if [ "$UNAME" = "Darwin" ] && [ -x "$(xcode-select -p)/usr/bin/ld-classic" ] ; then + # When the full XCode is installed and in use, for example after + # "sudo xcode-select -s /Applications/Xcode.app", then "xcode-select -p" + # gives "/Applications/Xcode.app/Contents/Developer", but "ld-classic" + # is not in the subdirectory "usr/bin/" but rather in the subdirectory + # "Toolchains/XcodeDefault.xctoolchain/usr/bin/". See #37237. + if [ "$UNAME" = "Darwin" ] && [ -x "$(xcode-select -p)/usr/bin/ld-classic" -o -x "$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld-classic" ] ; then LDFLAGS="-L$SAGE_LOCAL/lib -Wl,-ld_classic,-rpath,$SAGE_LOCAL/lib $LDFLAGS" else LDFLAGS="-L$SAGE_LOCAL/lib -Wl,-rpath,$SAGE_LOCAL/lib $LDFLAGS" From d26937b53e15b786cacb770b9b5f59a8cd164ef2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 6 Feb 2024 19:12:32 -0800 Subject: [PATCH 15/17] .github/workflows/ci-macos.yml (dist): Do not use tag to bump the version --- .github/workflows/ci-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 1b448c62722..d1a5d62f425 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -91,7 +91,7 @@ jobs: run: | git config --global user.email "nobody@example.com" git config --global user.name "Sage GitHub CI" - SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe --tags) || echo "(ignoring error)" + SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(cat src/VERSION.txt).dev0 || echo "(ignoring error)" - name: make dist run: | ./configure --enable-download-from-upstream-url && make dist From d47f1001c97bb7135733b0a4ada32772f0b1dc5c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 6 Feb 2024 20:55:23 -0800 Subject: [PATCH 16/17] src/bin/sage-env: Do not set the ld-classic flag when LD is set --- src/bin/sage-env | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/sage-env b/src/bin/sage-env index 074392150b9..61901897bcd 100644 --- a/src/bin/sage-env +++ b/src/bin/sage-env @@ -127,7 +127,7 @@ fi # The compilers are set in order of priority by # 1) environment variables # 2) compiler installed by sage -# 3) compiler set at configuration time +# 3) compiler set at configuration time if [ -z "$CC" ]; then if [ -n "$SAGE_LOCAL" -a -x "$SAGE_LOCAL/bin/gcc" ]; then CC=gcc @@ -378,7 +378,9 @@ if [ -n "$SAGE_LOCAL" ]; then # gives "/Applications/Xcode.app/Contents/Developer", but "ld-classic" # is not in the subdirectory "usr/bin/" but rather in the subdirectory # "Toolchains/XcodeDefault.xctoolchain/usr/bin/". See #37237. - if [ "$UNAME" = "Darwin" ] && [ -x "$(xcode-select -p)/usr/bin/ld-classic" -o -x "$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld-classic" ] ; then + # However, if LD is set explicitly, as it is within conda on macOS, + # do not not do this. + if [ "$UNAME" = "Darwin" ] && [ -z "$LD" ] && [ -x "$(xcode-select -p)/usr/bin/ld-classic" -o -x "$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld-classic" ] ; then LDFLAGS="-L$SAGE_LOCAL/lib -Wl,-ld_classic,-rpath,$SAGE_LOCAL/lib $LDFLAGS" else LDFLAGS="-L$SAGE_LOCAL/lib -Wl,-rpath,$SAGE_LOCAL/lib $LDFLAGS" From 2165bef51e682d8eeb80e85a6db1ea340b9d202f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 19 Feb 2024 22:09:03 -0800 Subject: [PATCH 17/17] .github/workflows/macos.yml: Pass inputs.extra_sage_packages to tox --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7344d839620..ccba31a3dcc 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -144,7 +144,7 @@ jobs: ;; esac (sleep ${{ inputs.timeout }}; pkill make) & - MAKE="make -j12" "${{ steps.python.outputs.python-path }}" -m pipx run tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS + MAKE="make -j12" EXTRA_SAGE_PACKAGES="${{ inputs.extra_sage_packages }}" "${{ steps.python.outputs.python-path }}" -m pipx run tox -e $TOX_ENV -- SAGE_NUM_THREADS=6 $TARGETS - name: Prepare logs artifact run: | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"