Merge branch 'develop' into hom-count

.ci/create-changes-html.sh

@@ -82,7 +82,7 @@ for block in diff_blocks:
         if content:
             with open(file_path, 'w') as file:
                 file.writelines(content)
-        path = 'html/' + doc
+        path = doc
         hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
         out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
                             + '\n<pre><code class="language-diff">'

.ci/retrofit-worktree.sh

@@ -12,11 +12,13 @@ export GIT_AUTHOR_EMAIL="ci-sage@example.com"
 export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
 export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
 
+set -e
+
 # Set globally for other parts of the workflow
 git config --global user.name "$GIT_AUTHOR_NAME"
 git config --global user.email "$GIT_AUTHOR_EMAIL"
 
-set -ex
+set -x
 
 # If actions/checkout downloaded our source tree using the GitHub REST API
 # instead of with git (because do not have git installed in our image),
@@ -36,6 +38,10 @@ git tag -f new
 # The changed files now show up as uncommitted changes.
 # The final "git add -N" makes sure that files that were added in "new" do not show
 # as untracked files, which would be removed by "git clean -fx".
+if [ -L $WORKTREE_NAME ]; then
+    rm -f $WORKTREE_NAME
+    git worktree prune --verbose
+fi
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME

.ci/write-dockerfile.sh

@@ -1,20 +1,41 @@
 #! /usr/bin/env bash
-## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
-## and satisfy the requirements tested by spkg-configure.m4
-## This is called by $SAGE_ROOT/tox.ini
 set -e
-SYSTEM="${1:-debian}"
 shopt -s extglob
+##
+## Write a Dockerfile for portability testing to stdout.
+##
+## This script needs to be run from SAGE_ROOT (root of the Sage repository).
+## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...'
+##
+## Positional arguments:
+##
+SYSTEM="${1:-debian}"
 SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
-#
+##
+## Environment variables that take influence:
+##
+## - BOOTSTRAP
+## - CONFIGURE_ARGS
+## - DEVTOOLSET
+## - DIST_UPGRADE
+## - DOCKER_BUILDKIT
+## - EXTRA_PATH
+## - EXTRA_REPOSITORIES
+## - EXTRA_SYSTEM_PACKAGES
+## - FULL_BASE_IMAGE_AND_TAG
+## - SKIP_SYSTEM_PKG_INSTALL
+## - USE_CONDARC
+## - __CHOWN
+## - __SUDO
+##
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
 SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES
-CONFIGURE_ARGS="--enable-option-checking "
+SYSTEM_CONFIGURE_ARGS="--enable-option-checking "
 for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
     SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG)
     if [ -n "${SYSTEM_PACKAGE}" ]; then
@@ -24,7 +45,7 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_
             # shell-quote package if necessary
             SYSTEM_PACKAGES+=$(printf " %q" "$a")
         done
-        CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
+        SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
 echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh"
@@ -247,19 +268,17 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-
 $ADD config/config.rpath /new/config/config.rpath
 $ADD src/doc/bootstrap /new/src/doc/bootstrap
 $ADD src/bin /new/src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
+$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
 $ADD m4 /new/m4
 $ADD pkgs /new/pkgs
 $ADD build /new/build
 $ADD .ci /new/.ci
 $ADD .upstream.d /new/.upstream.d
 RUN if [ -d /sage ]; then                                               \
         echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
-        if command -v git; then                                         \
-            (cd /new &&                                                 \
-             echo /src >> .gitignore &&                                 \
-             ./.ci/retrofit-worktree.sh worktree-image /sage);          \
-        else                                                            \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
+        if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
             for a in local logs; do                                     \
                 if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
             done;                                                       \
@@ -271,21 +290,21 @@ RUN if [ -d /sage ]; then                                               \
     fi
 WORKDIR /sage
 
-ARG BOOTSTRAP=${BOOTSTRAP-./bootstrap}
+ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
 
 FROM bootstrapped as configured
 #:configuring:
 RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
-ARG EXTRA_CONFIGURE_ARGS="${CONFIGURE_ARGS}"
+ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}"
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; echo "::group::config.log"; cat config.log; echo "::endgroup::"; ./configure --enable-build-as-root; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; echo "********** configuring without forcing ***********"; ./configure \${CONFIGURE_ARGS}; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 else
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 fi
 cat <<EOF
@@ -318,12 +337,11 @@ ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
 $ADD .gitignore /new/.gitignore
 $ADD src /new/src
-RUN if command -v git; then                             \
-        cd /new && rm -rf .git &&                       \
-        ./.ci/retrofit-worktree.sh worktree-pre /sage;  \
-    else                                                \
-        rm -rf /sage/src;                               \
-        mv /new/src /sage/src;                          \
+RUN cd /new && rm -rf .git && \
+    if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        rm -rf /sage/src;                                    \
+        mv src /sage/src;                                    \
+        cd /sage && ./bootstrap && ./config.status;          \
     fi
 
 ARG TARGETS="build"

.github/workflows/dist.yml

@@ -166,7 +166,7 @@ jobs:
             arch: x86_64
           - os: ubuntu-latest
             arch: i686
-          - os: macos-latest
+          - os: macos-13
             arch: x86_64
           - os: macos-14
             arch: arm64
@@ -175,16 +175,16 @@ jobs:
       # SPKGs to install as system packages
       SPKGS: _bootstrap _prereq
       # Non-Python packages to install as spkgs
-      TARGETS_PRE: gmpy2-build-deps
+      TARGETS_PRE: gmp mpfr mpc bliss coxeter3 mcqd meataxe sirocco boost_cropped tdlib
       # Disable building PyPy wheels on all platforms
       # Disable musllinux until #33083 provides alpine package information
       CIBW_SKIP: "pp* *-musllinux*"
       #
       CIBW_ARCHS: ${{ matrix.arch }}
       # https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python
-      CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9, <3.12"
+      CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9, <3.13"
       # Environment during wheel build
-      CIBW_ENVIRONMENT: "PATH=$(pwd)/local/bin:$PATH CPATH=$(pwd)/local/include:$CPATH LIBRARY_PATH=$(pwd)/local/lib:$LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/local/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal"
+      CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt"
       # Use 'build', not 'pip wheel'
       CIBW_BUILD_FRONTEND: build
     steps:
@@ -203,23 +203,36 @@ jobs:
           python-version: "3.8 - 3.12"
           update-environment: false
 
+      - name: Bootstrap
+        run: |
+          export PATH=$(pwd)/build/bin:$PATH
+          eval $(sage-print-system-package-command auto --sudo --yes --no-install-recommends --spkg install _bootstrap)
+          ./bootstrap
+
       - name: Build platform wheels
         # We build the wheels from the sdists so that MANIFEST filtering becomes effective.
         # But we must run cibuildwheel with the unpacked source directory, not a tarball,
         # so that SAGE_ROOT is copied into the build containers.
         #
         # In the CIBW_BEFORE_ALL phase, we install libraries using the Sage distribution.
         # https://cibuildwheel.readthedocs.io/en/stable/options/#before-all
-        # This is unfortunately repeated for each of the packages that we build wheels for
+        # For Linux, this is repeated for each of the packages that we build wheels for
         # because CIBW starts with a fresh container on each invocation.
+        # Therefore we cache it in a directory mounted from the host: /host
+        # https://cibuildwheel.pypa.io/en/stable/faq/#linux-builds-in-containers
+        #
+        # omit sagemath-{meataxe,sirocco} for now -- needs sagemath-modules
         run: |
           "${{ steps.python.outputs.python-path }}" -m pip install pipx
           export PATH=build/bin:$PATH
-          export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || echo error ignored) ) && ./bootstrap && ./configure --enable-build-as-root && make -j4 V=0 $TARGETS_PRE"
+          export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt"
           mkdir -p unpacked
-          for pkg in sagemath-objects sagemath-categories; do
+          for pkg in sagemath*objects sagemath*categories sagemath*bliss sagemath*coxeter3 sagemath*mcqd sagemath*tdlib; do
+              case "$pkg:${{ matrix.arch }}" in
+                  sagemath*tdlib:i686) continue;;  # broken - boost-related
+              esac
               (cd unpacked && tar xfz - ) < dist/$pkg*.tar.gz
-              "${{ steps.python.outputs.python-path }}" -m pipx run cibuildwheel==2.17.0 unpacked/$pkg*
+              "${{ steps.python.outputs.python-path }}" -m pipx run cibuildwheel==2.18.0 unpacked/$pkg*
           done
 
       - uses: actions/upload-artifact@v4

.github/workflows/docker.yml

@@ -25,7 +25,6 @@ on:
            "ubuntu-lunar",
            "ubuntu-mantic",
            "ubuntu-noble",
-           "debian-buster-gcc_spkg",
            "debian-bullseye",
            "debian-bookworm",
            "debian-trixie",

.github/workflows/docker_hub.yml

@@ -10,6 +10,9 @@ on:
         default: make-build
         type: string
 
+env:
+  CAN_LOGIN: ${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
+
 jobs:
   build-and-push:
     name: Build Docker image and push to DockerHub
@@ -71,11 +74,12 @@ jobs:
         if: env.JOB_DONE == 'false'
 
       - name: Login to Docker Hub
+        id: login
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: env.JOB_DONE == 'false'
+        if: env.JOB_DONE == 'false' && env.CAN_LOGIN == 'true'
 
       - name: Build and push make-build
         uses: docker/build-push-action@v5
@@ -85,8 +89,9 @@ jobs:
           target: ${{ inputs.dockerfile_target }}
           build-args: |
             MAKE_BUILD=${{ env.BASE }}
-          push: true
+          push: ${{ steps.login.outcome == 'success' }}
           tags: ${{ env.TAG_LIST }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-        if: env.JOB_DONE == 'false'
+        # run even if there are no push credentials (just don't push)
+        if: (success() || failure()) && env.JOB_DONE == 'false'

.github/workflows/lint.yml

@@ -37,7 +37,14 @@ jobs:
       id: deps
       run: pip install tox
 
-    - name: Code style check with pycodestyle
+    - name: Code style check with ruff-minimal
+      if: (success() || failure()) && steps.deps.outcome == 'success'
+      run: tox -e ruff-minimal
+      env:
+        # https://github.com/ChartBoost/ruff-action/issues/7#issuecomment-1887780308
+        RUFF_OUTPUT_FORMAT: github
+
+    - name: Code style check with pycodestyle-minimal
       if: (success() || failure()) && steps.deps.outcome == 'success'
       run: tox -e pycodestyle-minimal
 

.gitignore

@@ -204,6 +204,8 @@ build/pkgs/wheel/version_requirements.txt
 /pkgs/*/MANIFEST
 /pkgs/*/*.egg-info
 /pkgs/*/.tox
+/pkgs/*/venv*
+/pkgs/*/.venv*
 
 /pkgs/sage-conf_pypi/sage_root/config.log
 /pkgs/sage-conf_pypi/sage_root/config.status

CITATION.cff

@@ -4,8 +4,8 @@ title: SageMath
 abstract: SageMath is a free open-source mathematics software system.
 authors:
 - name: "The SageMath Developers"
-version: 10.4.beta6
+version: 10.4.beta7
 doi: 10.5281/zenodo.8042260
-date-released: 2024-05-12
+date-released: 2024-05-25
 repository-code: "https://github.com/sagemath/sage"
 url: "https://www.sagemath.org/"

Makefile

@@ -351,6 +351,7 @@ ptestoptionallong-nodoc:
 # CONFIGURE_DEPENDENCIES is the list of files that influence the generation of 'configure'.
 CONFIGURE_DEPENDENCIES =							\
 	configure.ac src/bin/sage-version.sh m4/*.m4				\
+	src/pyproject.toml							\
 	build/pkgs/*/spkg-configure.m4						\
 	build/pkgs/*/type build/pkgs/*/SPKG.rst					\
 	build/pkgs/*/checksums.ini build/pkgs/*/requirements.txt		\

VERSION.txt

@@ -1 +1 @@
-SageMath version 10.4.beta6, Release Date: 2024-05-12
+SageMath version 10.4.beta7, Release Date: 2024-05-25