Merge remote-tracking branch 'upstream/develop' into sagemath_categories_distribution_directives_2024

Author: Matthias Koeppe

.ci/retrofit-worktree.sh

@@ -32,18 +32,14 @@ git tag -f new
 # But $WORKTREE_DIRECTORY is not a git repository.
 # We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
 # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-# Then we update worktree and index with "git reset --hard new".
+# Then we update worktree and index with "git checkout new".
 # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-# The changed files now show up as uncommitted changes.
-# The final "git add -N" makes sure that files that were added in "new" do not show
-# as untracked files, which would be removed by "git clean -fx".
 if [ -L $WORKTREE_NAME ]; then
     rm -f $WORKTREE_NAME
-    git worktree prune --verbose
 fi
+git worktree prune --verbose
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
 if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
-(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout new && git status)

.ci/write-dockerfile.sh

@@ -88,7 +88,7 @@ case $SYSTEM in
                 cat <<EOF
 RUN if command -v unminimize > /dev/null; then  \
         (yes | unminimize) || echo "(ignored)"; \
-        rm -f "$(command -v unminimize)";       \
+        rm -f "\$(command -v unminimize)";       \
     fi
 EOF
                 if [ -n "$DIST_UPGRADE" ]; then
@@ -272,13 +272,14 @@ $ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4
 $ADD m4 /new/m4
 $ADD pkgs /new/pkgs
 $ADD build /new/build
-$ADD .ci /new/.ci
 $ADD .upstream.d /new/.upstream.d
+ADD .ci /.ci
 RUN if [ -d /sage ]; then                                               \
         echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
         printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
         printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
-        if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
+        if ! (cd /new && /.ci/retrofit-worktree.sh worktree-image /sage); then \
+            echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \
             for a in local logs; do                                     \
                 if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
             done;                                                       \
@@ -337,8 +338,12 @@ ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
 $ADD .gitignore /new/.gitignore
 $ADD src /new/src
+ADD .ci /.ci
 RUN cd /new && rm -rf .git && \
-    if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
+    if /.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        cd /sage && touch configure build/make/Makefile; \
+    else \
+        echo "retrofit-worktree.sh failed, falling back to replacing /sage/src"; \
         rm -rf /sage/src;                                    \
         mv src /sage/src;                                    \
         cd /sage && ./bootstrap && ./config.status;          \

.github/workflows/build.yml

@@ -175,11 +175,20 @@ jobs:
           ./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          # File extensions per sage.doctest.control.skipfile
+          files: src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}
+          files_ignore: src/{setup,conftest*}.py
+
       - name: Test changed files (sage -t --new)
+        if: steps.changed-files.outputs.all_changed_files
         run: |
           export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
-          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
-          ./sage -t --new -p4
+          # https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
+          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.all_changed_files }}
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
   test-mod:
@@ -283,6 +292,14 @@ jobs:
         image: registry:2
         ports:
           - 5000:5000
+    strategy:
+      fail-fast: false
+      matrix:
+        tests:
+          - "src/sage/[a-f]*"
+          - "src/sage/[g-o]*"
+          - "src/sage/[p-z]*"
+          - "src/doc src/sage_docbuild src/sage_setup"
     steps:
       - name: Maximize build disk space
         uses: easimon/maximize-build-space@v10
@@ -356,23 +373,138 @@ jobs:
 
       # Testing
 
-      - name: Test all files (sage -t --all --long)
+      - name: Test all files (sage -t --long ${{ matrix.tests }})
         run: |
+          mkdir .coverage
+          rm -rf /sage/.coverage
+          ln -s $(pwd)/.coverage /sage/
+          cd /sage
           ./sage -python -m pip install coverage
-          ./sage -python -m coverage run --rcfile=src/tox.ini src/bin/sage-runtests --all --long -p4 --format github --random-seed=286735480429121101562228604801325644303
-        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
+          ./sage -python -m coverage run --rcfile=src/tox.ini src/bin/sage-runtests --force-lib --long -p4 --format github --random-seed=286735480429121101562228604801325644303 ${{ matrix.tests }}
+        shell: sh .ci/docker-exec-script.sh BUILD . {0}
 
-      - name: Copy coverage results
+      - name: Combine coverage results
         if: (success() || failure()) && steps.container.outcome == 'success'
         run: |
           ./sage -python -m coverage combine --rcfile=src/tox.ini
-          ./sage -python -m coverage xml --rcfile=src/tox.ini
-          mkdir -p coverage-report
-          mv coverage.xml coverage-report/
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Upload coverage to codecov
+      - name: Prepare upload
+        id: copy-coverage
         if: (success() || failure()) && steps.container.outcome == 'success'
+        run: |
+          echo tests_id=$(echo "${{ matrix.tests }}" | sed -E 's, +,--,g;s,/,_,g;s/[^-_A-Za-z]//g;') >> "$GITHUB_OUTPUT"
+
+      - name: Upload coverage results
+        if: (success() || failure()) && steps.container.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ steps.copy-coverage.outputs.tests_id }}
+          path: .coverage
+
+  coverage-report:
+    runs-on: ubuntu-latest
+    needs: [test-long]
+    if: (success() || failure())
+    services:
+      # https://docs.docker.com/build/ci/github-actions/local-registry/
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Maximize build disk space
+        uses: easimon/maximize-build-space@v10
+        with:
+          # need space in /var for Docker images
+          root-reserve-mb:      30000
+          remove-dotnet:        true
+          remove-android:       true
+          remove-haskell:       true
+          remove-codeql:        true
+          remove-docker-images: true
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Install test prerequisites
+        # From docker.yml
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
+      - name: Merge CI fixes from sagemath/sage
+        # From docker.yml
+        # This step needs to happen after the commit sha is put in DOCKER_TAG
+        # so that multi-stage builds can work correctly.
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      # Building
+
+      - name: Generate Dockerfile
+        # From docker.yml
+        run: |
+          tox -e ${{ env.TOX_ENV }}
+          cp .tox/${{ env.TOX_ENV }}/Dockerfile .
+        env:
+          # Only generate the Dockerfile, do not run 'docker build' here
+          DOCKER_TARGETS: ""
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
+      - name: Build Docker image
+        id: image
+        uses: docker/build-push-action@v5
+        with:
+          push:       true
+          load:       false
+          context:    .
+          tags:       ${{ env.BUILD_IMAGE }}
+          target:     with-targets
+          cache-from: type=gha
+          cache-to:   type=gha,mode=max
+          build-args: |
+            NUMPROC=6
+            USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
+            TARGETS_PRE=build/make/Makefile
+            TARGETS=ci-build-with-fallback
+
+      - name: Start container
+        id: container
+        run: |
+          docker run --name BUILD -dit \
+                     --mount type=bind,src=$(pwd),dst=$(pwd) \
+                     --workdir $(pwd) \
+                     ${{ env.BUILD_IMAGE }} /bin/sh
+
+      # Combining
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .coverage
+          pattern: coverage-*
+
+      - name: Coverage report
+        # Using --omit to avoid "CoverageWarning: Couldn't parse '/tmp/tmp06qizzie/tmp_ldpu46ob.py': No source for code"
+        run: |
+          rm -rf /sage/.coverage
+          ln -s $(pwd)/.coverage /sage/
+          cd /sage
+          ./sage -python -m pip install coverage
+          ./sage -python -m coverage combine --rcfile=src/tox.ini .coverage/coverage-*/.coverage
+          ./sage -python -m coverage xml --rcfile=src/tox.ini --omit="/tmp/*"
+          mkdir -p .coverage/coverage-report
+          mv coverage.xml .coverage/coverage-report/
+        shell: sh .ci/docker-exec-script.sh BUILD . {0}
+
+      - name: Upload coverage to codecov
         uses: codecov/codecov-action@v4
         with:
-          directory: ./coverage-report
+          directory: .coverage/coverage-report

.github/workflows/ci-macos.yml

@@ -109,7 +109,7 @@ jobs:
       fail-fast: false
       max-parallel: 4
       matrix:
-        os: [ macos-11, macos-12, macos-14 ]
+        os: [ macos-12, macos-13, macos-14 ]
         tox_system_factor: [macos-nobootstrap]
         tox_packages_factor: [minimal]
         xcode_version_factor: [default]

.github/workflows/dist.yml

@@ -240,6 +240,20 @@ jobs:
           name: ${{ matrix.os }}-${{ matrix.arch }}-wheels
           path: ./wheelhouse/*.whl
 
+  upload_wheels:
+    # This needs to be a separate job because pypa/gh-action-pypi-publish cannot run on macOS
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    env:
+      CAN_DEPLOY: ${{ secrets.SAGEMATH_PYPI_API_TOKEN != '' }}
+    steps:
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: "*-*-wheels"
+          path: wheelhouse
+          merge-multiple: true
+
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__

.github/workflows/doc-build.yml

@@ -188,7 +188,7 @@ jobs:
           export SAGE_JUPYTER_SERVER=binder:sagemath/sage-binder-env/dev
           make doc-clean doc-uninstall
           ./config.status && make sagemath_doc_html-no-deps sagemath_doc_pdf-no-deps
-        shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0}
+        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
       - name: Copy live doc
         id: copylivedoc

.github/workflows/macos.yml

@@ -19,21 +19,19 @@ on:
         type: string
       # System configuration
       osversion_xcodeversion_toxenv_tuples:
-        # As of 2024-02, "runs-on: macos-latest" is macos-12.
-        # and "runs-on: macos-14" selects the new M1 runners.
+        # As of 2024-05, "runs-on: macos-latest" and "runs-on: macos-14" selects the new M1 runners.
         # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
         description: 'Stringified JSON object'
         default: >-
-          [["11",     "xcode_13.2.1", "homebrew-macos-usrlocal-minimal"],
-           ["12",     "",             "homebrew-macos-usrlocal-minimal"],
+          [["12",     "",             "homebrew-macos-usrlocal-minimal"],
            ["12",     "",             "homebrew-macos-usrlocal-standard"],
            ["12",     "",             "homebrew-macos-usrlocal-python3_xcode-standard"],
            ["12",     "",             "homebrew-macos-usrlocal-maximal"],
            ["13",     "xcode_15.0",   "homebrew-macos-usrlocal-standard"],
-           ["14",     "",             "homebrew-macos-opthomebrew-standard"],
-           ["latest", "",             "conda-forge-macos-minimal"],
-           ["latest", "",             "conda-forge-macos-standard"],
-           ["14",     "",             "conda-forge-macos-standard"]]
+           ["latest", "",             "homebrew-macos-opthomebrew-standard"],
+           ["13",     "",             "conda-forge-macos-minimal"],
+           ["13",     "",             "conda-forge-macos-standard"],
+           ["latest", "",             "conda-forge-macos-standard"]]
         type: string
       extra_sage_packages:
         description: 'Extra Sage packages to install as system packages'

CITATION.cff

@@ -4,8 +4,8 @@ title: SageMath
 abstract: SageMath is a free open-source mathematics software system.
 authors:
 - name: "The SageMath Developers"
-version: 10.4.beta7
+version: 10.4.beta8
 doi: 10.5281/zenodo.8042260
-date-released: 2024-05-25
+date-released: 2024-06-01
 repository-code: "https://github.com/sagemath/sage"
 url: "https://www.sagemath.org/"

VERSION.txt

@@ -1 +1 @@
-SageMath version 10.4.beta7, Release Date: 2024-05-25
+SageMath version 10.4.beta8, Release Date: 2024-06-01