diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..8fb235d7 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes index 897b242c..965a20f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ World\ population.ipynb linguist-documentation +.git_archival.txt export-subst diff --git a/.github/codecov.yml b/.github/codecov.yml index 9dc3686b..897013a1 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,9 +1,6 @@ codecov: notify: - after_n_builds: 16 - -comment: - after_n_builds: 16 + wait_for_ci: true coverage: status: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae67738a..07c7fd6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,9 @@ on: push: paths-ignore: - "CHANGELOG.md" - branches: [main] + branches: [ main ] pull_request: - branches: [main] + branches: [ main ] schedule: - cron: "0 11 * * 4" @@ -20,10 +20,6 @@ permissions: # All nested workflows will inherit these permissions and so no need to declare # in each step file contents: read - # Cannot use it in codeql nested workflow without declaring it on - # top level workflow - # Ref: https://docs.github.com/en/actions/using-workflows/reusing-workflows#access-and-permissions - security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -33,28 +29,47 @@ jobs: pre-commit: uses: ./.github/workflows/step_pre-commit.yml - codeql: - needs: [pre-commit] + static-analysis: + needs: [ pre-commit ] uses: ./.github/workflows/step_static-analysis.yml + permissions: + contents: read + security-events: write test-pip: - needs: [codeql] + needs: [ pre-commit ] uses: ./.github/workflows/step_tests-pip.yml + with: + coverage: ${{ github.event_name != 'schedule' }} - test-unit-functional-integration: - needs: [codeql] - uses: ./.github/workflows/step_test_unit_functional.yml + coverage: + needs: [ test-pip ] + uses: ./.github/workflows/step_coverage.yml + if: github.event_name != 'schedule' test-conda: - needs: [codeql] + needs: [ test-pip ] uses: ./.github/workflows/step_tests-conda.yml + with: + coverage: ${{ github.event_name != 'schedule' }} test-ui: - needs: [codeql] + needs: [ test-pip ] uses: ./.github/workflows/step_tests-ui.yml build: - needs: [test-pip, test-conda, test-unit-functional-integration, test-ui] + needs: [ test-pip, test-conda, test-ui ] uses: ./.github/workflows/step_build.yml with: upload: ${{ inputs.upload-build-artifacts || false }} + + pass: + name: Pass + needs: [ pre-commit, static-analysis, test-pip, coverage, test-conda, test-ui, build ] + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + allowed-skips: coverage + if: always() diff --git a/.github/workflows/comment-pr.yml b/.github/workflows/comment-pr.yml index db2cd9d5..267dcbb7 100644 --- a/.github/workflows/comment-pr.yml +++ b/.github/workflows/comment-pr.yml @@ -12,7 +12,7 @@ jobs: name: Comment PR steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Comment PR uses: thollander/actions-comment-pull-request@v2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 01278329..00000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Publish -on: - push: - tags: - - "v[0-9]+.[0-9]+.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+dev[0-9]+" - -jobs: - pre-commit: - uses: ./.github/workflows/step_pre-commit.yml - - codeql: - needs: [ pre-commit ] - uses: ./.github/workflows/step_static-analysis.yml - - test-pip: - needs: [ codeql ] - uses: ./.github/workflows/step_tests-pip.yml - - test-unit-functional-integration: - needs: [ codeql ] - uses: ./.github/workflows/step_test_unit_functional.yml - - test-conda: - needs: [ codeql ] - uses: ./.github/workflows/step_tests-conda.yml - - test-ui: - needs: [ codeql ] - uses: ./.github/workflows/step_tests-ui.yml - - build: - needs: [ test-pip, test-conda, test-unit-functional-integration, test-ui ] - uses: ./.github/workflows/step_build.yml - - publish: - needs: [ build ] - runs-on: ubuntu-latest - - environment: - name: pypi - url: https://pypi.org/p/jupytext - - permissions: - id-token: write - - steps: - - name: Checkout source - uses: actions/checkout@v3 - - - name: Base Setup - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - - name: Build package - run: | - python -m pip install wheel build - python -m build - - - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..81ffc0e3 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,96 @@ +name: Release +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+dev[0-9]+" + workflow_dispatch: + inputs: + skip-tests: + type: boolean + description: Skip + default: false + ref: + type: string + description: Tag to release + required: true + +permissions: + contents: read + +jobs: + pre-commit: + uses: ./.github/workflows/step_pre-commit.yml + if: "! inputs.skip-tests" + + test-pip: + needs: [ pre-commit ] + uses: ./.github/workflows/step_tests-pip.yml + with: + coverage: false + if: "! inputs.skip-tests" + + test-conda: + needs: [ test-pip ] + uses: ./.github/workflows/step_tests-conda.yml + with: + coverage: false + if: "! inputs.skip-tests" + + test-ui: + needs: [ test-pip ] + uses: ./.github/workflows/step_tests-ui.yml + if: "! inputs.skip-tests" + + test-status: + needs: [ pre-commit, test-pip, test-conda, test-ui ] + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@release/v1 + with: + allowed-skips: pre-commit, test-pip, test-conda, test-ui + jobs: ${{ toJSON(needs) }} + if: always() + + build: + needs: [ test-status ] + uses: ./.github/workflows/step_build.yml + with: + upload: true + ref: ${{ inputs.ref }} + + publish: + needs: [ build ] + runs-on: ubuntu-latest + + environment: + name: pypi + url: https://pypi.org/p/jupytext + + permissions: + contents: read + id-token: write + + steps: + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + + release: + needs: [ publish ] + name: Create release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - uses: softprops/action-gh-release@v1 + with: + name: Jupytext ${{ inputs.ref || github.ref_name }} + draft: true + prerelease: ${{ contains(inputs.ref || github.ref, 'rc') }} + generate_release_notes: true diff --git a/.github/workflows/step_build.yml b/.github/workflows/step_build.yml index d5cb4780..8ee9106e 100644 --- a/.github/workflows/step_build.yml +++ b/.github/workflows/step_build.yml @@ -1,5 +1,5 @@ name: build -run-name: Build test +run-name: Build package on: workflow_call: @@ -9,46 +9,29 @@ on: required: false default: false description: Upload build artifacts + ref: + type: string + description: Tag to build -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build - cancel-in-progress: true +permissions: + contents: read jobs: build: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: "3.x" - name: Build package - run: | - python -m pip install build wheel - - # NOTE: These builds and verifications of the builds can be done more - # robustly with jupyter-releaser. - # - # Removed the check on size of package as we are distributing tests/ with - # sdist now and they are around 8MB. Seems like original check was to make - # sure we are not distributing node_modules and we are quite safe with that - # with hatch build system. - # - # Build jupytext package - python -m build - - # Build lab extension(s) - npm pack --pack-destination dist jupyterlab/packages/* - - # Check that the lab is there - if (($(tar -tf dist/*.tar.gz | grep jupyterlab/jupyterlab_jupytext/labextension/package.json$ | wc -l)==0)); then echo "Missing jupyterlab_jupytext" && exit 1; fi - - # Install package and extension - python -m pip install dist/*.tar.gz - - echo "Install went OK" + run: hatch build - name: Archive build artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/step_test_unit_functional.yml b/.github/workflows/step_coverage.yml similarity index 58% rename from .github/workflows/step_test_unit_functional.yml rename to .github/workflows/step_coverage.yml index 1bf3c9ce..8220545d 100644 --- a/.github/workflows/step_test_unit_functional.yml +++ b/.github/workflows/step_coverage.yml @@ -1,41 +1,40 @@ -name: test-categories -run-name: Run Unit/Functional/Integration and External tests using Pip +name: coverage +run-name: Check coverage on: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-test_classification - cancel-in-progress: true +permissions: + contents: read jobs: - test-pip: - continue-on-error: ${{ matrix.experimental || false }} + coverage: + name: > + ${{ matrix.coverage }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: [ "3.11" ] coverage: [unit, functional, integration, external] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 with: - python_version: ${{ matrix.python-version }} + python_version: 3.x - name: Install from source - run: HATCH_BUILD_HOOKS_ENABLE=false python -m pip install -e '.[test-cov,test-${{ matrix.coverage }}]' markdown-it-py~=3.0 + run: HATCH_BUILD_HOOKS_ENABLE=false python -m pip install -e '.[test-cov,test-${{ matrix.coverage }}]' - name: Install a Jupyter Kernel run: python -m ipykernel install --name python_kernel --user - name: Run the tests - run: pytest tests/${{ matrix.coverage }} --cov + run: pytest tests/${{ matrix.coverage }} -n logical --cov --cov-report=xml - name: Upload the coverage uses: codecov/codecov-action@v3 diff --git a/.github/workflows/step_pre-commit.yml b/.github/workflows/step_pre-commit.yml index 11bb503e..a154d2b8 100644 --- a/.github/workflows/step_pre-commit.yml +++ b/.github/workflows/step_pre-commit.yml @@ -1,19 +1,18 @@ name: pre-commit -run-name: Pre-commit +run-name: Run pre-commit tests on: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-pre-commit - cancel-in-progress: true +permissions: + contents: read jobs: pre-commit: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -26,10 +25,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: "3.x" # Current repo organization will not permit to set up pre-commit config for # TS lint related packages due to absence of package.json in the top level diff --git a/.github/workflows/step_static-analysis.yml b/.github/workflows/step_static-analysis.yml index deee225d..14a0d845 100644 --- a/.github/workflows/step_static-analysis.yml +++ b/.github/workflows/step_static-analysis.yml @@ -4,9 +4,9 @@ run-name: Run CodeQL analysis on: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-static-analysis - cancel-in-progress: true +permissions: + contents: read + security-events: write jobs: codeql: @@ -14,7 +14,7 @@ jobs: continue-on-error: true steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/step_tests-conda.yml b/.github/workflows/step_tests-conda.yml index 1f00c1a2..f27f2b5a 100644 --- a/.github/workflows/step_tests-conda.yml +++ b/.github/workflows/step_tests-conda.yml @@ -1,19 +1,25 @@ name: test-conda -run-name: Run tests using Conda +run-name: Run Conda tests for different OS on: workflow_call: + inputs: + coverage: + type: boolean + description: Upload coverage to CodeCov + default: true -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-conda-test - cancel-in-progress: true +permissions: + contents: read jobs: test-conda: + name: > + ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] - python-version: [ 3.11 ] runs-on: ${{ matrix.os }} defaults: run: @@ -21,21 +27,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache conda uses: actions/cache@v3 with: path: ~/conda_pkgs_dir - key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('.ci/environment-ci.yml') }} + key: conda-${{ matrix.os }}-${{ hashFiles('.ci/environment-ci.yml') }} - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true auto-activate-base: false activate-environment: jupytext-ci - python-version: ${{ matrix.python-version }} channels: defaults,conda-forge environment-file: .ci/environment-ci.yml use-only-tar-bz2: true @@ -57,13 +62,11 @@ jobs: python -m jupyterlab.browser_check - name: Test with pytest - # We use an explicit coverage file otherwise the - # codecov action below does not find it - run: pytest --cov . --cov-report xml:coverage.xml + run: pytest -n logical --cov --cov-report=xml - name: Upload coverage uses: codecov/codecov-action@v3 with: fail_ci_if_error: true verbose: true - files: coverage.xml + if: inputs.coverage diff --git a/.github/workflows/step_tests-pip.yml b/.github/workflows/step_tests-pip.yml index d4622b70..eb3ccb52 100644 --- a/.github/workflows/step_tests-pip.yml +++ b/.github/workflows/step_tests-pip.yml @@ -1,29 +1,40 @@ name: test-pip -run-name: Run tests using Pip +run-name: Run main tests using Pip on: workflow_call: + inputs: + coverage: + type: boolean + description: Upload coverage to CodeCov + default: true -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-pip-test - cancel-in-progress: true +permissions: + contents: read jobs: test-pip: + name: > + 🐍 ${{ matrix.python-version }} + ${{ matrix.dependency_type && format('({0})', matrix.dependency_type) }} + ${{ matrix.markdown-it-py && format('(markdown-it-py {0})', matrix.markdown-it-py) }} + ${{ matrix.quarto && '(Quarto)' }} + ${{ matrix.no_kernel && '(No kernel)' }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12"] - markdown-it-py-version: ["~=3.0"] - pip-flags: [""] include: + # Test pre-release versions - python-version: "3.x" - pip-flags: "--pre --upgrade --upgrade-strategy eager" + dependency_type: "pre" + # Test minimum markdown-it-py supported (otherwise the most recent version is used) - python-version: "3.x" - markdown-it-py-version: "~=2.0" + markdown-it-py: "~=2.0" - python-version: "3.x" - markdown-it-py-version: "" + markdown-it-py: false - python-version: "3.x" no_kernel: true - python-version: "3.x" @@ -31,24 +42,28 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 with: python_version: ${{ matrix.python-version }} + dependency_type: ${{ matrix.dependency_type }} - name: Install from source - run: python -m pip install -e '.[test-cov,test-external]' jupyterlab ${{ matrix.markdown-it-py-version && format('markdown-it-py{0}', matrix.markdown-it-py-version) }} ${{ matrix.pip-flags }} + run: > + python -m pip install + -e '.[test-cov,test-external]' + jupyterlab + ${{ matrix.markdown-it-py && format('markdown-it-py{0}', matrix.markdown-it-py) || '' }} - name: Install a Jupyter Kernel if: ${{ !matrix.no_kernel }} run: python -m ipykernel install --name python_kernel --user - - name: Uninstall markdown-it-py # Markdown-it-py is a dependency of Jupytext, # but Jupytext should still work if it is not installed - if: ${{ matrix.markdown-it-py-version == '' }} + if: ${{ matrix.markdown-it-py == 'false' }} run: python -m pip uninstall markdown-it-py --yes - name: Install Quarto @@ -69,10 +84,11 @@ jobs: python -m jupyterlab.browser_check - name: Test with pytest - run: pytest --cov + run: pytest -n logical --cov --cov-report=xml - name: Upload coverage uses: codecov/codecov-action@v3 with: fail_ci_if_error: true verbose: true + if: inputs.coverage diff --git a/.github/workflows/step_tests-ui.yml b/.github/workflows/step_tests-ui.yml index 2ba28b14..abc48c27 100644 --- a/.github/workflows/step_tests-ui.yml +++ b/.github/workflows/step_tests-ui.yml @@ -4,9 +4,8 @@ run-name: Run UI tests with Galata on: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-ui-test - cancel-in-progress: true +permissions: + contents: read jobs: test-ui: @@ -15,10 +14,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: 3.x - name: Install from source run: python -m pip install -e '.[test-ui]' diff --git a/pyproject.toml b/pyproject.toml index a89176d1..ddd60caf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,8 @@ classifiers = [ ] dependencies = [ "nbformat", - "mdit_py_plugins", - "markdown-it-py>=1.0.0", + "mdit-py-plugins", + "markdown-it-py>=2.0", "packaging", "pyyaml", "toml", @@ -86,7 +86,7 @@ test-cov = [ # Galata test env # Install a non python kernel to ensure extension is working as expected test-ui = [ - "calysto_bash", + "calysto-bash", ] dev = [ "jupytext[test-cov,test-external]", @@ -94,9 +94,9 @@ dev = [ # Documentation dependencies docs = [ "sphinx", - "sphinx_copybutton", - "sphinx_rtd_theme", - "myst_parser", + "sphinx-copybutton", + "sphinx-rtd-theme", + "myst-parser", ] [project.scripts] @@ -122,7 +122,6 @@ path = "src/jupytext/version.py" # Following config is related to JupyterLab extension [tool.hatch.build.targets.sdist] artifacts = ["jupyterlab/jupyterlab_jupytext/labextension"] -exclude = ["demo", "**/.yarn", "**/node_modules"] [tool.hatch.build.targets.wheel] packages = ["src/jupytext", "src/jupytext_config", "jupyterlab/jupyterlab_jupytext"] @@ -140,7 +139,7 @@ enable-by-default = true # Runtime dependency for this build hook # We need jupyterlab as build time depdendency to get jlpm (wrapper around yarn) dependencies = [ - "hatch-jupyter-builder>=0.5", "jupyterlab>=4,<5" + "hatch-jupyter-builder>=0.5", "jupyterlab>=4" ] # We use npm_builder to build the jupyterlab extension build-function = "hatch_jupyter_builder.npm_builder" @@ -267,6 +266,3 @@ omit = [ "tests/*", "src/jupytext/version.py", ] - -[tool.coverage.xml] -output = "coverage.xml"