refactor toxenvs, use OpenAstronomy workflows, and move MacOS jobs to…

… schedule (#75)

* refactor CI

* update `.flake8` configuration to ignore build directory

* run jobs concurrently

* fix

* echo CRDS variables for debugging

* use `ruff`

* syntax fix

* use OpenAstronomy workflow

* remove posargs from workflow

* minor fixes

* split envs into sections

* pass through CI env variable

* run coverage immediately

* move MacOS jobs to schedule

* use environment variables for downstream tests

* replace numpy tests with minimum dependencies test

* pin `asdf>=2.13` to satisfy dependencies

* drop `--ignore-installed`

* ignore scripts in JWST tests

* add change log entry

.flake8

@@ -7,4 +7,7 @@ max-line-length = 130
 exclude =
     docs,
     src/stpipe/extern,
+    build,
+    .tox,
+    .eggs
 ignore = E203, W503, W504, W605

.github/workflows/ci.yml

@@ -12,120 +12,39 @@ on:
     # * is a special character in YAML so you have to quote this string
     - cron: '0 9 * * 1'
 
-env:
-  CRDS_PATH: $HOME/crds_cache
-  CRDS_CLIENT_RETRY_COUNT: 3
-  CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
-
 jobs:
-  style:
-    name: Code style checks
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: style-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install ruff
-      - run: ruff src
-  audit:
-    name: Bandit security audit
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: audit-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install bandit
-      - run: bandit -r -ll src
+  check:
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      envs: |
+        - linux: check-style
+        - linux: check-security
+        - linux: build-dist
   test:
-    name: test
-    needs: [ style, audit ]
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python: [ 3.8, 3.9, '3.10' ]
-        os: [ ubuntu-latest, macos-latest ]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: test-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install ".[test]" pytest-xdist
-      - run: pip freeze
-      - run: pytest -n auto
-  test_with_coverage:
-    name: test base library with coverage
-    needs: [ style, audit ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: test-coverage-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install -e ".[test]" pytest-xdist pytest-cov
-      - run: pip freeze
-      - run: pytest -n auto --cov-report=xml --cov=src/stpipe
-      - run: coverage report -m
-      - uses: codecov/codecov-action@v2
-        with:
-          file: ./coverage.xml
-  test_jwst_with_coverage:
-    name: test JWST pipeline with coverage
-    needs: [ style, audit ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: test-coverage-jwst-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install -e ".[test]" pytest-xdist pytest-cov
-      - run: pip install "jwst[test] @ git+https://github.com/spacetelescope/jwst.git"
-      - run: pip freeze
-      - run: pytest -n auto --cov-report=xml --cov=src/stpipe --ignore-glob=timeconversion --ignore-glob=associations --ignore-glob=scripts --pyargs jwst
-        env:
-          CRDS_SERVER_URL: https://jwst-crds.stsci.edu
-      - run: coverage report -m
-      - uses: codecov/codecov-action@v2
-        with:
-          file: ./coverage.xml
-  test_roman_with_coverage:
-    name: test Roman pipeline with coverage
-    needs: [ style, audit ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}
-          key: test-coverage-roman-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
-      - run: pip install -e ".[test]" pytest-xdist pytest-cov
-      - run: pip install "romancal[test] @ git+https://github.com/spacetelescope/romancal.git"
-      - run: pip freeze
-      - run: pytest -n auto --cov-report=xml --cov=src/stpipe --pyargs romancal
-        env:
-          CRDS_SERVER_URL: https://roman-crds-test.stsci.edu
-      - run: coverage report -m
-      - uses: codecov/codecov-action@v2
-        with:
-          file: ./coverage.xml
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      envs: |
+        - linux: test-oldestdeps-cov-xdist
+          python-version: 3.8
+        - linux: test-xdist
+          python-version: '3.8'
+        - linux: test-xdist
+          python-version: '3.9'
+        - linux: test-xdist
+          python-version: '3.10'
+        - linux: test-xdist
+          python-version: '3.11'
+        - macos: test-xdist
+          python-version: '3.11'
+        - linux: test-cov-xdist
+          coverage: 'codecov'
+  test_downstream:
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
+    with:
+      setenv: |
+        CRDS_PATH: ${{ needs.crds.outputs.path }}
+        CRDS_CLIENT_RETRY_COUNT: 3
+        CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
+      envs: |
+        - linux: test-jwst-xdist
+        - linux: test-romancal-xdist

.github/workflows/ci_cron.yml

@@ -0,0 +1,20 @@
+
+name: Weekly cron
+
+on:
+  schedule:
+    # Weekly Monday 6AM build
+    - cron: "0 0 * * 1"
+  workflow_dispatch:
+
+jobs:
+  test:
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      envs: |
+        - macos: test-xdist
+          python-version: 3.8
+        - macos: test-xdist
+          python-version: 3.9
+        - macos: test-xdist
+          python-version: 3.10

CHANGES.rst

@@ -5,6 +5,7 @@
   usage of pkg_resources [#84]
 - update minimum python to 3.8 and ASDF version to 2.8 [#87]
 - replace legacy AsdfExtension with resource_mapping [#82]
+- update minimum version of ``asdf`` to ``2.13`` and add minimum dependency testing to CI [#75]
 
 0.4.5 (2022-12-23)
 ==================

pyproject.toml

@@ -12,7 +12,7 @@ classifiers = [
     'Programming Language :: Python :: 3',
 ]
 dependencies = [
-    'asdf>=2.8.0',
+    'asdf>=2.13',
     'crds>=7.4.1.3',
     'astropy>=5.0.4',
     'stdatamodels>=0.2.4',

tox.ini

@@ -0,0 +1,88 @@
+[tox]
+envlist =
+    check-{style,security,build}
+    test{,-warnings,-cov}-xdist
+    test-numpy{120,121,122}
+    test-{jwst,romancal}-xdist
+    build-{docs,dist}
+
+# tox environments are constructed with so-called 'factors' (or terms)
+# separated by hyphens, e.g. test-devdeps-cov. Lines below starting with factor:
+# will only take effect if that factor is included in the environment name. To
+# see a list of example environments that can be run, along with a description,
+# run:
+#
+#     tox -l -v
+#
+
+[testenv:check-style]
+description = check code style, e.g. with ruff
+skip_install = true
+deps =
+    ruff
+commands =
+    ruff . {posargs}
+
+[testenv:check-security]
+description = run bandit to check security compliance
+skip_install = true
+deps =
+    bandit>=1.7
+commands =
+    bandit -r -ll src
+
+[testenv]
+description =
+    run tests
+    jwst: of JWST pipeline
+    romancal: of Romancal pipeline
+    oldestdeps: with the oldest supported version of key dependencies
+    warnings: treating warnings as errors
+    cov: with coverage
+    xdist: using parallel processing
+extras =
+    test
+deps =
+    xdist: pytest-xdist
+    cov: pytest-cov
+    jwst: jwst[test] @ git+https://github.com/spacetelescope/jwst.git
+    romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git
+    oldestdeps: minimum_dependencies
+pass_env =
+    CRDS_*
+    CI
+set_env =
+    jwst: CRDS_SERVER_URL=https://jwst-crds.stsci.edu
+    romancal: CRDS_SERVER_URL=https://roman-crds.stsci.edu
+package = 
+    !cov: wheel
+    cov: editable
+allowlist_externals =
+    echo
+commands_pre =
+    oldestdeps: minimum_dependencies stpipe --filename requirements-min.txt
+    oldestdeps: pip install -r requirements-min.txt
+    pip freeze
+commands =
+    pytest \
+    warnings: -W error \
+    xdist: -n auto \
+    jwst: --pyargs jwst --ignore-glob=timeconversion --ignore-glob=associations --ignore-glob=*/scripts/* \
+    romancal: --pyargs romancal \
+    cov: --cov=src/stpipe --cov-config=pyproject.toml --cov-report=term-missing --cov-report=xml \
+    {posargs}
+
+[testenv:build-docs]
+description = invoke sphinx-build to build the HTML docs
+extras =
+    docs
+commands =
+    sphinx-build -W docs docs/_build
+
+[testenv:build-dist]
+description = build wheel and sdist
+skip_install = true
+deps =
+    build
+commands =
+    python -m build .