Merge pull request #204 from pybop-team/139-split-integration-unit-tests

Splits integration / examples / unit tests

.github/workflows/scheduled_tests.yaml

@@ -6,9 +6,10 @@ on:
     branches:
       - main
 
-  # runs every day at 09:00 UTC
+  # runs every day at 09:00 and 15:00 UTC
   schedule:
     - cron: '0 9 * * *'
+    - cron: '0 15 * * *'
 
 # Check noxfile.py for associated environment variables
 env:
@@ -84,7 +85,7 @@ jobs:
           python -m pip install --upgrade pip nox
 
       - name: Unit tests with nox
-        run: python -m nox -s unit
+        run: python -m nox -s coverage
 
       - name: Run notebooks with nox
         run: python -m nox -s notebooks
@@ -117,7 +118,7 @@ jobs:
           eval "$(pyenv init -)"
           pyenv activate pybop-${{ matrix.python_version }}-${{ matrix.pybamm_version }}
           python -m pip install --upgrade pip nox
-          python -m nox -s unit
+          python -m nox -s coverage
           python -m nox -s notebooks
 
       - name: Uninstall pyenv-virtualenv & python

.github/workflows/test_on_push.yaml

@@ -1,4 +1,4 @@
-name: test_on_push
+name: PyBOP
 
 on:
   workflow_dispatch:
@@ -27,18 +27,42 @@ jobs:
           python -m pip install pre-commit
           pre-commit run ruff
 
+  integration_tests:
+    needs: style
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-14]
+        python-version: ["3.12"]
+
+    name: Integration tests (${{ matrix.os }} / Python ${{ matrix.python-version }})
 
-  build:
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip nox
+
+      - name: Integration tests
+        run: |
+          nox -s integration
+
+  unit_tests:
     needs: style
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-        exclude: # We run the coverage tests on Ubuntu with Python 3.11
+        exclude: # We run the coverage tests on Ubuntu with Python 3.12
           - os: ubuntu-latest
-            python-version: "3.11"
+            python-version: "3.12"
         # Include MacOS M-series Runners
         include:
           - os: macos-14
@@ -48,6 +72,8 @@ jobs:
           - os: macos-14
             python-version: "3.12"
 
+    name: Unit tests (${{ matrix.os }} / Python ${{ matrix.python-version }})
+
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -57,33 +83,63 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip nox
-      - name: Unit and notebook tests with nox
+
+      - name: Unit tests
         run: |
           nox -s unit
+
+  example_tests:
+    needs: style
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-14]
+        python-version: ["3.12"]
+
+    name: Test examples (${{ matrix.os }} / Python ${{ matrix.python-version }})
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip nox
+
+      - name: Run examples
+        run: |
+          nox -s examples
+
+      - name: Run notebooks
+        run: |
           nox -s notebooks
 
-  # Runs only on Ubuntu with Python 3.11
+  # Runs only on Ubuntu with Python 3.12
   check_coverage:
+    needs: style
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-    name: Coverage tests (ubuntu-latest / Python 3.11)
+    name: Coverage tests (ubuntu-latest / Python 3.12)
 
     steps:
       - name: Check out PyBOP repository
         uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: 'pip'
           cache-dependency-path: setup.py
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip nox
-      - name: Run coverage tests for Ubuntu with Python 3.11 and generate report
+      - name: Run coverage tests for Ubuntu with Python 3.12 and generate report
         run: nox -s coverage
 
       - name: Upload coverage report

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Features
 
+- [#204](https://github.com/pybop-team/PyBOP/pull/204) - Splits integration, unit, examples, plots tests, update workflows. Adds pytest `--examples`, `--integration`, `--plots` args. Adds tests for coverage after removal of examples. Adds examples and integrations nox sessions. Adds `pybop.RMSE._evaluateS1()` method
 - [#206](https://github.com/pybop-team/PyBOP/pull/206) - Adds Python 3.12 support with corresponding github actions changes.
 - [#18](https://github.com/pybop-team/PyBOP/pull/18) - Adds geometric parameter fitting capability, via `model.rebuild()` with `model.rebuild_parameters`.
 - [#203](https://github.com/pybop-team/PyBOP/pull/203) - Adds support for modern Python packaging via a `pyproject.toml` file and configures the `pytest` test runner and `ruff` linter to use their configurations stored as declarative metadata.

conftest.py

@@ -10,9 +10,21 @@ def pytest_addoption(parser):
     parser.addoption(
         "--unit", action="store_true", default=False, help="run unit tests"
     )
+    parser.addoption(
+        "--integration",
+        action="store_true",
+        default=False,
+        help="run integration tests",
+    )
     parser.addoption(
         "--examples", action="store_true", default=False, help="run examples tests"
     )
+    parser.addoption(
+        "--plots", action="store_true", default=False, help="run plotting tests"
+    )
+    parser.addoption(
+        "--notebooks", action="store_true", default=False, help="run notebook tests"
+    )
 
 
 def pytest_terminal_summary(terminalreporter, exitstatus, config):
@@ -25,28 +37,47 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "unit: mark test as a unit test")
+    config.addinivalue_line("markers", "integration: mark test as an integration test")
     config.addinivalue_line("markers", "examples: mark test as an example")
+    config.addinivalue_line("markers", "plots: mark test as a plot test")
+    config.addinivalue_line("markers", "notebook: mark test as a notebook test")
 
 
 def pytest_collection_modifyitems(config, items):
-    unit_option = config.getoption("--unit")
-    examples_option = config.getoption("--examples")
+    options = {
+        "unit": "unit",
+        "examples": "examples",
+        "integration": "integration",
+        "plots": "plots",
+        "notebooks": "notebooks",
+    }
+    selected_markers = [
+        marker for option, marker in options.items() if config.getoption(option)
+    ]
 
-    if not unit_option and not examples_option:
-        skip_all = pytest.mark.skip(reason="need --unit or --examples option to run")
-        for item in items:
-            item.add_marker(skip_all)
+    if (
+        "notebooks" in selected_markers
+    ):  # Notebooks are meant to be run as an individual session
+        return
 
-    elif unit_option and not examples_option:
-        skip_examples = pytest.mark.skip(
-            reason="need --examples option to run examples tests"
+    # If no options were passed, skip all tests
+    if not selected_markers:
+        skip_all = pytest.mark.skip(
+            reason="Need at least one of --unit, --examples, --integration, or --plots option to run"
         )
         for item in items:
-            if "examples" in item.keywords:
-                item.add_marker(skip_examples)
+            item.add_marker(skip_all)
+        return
 
-    if examples_option and not unit_option:
-        skip_unit = pytest.mark.skip(reason="need --unit option to run unit tests")
-        for item in items:
-            if "unit" in item.keywords:
-                item.add_marker(skip_unit)
+    # Skip tests that don't match any of the selected markers
+    for item in items:
+        item_markers = {
+            mark.name for mark in item.iter_markers()
+        }  # Gather markers of the test item
+        if not item_markers.intersection(
+            selected_markers
+        ):  # Skip if there's no intersection with selected markers
+            skip_this = pytest.mark.skip(
+                reason=f"Test does not match the selected options: {', '.join(selected_markers)}"
+            )
+            item.add_marker(skip_this)

noxfile.py

@@ -24,13 +24,30 @@ def coverage(session):
     session.install("-e", ".[all,dev]", silent=False)
     if PYBOP_SCHEDULED:
         session.run("pip", "install", f"pybamm=={PYBAMM_VERSION}", silent=False)
+    session.run("pytest", "--unit", "--cov", "--cov-append", "--cov-report=xml")
     session.run(
         "pytest",
-        "--unit",
-        "--examples",
+        "--integration",
         "--cov",
+        "--cov-append",
         "--cov-report=xml",
     )
+    session.run(
+        "pytest", "--plots", "--cov", "--cov-append", "--cov-report=xml", "-n", "1"
+    )
+
+
+@nox.session
+def integration(session):
+    session.install("-e", ".[all,dev]", silent=False)
+    session.install("pytest", "pytest-mock")
+    session.run("pytest", "--integration")
+
+
+@nox.session
+def examples(session):
+    session.install("-e", ".[all,dev]", silent=False)
+    session.run("pytest", "--examples")
 
 
 @nox.session
@@ -39,7 +56,12 @@ def notebooks(session):
     session.install("-e", ".[all,dev]", silent=False)
     if PYBOP_SCHEDULED:
         session.run("pip", "install", f"pybamm=={PYBAMM_VERSION}", silent=False)
-    session.run("pytest", "--nbmake", "--examples", "examples/", external=True)
+    session.run(
+        "pytest",
+        "--notebooks",
+        "--nbmake",
+        "examples/",
+    )
 
 
 @nox.session

pybop/_optimisation.py

@@ -449,7 +449,7 @@ def set_max_iterations(self, iterations=1000):
                 raise ValueError("Maximum number of iterations cannot be negative.")
         self._max_iterations = iterations
 
-    def set_max_unchanged_iterations(self, iterations=25, threshold=1e-5):
+    def set_max_unchanged_iterations(self, iterations=5, threshold=1e-5):
         """
         Set the maximum number of iterations without significant change as a stopping criterion.
         Credit: PINTS

pybop/costs/fitting_costs.py

@@ -44,6 +44,46 @@ def _evaluate(self, x, grad=None):
         else:
             return np.sqrt(np.mean((prediction - self._target) ** 2))
 
+    def _evaluateS1(self, x):
+        """
+        Compute the cost and its gradient with respect to the parameters.
+
+        Parameters
+        ----------
+        x : array-like
+            The parameters for which to compute the cost and gradient.
+
+        Returns
+        -------
+        tuple
+            A tuple containing the cost and the gradient. The cost is a float,
+            and the gradient is an array-like of the same length as `x`.
+
+        Raises
+        ------
+        ValueError
+            If an error occurs during the calculation of the cost or gradient.
+        """
+        y, dy = self.problem.evaluateS1(x)
+        if len(y) < len(self._target):
+            e = np.float64(np.inf)
+            de = self._de * np.ones(self.n_parameters)
+        else:
+            dy = dy.reshape(
+                (
+                    self.problem.n_time_data,
+                    self.n_outputs,
+                    self.n_parameters,
+                )
+            )
+            r = y - self._target
+            e = np.sqrt(np.mean((r) ** 2))
+            de = np.mean((r.T * dy.T), axis=2) / np.sqrt(
+                np.mean((r.T * dy.T) ** 2, axis=2)
+            )
+
+        return e, de.flatten()
+
 
 class SumSquaredError(BaseCost):
     """

pyproject.toml

@@ -56,7 +56,7 @@ include = ["pybop", "pybop.*"]
 Homepage = "https://github.com/pybop-team/PyBOP"
 
 [tool.pytest.ini_options]
-addopts = "--showlocals -v"
+addopts = "--showlocals -v -n auto"
 
 [tool.ruff]
 extend-include = ["*.ipynb"]