Inference flavor experiments

.github/workflows/python-test.yml

@@ -1,56 +1,59 @@
 name: Python test
 
-on: [push]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
     build:
-        runs-on: ubuntu-latest
+        strategy:
+          matrix:
+            os: ["ubuntu-latest", "macos-latest"]
+            python-version: [ "3.10", "3.11", "3.12" ]
+            mpi: [ "openmpi" ]  # "mpich", "intelmpi"
+#            exclude:
+#              - os: "macos-latest"
+#                mpi: "intelmpi"
+
+        runs-on: ${{ matrix.os }}
+
         steps:
-            - uses: actions/checkout@v4
-            - name: Set up MPI
-              uses: mpi4py/setup-mpi@v1
-              with:
-                mpi: 'openmpi'
-            - name: Set up Python
-              uses: actions/setup-python@v5
-              with:
-                  python-version: '3.10'
-            - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install ."[dev]"
-                  pip install genbadge"[coverage]"
-            - name: Lint with ruff
-              run: |
-                  # Stop the build if there are Python syntax errors or undefined names.
-                  ruff check --output-format=github --select=E9,F63,F7,F82 --target-version=py39 .
-                  # Default set of ruff rules with GitHub Annotations
-                  ruff check --output-format=github --target-version=py39 .
-            - name: Test with pytest and measure coverage
-              run: |
-                coverage run --omit "*/tests/*" --rcfile=./pyproject.toml -m pytest
-                mpirun -n 4 coverage run --rcfile=./pyproject.toml -m mpi4py -m pytest --with-mpi
-                coverage combine
-                coverage report  --omit "*/tests/*" -m
-                coverage xml
-                genbadge coverage -i coverage.xml -o coverage.svg
-            - name: Verify Changed files
-              uses: tj-actions/verify-changed-files@v19
-              id: verify-changed-files
-              with:
-                files: coverage.svg
-
-            - name: Commit files
-              if: steps.verify-changed-files.outputs.files_changed == 'true'
-              run: |
-                git config --local user.email "github-actions[bot]@users.noreply.github.com"
-                git config --local user.name "github-actions[bot]"
-                git add coverage.svg
-                git commit -m "update coverage badge"
-
-            - name: Push changes
-              if: steps.verify-changed-files.outputs.files_changed == 'true'
-              uses: ad-m/github-push-action@master
-              with:
-                github_token: ${{ secrets.github_token }}
-                branch: ${{ github.ref }}
+          - name: Check out repository
+            uses: actions/checkout@v4
+
+          - name: Setup MPI ${{ matrix.mpi }}
+            uses: mpi4py/setup-mpi@v1
+            with:
+              mpi: ${{ matrix.mpi }}
+
+          - name: Set up Python ${{ matrix.python-version }}
+            uses: actions/setup-python@v5
+            with:
+              python-version: ${{ matrix.python-version }}
+
+          - name: Install dependencies
+            run: |
+                python -m pip install --upgrade pip
+                pip install ."[test]"
+
+          - name: Test with pytest and measure coverage
+            run: |
+              coverage run --rcfile=./pyproject.toml -m pytest
+              mpirun -n 4 coverage run --rcfile=./pyproject.toml -m mpi4py -m pytest --with-mpi
+              coverage combine
+              coverage report -m --format markdown > cov_report.txt
+              coverage xml
+
+          - name: Upload coverage reports to Codecov
+            uses: codecov/codecov-action@v4.0.1
+            with:
+              token: ${{ secrets.CODECOV_TOKEN }}
+
+          - name: Post coverage report to PR
+            uses: marocchino/sticky-pull-request-comment@v2
+            with:
+              path: cov_report.txt

.pre-commit-config.yaml

@@ -1,12 +1,11 @@
 repos:
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.0 # Use the sha / tag you want to point at
+    rev: v1.11.2 # Use the sha / tag you want to point at
     hooks:
       - id: mypy
-        additional_dependencies: [types-all]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.4.10
+    rev: v0.6.7
     hooks:
       # Run the linter.
       - id: ruff

README.md

@@ -4,9 +4,9 @@
 [![License: BSD-3](https://img.shields.io/badge/License-BSD--3-blue)](https://opensource.org/licenses/BSD-3-Clause)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![](https://img.shields.io/badge/Python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![codecov](https://codecov.io/gh/Helmholtz-AI-Energy/special-couscous/graph/badge.svg?token=ISGFUW08BK)](https://codecov.io/gh/Helmholtz-AI-Energy/special-couscous)
 [![](https://img.shields.io/badge/Contact-marie.weiel%40kit.edu-blue)](mailto:marie.weiel@kit.edu)
-![](./coverage.svg)
-
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/Helmholtz-AI-Energy/special-couscous/main.svg)](https://results.pre-commit.ci/latest/github/Helmholtz-AI-Energy/special-couscous/main)
 ``specialcouscous`` is a Python package for large-scale distributed training and inference of random forests on
 supercomputers. It uses the message passing interface (MPI) for communication between parallel processes.
 

pyproject.toml

@@ -43,6 +43,13 @@ dev = [
     "sphinxemoji",
 ]
 
+test = [
+    "coverage",
+    "pytest",
+    "pytest-cov",
+    "pytest-mpi"
+]
+
 [project.urls]
 Homepage = "https://github.com/Helmholtz-AI-Energy/special-couscous"
 Issues = "https://github.com/Helmholtz-AI-Energy/special-couscous/issues"
@@ -150,3 +157,7 @@ testpaths = [
 
 [tool.coverage.run]
 parallel = true
+omit = ["*/tests/*"]
+
+[tool.coverage.report]
+omit = ["*/tests/*"]

scripts/analysis/acc_drop.py

@@ -1,5 +1,6 @@
 import os
 import pathlib
+import re
 import sys
 from collections import defaultdict
 
@@ -52,19 +53,46 @@
                     global_test_accuracy
                 )
 
+print(results)
+
+for dirpath, dirnames, filenames in os.walk(root_dir):
+    for filename in filenames:
+        if filename.endswith(".out"):
+            parts = dirpath.split(os.sep)
+            number_of_tasks = int(parts[-2].split("_")[1])
+            data_seed = int(parts[-1].split("_")[1])  # Extract data seed from path.
+            model_seed = int(parts[-1].split("_")[2])  # Extract model seed from path.
+            pattern_energy = r"(?<=\/ )\d+(\.\d+)?(?= Watthours)"
+            with open(
+                os.path.join(dirpath, filename), "r"
+            ) as file:  # Load input text from the file.
+                input_text = file.read()
+                print(dirpath)
+            energy_match = re.search(pattern_energy, input_text)
+            energy_consumed = float(energy_match.group(0))  # type:ignore
+            print(f"Energy Consumed: {energy_consumed:.2f} Watthours")
+            results[(data_set, number_of_tasks, data_seed, model_seed)].append(
+                energy_consumed
+            )
+
+print(results)
+
 # Save the results to a pandas dataframe.
 results_df = pd.DataFrame(
-    [(k[0], k[1], k[2], k[3], v[0]) for k, v in results.items()],
+    [(k[0], k[1], k[2], k[3], v[0], v[1]) for k, v in results.items()],
     columns=[
         "Dataset",
         "Number of tasks",
         "Data seed",
         "Model seed",
         "Global test accuracy",
+        "Energy consumed",
     ],
 )
 results_df = results_df.sort_values(by=["Number of tasks", "Data seed", "Model seed"])
 
+overall_energy = results_df["Energy consumed"].sum()
+
 avg_data_seeds = (
     results_df.groupby(["Number of tasks", "Data seed"])
     .agg({"Global test accuracy": "mean"})
@@ -78,7 +106,7 @@
 )
 print(avg_n_tasks)
 
-plt.figure(figsize=(10, 6))
+f, ax = plt.subplots(figsize=(10, 6))
 plt.grid(True)
 
 # Plot individual test accuracy vs number of tasks as small dots
@@ -128,6 +156,16 @@
     fontweight="bold",
 )
 plt.legend(loc="lower left", fontsize="small")
+energy_str = f"Overall {(overall_energy / 1000):.2f} kWh consumed"
+ax.text(
+    0.75,
+    0.95,
+    energy_str,
+    transform=ax.transAxes,
+    fontsize="small",
+    verticalalignment="top",
+    fontweight="bold",
+)
 # plt.legend(loc='upper left', bbox_to_anchor=(0.5, 0.82), fontsize="small")
 plt.savefig(pathlib.Path(root_dir) / f"{data_set}_acc_drop.png")