Otter Grade Write Individual Notebook Results

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Changelog
 
 **v5.6.0 (unreleased):**
-
+* Updated Otter Grade to write grading summary for each notebook per [#814](https://github.com/ucbds-infra/otter-grader/issues/814)
 * Updated Otter Grade CSV to indicate which notebooks timeout per [#813](https://github.com/ucbds-infra/otter-grader/issues/813)
 * Updated Otter Grade CSV to include the number of points per question in the first row
 * Updated Otter Grade CSV to include total points column

otter/cli.py

@@ -132,6 +132,7 @@ def generate_cli(*args, **kwargs):
 @click.option("-a", "--autograder", default=defaults["autograder"], help="Path to autograder zip file")
 @click.option("-o", "--output-dir", default=defaults["output_dir"], help="Directory to which to write output")
 @click.option("--ext", default=defaults["ext"], type=click.Choice(_ALLOWED_EXTENSIONS), help="The extension to glob for submissions")
+@click.option("--summaries", is_flag=True, help="Whether to write the otter run results for each graded notebook")
 @click.option("--pdfs", is_flag=True, help="Whether to copy notebook PDFs out of containers")
 @click.option("--containers", default=defaults["containers"], type=click.INT, help="Specify number of containers to run in parallel")
 @click.option("--image", default=defaults["image"], help="A Docker image tag to use as the base image")

otter/grade/__init__.py

@@ -10,6 +10,7 @@
 from .utils import (
     merge_csv,
     prune_images,
+    POINTS_POSSIBLE_LABEL,
     SCORES_DICT_FILE_KEY,
     SCORES_DICT_PERCENT_CORRECT_KEY,
     SCORES_DICT_TOTAL_POINTS_KEY,
@@ -33,6 +34,7 @@ def main(
     autograder: str = "./autograder.zip",
     containers: int = 4, 
     ext: str = "ipynb",
+    summaries: bool = False,
     no_kill: bool = False,
     image: str = "ubuntu:22.04", 
     pdfs: bool = False,
@@ -155,6 +157,17 @@ def main(
     # write to CSV file
     output_df.to_csv(os.path.join(output_dir, "final_grades.csv"), index=False)
 
+    # write score summaries to files
+    if summaries:
+        grading_summary_path = os.path.join(output_dir, "grading-summaries")
+        if not os.path.exists(grading_summary_path):
+            os.mkdir(grading_summary_path)
+        for df in grade_dfs:
+            df_dict = df.to_dict()
+            if df_dict['file'][0] != POINTS_POSSIBLE_LABEL:
+                with open(os.path.join(grading_summary_path, f"{df_dict['file'][0]}.txt"), mode="w") as f:
+                    f.write(df_dict["summary"][0])
+
     # return percentage if a single file was graded
     if len(paths) == 1 and os.path.isfile(paths[0]):
         return output_df[SCORES_DICT_PERCENT_CORRECT_KEY][1]

test/test_cli.py

@@ -479,6 +479,10 @@ def test_grade(mocked_grade, run_cli):
         assert_cli_result(result, expect_error=False)
         mocked_grade.assert_called_with(**{**std_kwargs, "ext": ext})
 
+    result = run_cli([*cmd_start, "--summaries"])
+    assert_cli_result(result, expect_error=False)
+    mocked_grade.assert_called_with(**{**std_kwargs, "summaries": True})
+
     result = run_cli([*cmd_start, "--pdfs"])
     assert_cli_result(result, expect_error=False)
     mocked_grade.assert_called_with(**{**std_kwargs, "pdfs": True})

test/test_grade/test_integration.py

@@ -37,6 +37,8 @@ def cleanup_output(cleanup_enabled):
             shutil.rmtree("test/submission_pdfs")
         if os.path.exists(ZIP_SUBM_PATH):
             os.remove(ZIP_SUBM_PATH)
+        if os.path.exists("test/grading-summaries"):
+            shutil.rmtree("test/grading-summaries")
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -362,3 +364,24 @@ def test_config_overrides_integration():
     got = got.reindex(sorted(got.columns), axis=1)
     want = want.reindex(sorted(want.columns), axis=1)
     assert got.equals(want)
+
+
+@pytest.mark.slow
+@pytest.mark.docker
+def test_grade_summaries():
+    """
+    Checks that are grading summaries are written to the disck
+    """
+    grade(
+        name = ASSIGNMENT_NAME,
+        paths = [FILE_MANAGER.get_path("notebooks")],
+        output_dir = "test/",
+        autograder = AG_ZIP_PATH,
+        summaries = True
+    )
+    for filename in os.listdir(FILE_MANAGER.get_path("notebooks")):
+        file_path = os.path.join("test/grading-summaries", filename)
+        if os.path.isfile(file_path):
+            assert os.path.exists(file_path)
+            with open(file_path, 'r') as file:
+                assert file.read()