PCA Resyncing Logic

backend/alert/views.py

@@ -132,15 +132,16 @@ def accept_webhook(request):
         else:
             response = JsonResponse({"message": "webhook recieved"})
 
-        u = record_update(
-            section,
-            course_term,
-            prev_status,
-            course_status,
-            alert_for_course_called,
-            request.body,
-        )
-        update_course_from_record(u)
+        with transaction.atomic():
+            u = record_update(
+                section,
+                course_term,
+                prev_status,
+                course_status,
+                alert_for_course_called,
+                request.body,
+            )
+            update_course_from_record(u)
     except (ValidationError, ValueError) as e:
         logger.error(e, extra={"request": request})
         response = JsonResponse(

backend/courses/management/commands/loadstatus.py

@@ -1,40 +1,93 @@
+import json
 import logging
 
 from django.core.management.base import BaseCommand
 from tqdm import tqdm
 
 from courses import registrar
 from courses.models import Course, Section
-from courses.util import get_course_and_section, get_current_semester
+from courses.util import (
+    get_course_and_section,
+    get_current_semester,
+    record_update,
+    translate_semester_inv,
+)
 
 
-def set_all_status(semester=None):
+def set_all_status(semester=None, add_status_update=False):
     if semester is None:
         semester = get_current_semester()
     statuses = registrar.get_all_course_status(semester)
     if not statuses:
         return
     for status in tqdm(statuses):
+        statuses_out_of_sync = []
+        status_updates_out_of_sync = []
+
         section_code = status.get("section_id_normalized")
         if section_code is None:
             continue
 
+        course_status = status.get("status")
+        if course_status is None:
+            continue
+
+        course_term = status.get("term")
+        if course_term is None:
+            continue
+        if any(course_term.endswith(s) for s in ["10", "20", "30"]):
+            course_term = translate_semester_inv(course_term)
+
+        # Ignore sections not in db
         try:
             _, section = get_course_and_section(section_code, semester)
         except (Section.DoesNotExist, Course.DoesNotExist):
             continue
-        section.status = status["status"]
-        section.save()
+
+        # Resync database (doesn't need to be atomic)
+        last_status_update = section.last_status_update
+        current_status = section.status
+
+        # Change status attribute of section model (might want to use bulk update)
+        if current_status != course_status:
+            statuses_out_of_sync.append(section_code)
+            section.status = course_status
+            section.save()
+
+        # Add corresponding status update object
+        if add_status_update and last_status_update.new_status != course_status:
+            status_updates_out_of_sync.append(section_code)
+            record_update(
+                section,
+                course_term,
+                last_status_update.new_status,
+                course_status,
+                False,
+                json.dumps(status),
+            )
+
+    print(f"{len(statuses_out_of_sync)} statuses were out of sync.")
+    print(statuses_out_of_sync)
+
+    print(f"{len(status_updates_out_of_sync)} status updates were out of sync.")
+    print(status_updates_out_of_sync)
 
 
 class Command(BaseCommand):
-    help = "Load course status for courses in the DB"
+    help = "Load course status for courses in the DB. Conditionally adds StatusUpdate objects."
 
     def add_arguments(self, parser):
         parser.add_argument("--semester", default=None, type=str)
+        parser.add_argument(
+            "--create-status-updates", action="store_true", help="Create status updates if set"
+        )
 
     def handle(self, *args, **kwargs):
         root_logger = logging.getLogger("")
         root_logger.setLevel(logging.DEBUG)
 
-        set_all_status(semester=kwargs["semester"])
+        print(f"Create status updates is {kwargs['create_status_updates']}")
+
+        set_all_status(
+            semester=kwargs["semester"], add_status_update=kwargs["create_status_updates"]
+        )

backend/courses/management/commands/pathopendata.py

@@ -0,0 +1,130 @@
+import json
+import logging
+
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from courses import registrar
+from courses.models import Course, Section
+from courses.util import (
+    get_course_and_section,
+    get_current_semester,
+    record_update,
+    translate_semester_inv,
+)
+
+import requests
+from urllib.parse import quote
+
+
+def status_on_path_at_penn(course_code, path_at_penn_semester="202430"):
+    # Note: this api is actually unauthenticated as far as I can tell
+    # so no cookies needed!
+
+    headers = {
+        "accept": "application/json, text/javascript, */*; q=0.01",
+        "accept-language": "en-US,en;q=0.9",
+        "content-type": "application/json",
+        "origin": "https://courses.upenn.edu",
+        "priority": "u=1, i",
+        "referer": "https://courses.upenn.edu/",
+        "sec-ch-ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-origin",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        "x-requested-with": "XMLHttpRequest",
+    }
+
+    params = {
+        "page": "fose",
+        "route": "search",
+        "alias": course_code,
+    }
+
+    data = quote(
+        f'{{"other":{{"srcdb":"{path_at_penn_semester}"}},"criteria":[{{"field":"alias","value":"{course_code}"}}]}}'
+    )
+    response = requests.post(
+        "https://courses.upenn.edu/api/", params=params, headers=headers, data=data
+    )
+    if response.ok:
+        return {
+            ("-".join(result["code"].split(" ")) + "-" + result["no"]): result["stat"]
+            for result in response.json()["results"]
+        }
+
+
+def map_opendata_to_path(course_status):
+    return "A" if course_status == "O" else "F"
+
+
+def get_path_code(section_code):
+    parts = section_code.split("-")
+    return f"{parts[0]} {parts[1]}"
+
+
+def find_diff(semester=None, add_status_update=False):
+    if semester is None:
+        semester = get_current_semester()
+    statuses = registrar.get_all_course_status(semester)
+    if not statuses:
+        return
+    course_map = {}
+    for status in statuses:
+        section_code = status.get("section_id_normalized")
+        if section_code is None:
+            continue
+
+        course_status = status.get("status")
+        if course_status is None:
+            continue
+
+        course_term = status.get("term")
+        if course_term is None:
+            continue
+        if any(course_term.endswith(s) for s in ["10", "20", "30"]):
+            course_term = translate_semester_inv(course_term)
+
+        if course_term != "2024C":
+            continue
+
+        course_map[section_code] = map_opendata_to_path(course_status)
+
+    out_of_sync = []
+    visited = set()
+    for section_code in tqdm(course_map):
+        if section_code in visited:
+            continue
+        path_code = get_path_code(section_code)
+        results = status_on_path_at_penn(path_code)
+
+        for result in results:
+            visited.add(result)
+            path_status = results[result]
+            if path_status != "A" and path_status != "F":
+                continue
+            section_status = course_map[result]
+
+            if path_status != section_status:
+                out_of_sync.append((result, section_status, path_status))
+
+    for code, status1, status2 in out_of_sync:
+        print(
+            f"{code} is out of sync. OpenData has status {status1} and Path has status {status2}."
+        )
+
+
+class Command(BaseCommand):
+    help = "Report the difference between OpenData and Path registrations."
+
+    def add_arguments(self, parser):
+        parser.add_argument("--semester", default=None, type=str)
+
+    def handle(self, *args, **kwargs):
+        root_logger = logging.getLogger("")
+        root_logger.setLevel(logging.DEBUG)
+
+        find_diff(semester=kwargs["semester"])

backend/courses/models.py

@@ -1027,7 +1027,7 @@ def save(self, *args, **kwargs):
         self.section.has_status_updates = True
         self.section.save()
 
-        section_demand_change.delay(self.section.id, self.created_at)
+        # section_demand_change.delay(self.section.id, self.created_at)
 
 
 """

backend/tests/alert/test_alert.py

@@ -806,9 +806,7 @@ def test_alert_called_wrong_sem(self, mock_alert):
             "sent" in json.loads(res.content)["message"],
         )
         self.assertFalse(mock_alert.called)
-        self.assertEqual(1, StatusUpdate.objects.count())
-        u = StatusUpdate.objects.get()
-        self.assertFalse(u.alert_sent)
+        self.assertEqual(0, StatusUpdate.objects.count())
 
     def test_alert_called_alerts_off(self, mock_alert):
         Option.objects.update_or_create(