Make check-onboarded-projects a daily job (#2352)

Make check-onboarded-projects a daily job

This should hopefully fix the long prefetch task queue in celery

Reviewed-by: Laura Barcziová

packit_service/celery_config.py

@@ -29,6 +29,11 @@
         "schedule": crontab(minute=0, hour=1),  # nightly at 1AM
         "options": {"queue": "long-running"},
     },
+    "check-onboarded-projects": {
+        "task": "packit_service.worker.tasks.run_check_onboarded_projects",
+        "schedule": crontab(minute=0, hour=2),  # nightly at 2AM
+        "options": {"queue": "long-running"},
+    },
 }
 
 # http://mher.github.io/flower/prometheus-integration.html#set-up-your-celery-application

packit_service/service/api/usage.py

@@ -26,7 +26,6 @@
     SyncReleaseTargetModel,
 )
 from packit_service.service.api.utils import response_maker
-from packit_service.celerizer import celery_app
 
 logger = getLogger("packit_service")
 
@@ -656,27 +655,6 @@ def calculate(cls):
             known_onboarded_projects
         )
 
-        if recheck_if_onboarded:
-            # Run the long running task in Celery.
-            # The task will collect data about merged PR for
-            # every given project. If a Packit merged PR is
-            # found the long running task will save the
-            # onboarded_downstream flag in the git projects table.
-            celery_app.send_task(
-                name="task.check_onboarded_projects",
-                kwargs={
-                    "projects": [
-                        {
-                            "id": project.id,
-                            "instance_url": project.instance_url,
-                            "namespace": project.namespace,
-                            "repo_name": project.repo_name,
-                        }
-                        for project in recheck_if_onboarded
-                    ]
-                },
-            )
-
         onboarded = {
             project.id: project.project_url
             for project in onboarded_projects.union(known_onboarded_projects)

packit_service/worker/handlers/usage.py

@@ -1,20 +1,20 @@
 # Copyright Contributors to the Packit project.
 # SPDX-License-Identifier: MIT
 
-from typing import List, Dict
+from typing import Set
 
 from ogr.abstract import PRStatus
 from packit_service.config import ServiceConfig
 from packit_service.models import GitProjectModel
 
 
-def check_onboarded_projects(projects: List[Dict]):
+def check_onboarded_projects(projects: Set[GitProjectModel]):
     """For every given project check if it has a merged Packit PR.
 
     If yes it is onboarded: save the flag in the git projects table.
     """
     for project in projects:
-        downstream_project_url = f"https://{project['instance_url']}"
+        downstream_project_url = f"https://{project.instance_url}"
         if downstream_project_url != "https://src.fedoraproject.org":
             continue
 
@@ -24,12 +24,12 @@ def check_onboarded_projects(projects: List[Dict]):
             if service.instance_url == downstream_project_url
         ][0]
         ogr_project = pagure_service.get_project(
-            namespace=project["namespace"],
-            repo=project["repo_name"],
+            namespace=project.namespace,
+            repo=project.repo_name,
         )
         prs = ogr_project.get_pr_list(status=PRStatus.merged)
         prs_from_packit = [pr for pr in prs if pr.author in ("packit", "packit-stg")]
 
         if prs_from_packit:
-            db_project = GitProjectModel.get_by_id(project["id"])
+            db_project = GitProjectModel.get_by_id(project.id)
             db_project.set_onboarded_downstream(True)

packit_service/worker/tasks.py

@@ -4,7 +4,7 @@
 import logging
 import socket
 from os import getenv
-from typing import List, Optional, Dict
+from typing import List, Optional
 
 from celery import Task
 from celery._state import get_current_task
@@ -22,7 +22,11 @@
     DEFAULT_RETRY_BACKOFF,
     CELERY_DEFAULT_MAIN_TASK_NAME,
 )
-from packit_service.models import VMImageBuildTargetModel
+from packit_service.models import (
+    VMImageBuildTargetModel,
+    GitProjectModel,
+    SyncReleaseTargetModel,
+)
 from packit_service.utils import (
     load_job_config,
     load_package_config,
@@ -571,8 +575,11 @@ def babysit_pending_vm_image_builds() -> None:
 # Usage / statistics tasks
 
 
-@celery_app.task(
-    name=TaskName.check_onboarded_projects, queue="long-running", rate_limit="1/h"
-)
-def run_check_onboarded_projects(projects=List[Dict]) -> None:
-    check_onboarded_projects(projects)
+@celery_app.task
+def run_check_onboarded_projects() -> None:
+    known_onboarded_projects = GitProjectModel.get_known_onboarded_downstream_projects()
+    downstream_synced_projects = SyncReleaseTargetModel.get_all_downstream_projects()
+    almost_onboarded_projects = downstream_synced_projects.difference(
+        known_onboarded_projects
+    )
+    check_onboarded_projects(almost_onboarded_projects)