diff --git a/packit_service/celery_config.py b/packit_service/celery_config.py index 47da0e10e..56d91a987 100644 --- a/packit_service/celery_config.py +++ b/packit_service/celery_config.py @@ -29,6 +29,11 @@ "schedule": crontab(minute=0, hour=1), # nightly at 1AM "options": {"queue": "long-running"}, }, + "check-onboarded-projects": { + "task": "packit_service.worker.tasks.run_check_onboarded_projects", + "schedule": crontab(minute=0, hour=2), # nightly at 2AM + "options": {"queue": "long-running"}, + }, } # http://mher.github.io/flower/prometheus-integration.html#set-up-your-celery-application diff --git a/packit_service/service/api/usage.py b/packit_service/service/api/usage.py index 9e0f35100..1de86d16d 100644 --- a/packit_service/service/api/usage.py +++ b/packit_service/service/api/usage.py @@ -26,7 +26,6 @@ SyncReleaseTargetModel, ) from packit_service.service.api.utils import response_maker -from packit_service.celerizer import celery_app logger = getLogger("packit_service") @@ -656,27 +655,6 @@ def calculate(cls): known_onboarded_projects ) - if recheck_if_onboarded: - # Run the long running task in Celery. - # The task will collect data about merged PR for - # every given project. If a Packit merged PR is - # found the long running task will save the - # onboarded_downstream flag in the git projects table. - celery_app.send_task( - name="task.check_onboarded_projects", - kwargs={ - "projects": [ - { - "id": project.id, - "instance_url": project.instance_url, - "namespace": project.namespace, - "repo_name": project.repo_name, - } - for project in recheck_if_onboarded - ] - }, - ) - onboarded = { project.id: project.project_url for project in onboarded_projects.union(known_onboarded_projects) diff --git a/packit_service/worker/handlers/usage.py b/packit_service/worker/handlers/usage.py index 34bee061c..ae0ca1d23 100644 --- a/packit_service/worker/handlers/usage.py +++ b/packit_service/worker/handlers/usage.py @@ -1,20 +1,20 @@ # Copyright Contributors to the Packit project. # SPDX-License-Identifier: MIT -from typing import List, Dict +from typing import Set from ogr.abstract import PRStatus from packit_service.config import ServiceConfig from packit_service.models import GitProjectModel -def check_onboarded_projects(projects: List[Dict]): +def check_onboarded_projects(projects: Set[GitProjectModel]): """For every given project check if it has a merged Packit PR. If yes it is onboarded: save the flag in the git projects table. """ for project in projects: - downstream_project_url = f"https://{project['instance_url']}" + downstream_project_url = f"https://{project.instance_url}" if downstream_project_url != "https://src.fedoraproject.org": continue @@ -24,12 +24,12 @@ def check_onboarded_projects(projects: List[Dict]): if service.instance_url == downstream_project_url ][0] ogr_project = pagure_service.get_project( - namespace=project["namespace"], - repo=project["repo_name"], + namespace=project.namespace, + repo=project.repo_name, ) prs = ogr_project.get_pr_list(status=PRStatus.merged) prs_from_packit = [pr for pr in prs if pr.author in ("packit", "packit-stg")] if prs_from_packit: - db_project = GitProjectModel.get_by_id(project["id"]) + db_project = GitProjectModel.get_by_id(project.id) db_project.set_onboarded_downstream(True) diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index b7161e266..d6da4cad9 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -4,7 +4,7 @@ import logging import socket from os import getenv -from typing import List, Optional, Dict +from typing import List, Optional from celery import Task from celery._state import get_current_task @@ -22,7 +22,11 @@ DEFAULT_RETRY_BACKOFF, CELERY_DEFAULT_MAIN_TASK_NAME, ) -from packit_service.models import VMImageBuildTargetModel +from packit_service.models import ( + VMImageBuildTargetModel, + GitProjectModel, + SyncReleaseTargetModel, +) from packit_service.utils import ( load_job_config, load_package_config, @@ -571,8 +575,11 @@ def babysit_pending_vm_image_builds() -> None: # Usage / statistics tasks -@celery_app.task( - name=TaskName.check_onboarded_projects, queue="long-running", rate_limit="1/h" -) -def run_check_onboarded_projects(projects=List[Dict]) -> None: - check_onboarded_projects(projects) +@celery_app.task +def run_check_onboarded_projects() -> None: + known_onboarded_projects = GitProjectModel.get_known_onboarded_downstream_projects() + downstream_synced_projects = SyncReleaseTargetModel.get_all_downstream_projects() + almost_onboarded_projects = downstream_synced_projects.difference( + known_onboarded_projects + ) + check_onboarded_projects(almost_onboarded_projects)