Merge pull request #1585 from Bidaya0/feat/celery-project-timestamp-u…

…pdate

Feat/celery project timestamp update

dongtai_conf/celery.py

@@ -71,6 +71,7 @@
     "dongtai_protocol.report.handler.api_route_handler.api_route_gather": {'queue': 'dongtai-api-route-handler', 'routing_key': 'dongtai-api-route-handler'},
     "dongtai_engine.tasks.search_vul_from_method_pool": {'queue': 'dongtai-method-pool-scan', 'routing_key': 'dongtai-method-pool-scan'},
     "dongtai_engine.plugins.project_time_update.project_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'},
+    "dongtai_engine.plugins.project_time_update.project_version_time_stamp_update": {'queue': 'dongtai-project-time-stamp-update', 'routing_key': 'dongtai-project-time-stamp-update'},
     "dongtai_engine.tasks.search_vul_from_replay_method_pool": {'exchange': 'dongtai-replay-vul-scan', 'routing_key': 'dongtai-replay-vul-scan'},
     "dongtai_web.dongtai_sca.scan.utils.update_one_sca": {'exchange': 'dongtai-sca-task', 'routing_key': 'dongtai-sca-task'},
     "dongtai_engine.preheat.function_flush": {'exchange': 'dongtai-function-flush-data', 'routing_key': 'dongtai-function-flush-data'},

dongtai_engine/plugins/project_time_update.py

@@ -2,6 +2,8 @@
 from celery import shared_task
 from time import time
 from dongtai_common.models.project import IastProject
+from dongtai_common.models.project_version import IastProjectVersion
+
 
 @shared_task(
     queue='dongtai-project-time-stamp-update',
@@ -11,3 +13,14 @@
 def project_time_stamp_update(project_id):
     timestamp = int(time())
     IastProject.objects.filter(pk=project_id).update(latest_time=timestamp)
+
+
+@shared_task(
+    queue='dongtai-project-time-stamp-update',
+    base=Singleton,
+    lock_expiry=20,
+)
+def project_version_time_stamp_update(project_version_id):
+    timestamp = int(time())
+    IastProjectVersion.objects.filter(pk=project_version_id).update(
+        update_time=timestamp)

dongtai_engine/plugins/strategy_headers.py

@@ -19,7 +19,10 @@
 from dongtai_web.vul_log.vul_log import log_vul_found
 from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail
 from django.db import IntegrityError
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from dongtai_engine.signals import send_notify
 
 
@@ -138,6 +141,8 @@ def save_vul(vul_type, method_pool, position=None, data=None):
     timestamp = int(time.time())
     project_time_stamp_update.apply_async(
         (method_pool.agent.bind_project_id, ), countdown=5)
+    project_version_time_stamp_update.apply_async(
+        (method_pool.agent.project_version_id, ), countdown=5)
     if vul:
         vul.url = ''
         vul.req_header = method_pool.req_header

dongtai_engine/signals/handlers/vul_handler.py

@@ -21,7 +21,10 @@
 from typing import List, Optional, Callable
 from collections import defaultdict
 from dongtai_common.models.profile import IastProfile
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from dongtai_engine.signals import send_notify
 
 
@@ -289,6 +292,8 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack,
     ).order_by('-latest_time').first()
     project_time_stamp_update.apply_async(
         (vul_meta.agent.bind_project_id, ), countdown=5)
+    project_version_time_stamp_update.apply_async(
+        (vul_meta.agent.project_version_id, ), countdown=5)
     if vul:
         vul.url = vul_meta.url
         vul.uri = vul_meta.uri
@@ -448,6 +453,8 @@ def handler_replay_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack,
             update_fields=['status_id', 'latest_time', 'latest_time_desc'])
         project_time_stamp_update.apply_async(
             (vul_meta.agent.bind_project_id, ), countdown=5)
+        project_version_time_stamp_update.apply_async(
+            (vul_meta.agent.project_version_id, ), countdown=5)
 
         IastReplayQueue.objects.filter(id=kwargs['replay_id']).update(
             state=const.SOLVED,

dongtai_engine/tasks.py

@@ -37,7 +37,10 @@
 import requests
 from dongtai_engine.task_base import replay_payload_data
 from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from dongtai_web.vul_log.vul_log import log_recheck_vul
 
 RETRY_INTERVALS = [10, 30, 90]
@@ -160,6 +163,8 @@ def search_and_save_vul(engine: Optional[VulEngine],
                 update_time=timestamp)
             project_time_stamp_update.apply_async(
                 (method_pool_model.agent.bind_project_id, ), countdown=5)
+            project_version_time_stamp_update.apply_async(
+                (method_pool_model.agent.project_version_id, ), countdown=5)
         except Exception as e:
             logger.info(f'漏洞数据处理出错，原因：{e}')
 

dongtai_protocol/report/handler/api_route_handler.py

@@ -17,7 +17,10 @@
 from django.utils.translation import gettext_lazy as _
 from django.db import transaction
 from dongtai_common.models.project import IastProject
-from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.plugins.project_time_update import (
+    project_time_stamp_update,
+    project_version_time_stamp_update,
+)
 from celery import shared_task
 
 logger = logging.getLogger('dongtai.openapi')
@@ -81,6 +84,8 @@ def api_route_gather(agent_id, api_routes):
             logger.info(_('API navigation log record successfully'))
         project_time_stamp_update.apply_async((agent.bind_project_id, ),
                                               countdown=5)
+        project_version_time_stamp_update.apply_async(
+            (agent.project_version_id, ), countdown=5)
     except Exception as e:
         logger.info(_('API navigation log failed, why: {}').format(e),
                     exc_info=e)

dongtai_web/views/project_summary.py

@@ -8,6 +8,7 @@
 from dongtai_common.endpoint import UserEndPoint
 from dongtai_common.models.agent import IastAgent
 from dongtai_common.models.project import IastProject
+from dongtai_common.models.project_version import IastProjectVersion
 from dongtai_common.models.vul_level import IastVulLevel
 from dongtai_common.models.vulnerablity import IastVulnerabilityModel
 from dongtai_web.base.project_version import get_project_version, get_project_version_by_id, ProjectsVersionDataSerializer
@@ -119,8 +120,7 @@ def get(self, request, id):
         data['level_count'] = []
 
         if not version_id:
-            current_project_version = get_project_version(
-                project.id)
+            current_project_version = get_project_version(project.id)
         else:
             current_project_version = get_project_version_by_id(version_id)
         data['versionData'] = current_project_version
@@ -134,4 +134,9 @@ def get(self, request, id):
             project).data['agent_language']
         data['agent_alive'] = IastAgent.objects.filter(
             bind_project_id=project.id, online=const.RUNNING).count()
+        project_version = IastProjectVersion.objects.filter(
+            pk=current_project_version.get("version_id", 0)).first()
+        data[
+            'project_version_latest_time'] = project_version.update_time if project_version else project.latest_time
+        data['type_summary'] = []
         return R.success(data=data)