Merge pull request #1341 from Bidaya0/fix/change-logging-level

fix: change logging level to reduce log cost.

dongtai_common/endpoint/__init__.py

@@ -32,6 +32,7 @@
 from dongtai_common.models.department import Department
 from functools import reduce
 from operator import ior
+from rest_framework.exceptions import AuthenticationFailed
 
 if TYPE_CHECKING:
     from django.core.paginator import _SupportsPagination
@@ -107,9 +108,13 @@ def dispatch(self, request, *args, **kwargs):
             else:
                 handler = self.http_method_not_allowed
             response = handler(request, *args, **kwargs)
+        except AuthenticationFailed as exc:
+            logger.debug(f'url: {self.request.path},exc:{exc}')
+            response = self.handle_exception(exc)
         except Exception as exc:
-            logger.error(f'url: {self.request.path},exc:{exc}', exc_info=True)
+            logger.warning(f'url: {self.request.path},exc:{exc}', exc_info=exc)
             response = self.handle_exception(exc)
+        finally:
             return self.finalize_response(request, response, *args, **kwargs)
 
         self.response = self.finalize_response(request, response, *args,
@@ -198,7 +203,7 @@ def get_paginator(
         except EmptyPage:
             return page_summary, queryset.none()
         except BaseException as e:
-            logger.error(e, exc_info=e)
+            logger.info(e, exc_info=e)
             return page_summary, queryset.none()
         return page_summary, page_list
 

dongtai_common/utils/http.py

@@ -20,5 +20,5 @@ def build_response(header, body):
         _data = base64.b64decode(header.encode("utf-8")).decode("utf-8")
     except Exception as e:
         _data = ''
-        logger.error(f'Response Header解析出错，错误原因：{e}')
+        logger.warning(f'Response Header解析出错，错误原因：{e}', exc_info=e)
     return '{header}\n\n{body}'.format(header=_data, body=body)

dongtai_engine/plugins/strategy_headers.py

@@ -98,8 +98,10 @@ def check_response_header(method_pool):
                      method_pool,
                      position='HTTP Response Header')
     except Exception as e:
-        logger.error("check_response_header failed, reason: " + str(e),
-                     exc_info=e)
+        logger.warning(
+            "check_response_header failed, reason: " + str(e),
+            exc_info=e,
+        )
 
 
 def save_vul(vul_type, method_pool, position=None, data=None):
@@ -110,7 +112,7 @@ def save_vul(vul_type, method_pool, position=None, data=None):
         state=const.STRATEGY_ENABLE,
         user_id__in=(1, method_pool.agent.user.id)).first()
     if vul_strategy is None:
-        logger.error(
+        logger.warning(
             f'There is no corresponding strategy for the current vulnerability: {vul_type}'
         )
 

dongtai_engine/plugins/strategy_sensitive.py

@@ -49,9 +49,9 @@ def check_response_content(method_pool):
                                      position=key,
                                      data=result.group(0))
                     except Exception as e:
-                        logger.error(
+                        logger.warning(
                             f'check_response_content error, rule: {rule.id}, rule name: {rule.strategy.vul_type}, reason: {e}',
-                            exc_info=True)
+                            exc_info=e)
             elif json_response and rule.pattern_type.id == 2:
                 pattern = jq.compile(rule.pattern)
                 result = pattern.input(json_response).all()
@@ -61,9 +61,9 @@ def check_response_content(method_pool):
                              position='HTTP Response Body',
                              data=' '.join(result))
         except Exception as e:
-            logger.error(
+            logger.warning(
                 f'check_response_content error, rule: {rule.id}, rule name: {rule.strategy.vul_type}, reason: {e}',
-                exc_info=True)
+                exc_info=e)
 
     search_id_card_leak(method_pool)
 
@@ -89,8 +89,9 @@ def search_id_card_leak(method_pool):
                 # todo: add highlight to id_card
                 save_vul(vul_type='ID Number Leak', method_pool=method_pool, position=key, data=card)
         except Exception as e:
-            logger.error(
-                f'check_response_content error, rule name: ID Number Leak, Method Pool ID: {method_pool.id}, reason: {e}')
+            logger.warning(
+                f'check_response_content error, rule name: ID Number Leak, Method Pool ID: {method_pool.id}, reason: {e}',
+                exc_info=e)
 
 
 def check_id_card(id_card):

dongtai_engine/tasks.py

@@ -135,7 +135,7 @@ def search_and_save_vul(engine: Optional[VulEngine],
         )
         return
     if not queryset.values('id').exists():
-        logger.error(
+        logger.warning(
             f'current method pool hit rule {strategy.get("type")}, but no vul strategy.'
         )
         return
@@ -240,7 +240,7 @@ def search_vul_from_method_pool(self, method_pool_sign, agent_id, retryable=Fals
                     tries = self.request.retries + 1
                     raise RetryableException(f'漏洞检测方法池 {method_pool_sign} 不存在，重试第 {tries} 次')
                 else:
-                    logger.error(f'漏洞检测超过最大重试次数 {self.max_retries}，方法池 {method_pool_sign} 不存在')
+                    logger.warning(f'漏洞检测超过最大重试次数 {self.max_retries}，方法池 {method_pool_sign} 不存在')
             else:
                 logger.warning(f'漏洞检测终止，方法池 {method_pool_sign} 不存在')
             return
@@ -270,10 +270,9 @@ def search_vul_from_method_pool(self, method_pool_sign, agent_id, retryable=Fals
             delay = 5 + pow(3, self.request.retries) * 10
             self.retry(exc=e, countdown=delay)
         else:
-            logger.error(f'漏洞检测超过最大重试次数，错误原因：{e}')
+            logger.info(f'漏洞检测超过最大重试次数，错误原因：{e}')
     except Exception as e:
-        logger.error(e, exc_info=True)
-        logger.error(f'漏洞检测出错，方法池 {method_pool_sign}. 错误原因：{e}')
+        logger.error(f'漏洞检测出错，方法池 {method_pool_sign}. 错误原因：{e}', exc_info=e)
 
 
 @shared_task(queue='dongtai-replay-vul-scan')
@@ -559,7 +558,7 @@ def vul_recheck():
                 try:
                     params = json.loads(vulnerability['param_name'])
                 except JSONDecodeError as e:
-                    logger.error(f'污点数据解析出错，原因：{e}')
+                    logger.warning(f'污点数据解析出错，原因：{e}', exc_info=e)
                     Replay.replay_failed(replay=replay, timestamp=timestamp)
                     con = 1
             else:
@@ -619,7 +618,7 @@ def vul_recheck():
                         try:
                             headers = base64.b64encode('\n'.join(header_raw))
                         except Exception as e:
-                            logger.error(f'请求头解析失败，漏洞ID: {vulnerability["id"]}')
+                            logger.warning(f'请求头解析失败，漏洞ID: {vulnerability["id"]}', exc_info=e)
                     elif position == 'COOKIE':
                         import base64
                         header_raw = base64.b64decode(headers).decode('utf-8').split('\n')

dongtai_protocol/report/handler/saas_method_pool_handler.py

@@ -191,13 +191,13 @@ def save(self):
                         logger.info(
                             f"record method failed : {self.agent_id} {self.http_uri} {self.http_method}"
                         )
-                        logger.warning(e, exc_info=True)
+                        logger.warning(e, exc_info=e)
                 try:
                     logger.info(f"send normal method pool {self.agent_id} {self.http_uri} {pool_sign} to celery ")
                     self.send_to_engine(method_pool_sign=pool_sign,
                                         update_record=update_record)
                 except Exception as e:
-                    logger.warning(e, exc_info=True)
+                    logger.warning(e, exc_info=e)
 
     def to_json(self, pool_sign: str):
         timestamp = int(time.time())
@@ -369,7 +369,7 @@ def send_to_engine(self, method_pool_id="", method_pool_sign="", update_record=F
                 )
                 # requests.get(url=settings.REPLAY_ENGINE_URL.format(id=method_pool_id))
         except Exception as e:
-            logger.warning(f'[-] Failure: send method_pool [{method_pool_id}{method_pool_sign}], Error: {e}')
+            logger.error(f'[-] Failure: send method_pool [{method_pool_id}{method_pool_sign}], Error: {e}', exc_info=e)
 
     def calc_hash(self):
         sign_raw = '-'.join(

dongtai_protocol/report/report_handler_factory.py

@@ -75,7 +75,7 @@ def handler(reports, user):
             result = class_of_handler().handle(reports, user)
             return result
         except Exception as e:
-            logger.error(e, exc_info=True)
+            logger.error(e, exc_info=e)
         return None
 
     @classmethod

dongtai_protocol/views/report_upload.py

@@ -36,5 +36,5 @@ def post(self, request):
             data = ReportHandler.handler(report, request.user)
             return R.success(msg="report upload success.", data=data)
         except Exception as e:
-            logger.error(f"report upload failed, reason: {e}", exc_info=True)
+            logger.error(f"report upload failed, reason: {e}", exc_info=e)
             return R.failure(msg="report upload failed")