chore: reduce log pollution and increase retention period

api/my_flask.py

@@ -126,7 +126,7 @@ def process_response(self, response):
     def preprocess_request(self):
         super().preprocess_request()
         g.special_id = 0
-        if 'admin/' not in request.path or 'api/job/' not in request.path:
+        if not any(i in request.path for i in ('admin/', 'api/job', 'healthcheck/')):
             self.logger.info(request.path)
             client_ip = request.remote_addr
             data = MatomoTrackerData(self.config.m_matomo_api_url, self.config.m_matomo_site_id)

api/views/health_check.py

@@ -90,11 +90,9 @@ def health_check_opensearch():
     try:
         # try to ping OpenSearch
         if app_config.opensearch_manager.ping():
-            bp.logger.info('Successfully connected to OpenSearch')
             # get health of cluster
             health = app_config.opensearch_manager.cluster_health()
             health_status = health.get('status')
-            bp.logger.info('Health status of cluster: {}'.format(health_status))
             # get list of indices
             indices = app_config.opensearch_manager.get_indices()
             if len(indices) > 0:
@@ -162,7 +160,6 @@ def health_check_redis():
     try:
         result = app_config.redis.ping()
         if result:
-            bp.logger.info('Redis ping responsed successfully')
             response = {'info': 'Success'}
         else:
             bp.logger.error('Redis ping failed to respond')

api/views/yang_search/yang_search.py

@@ -610,32 +610,32 @@ def get_modules_revision_organization(module_name: str, revision: t.Optional[str
         :param warnings         (bool) Whether return with warnings or not
     :return: tuple (list of revisions and organization) of specified module name
     """
-    try:
-        if revision is None:
-            hits = app_config.opensearch_manager.get_sorted_module_revisions(OpenSearchIndices.YINDEX, module_name)
-        else:
-            module = {'name': module_name, 'revision': revision}
-            hits = app_config.opensearch_manager.get_module_by_name_revision(OpenSearchIndices.YINDEX, module)
-
-        organization = hits[0]['_source']['organization']
-        revisions = []
-        for hit in hits:
-            hit = hit['_source']
-            revision = hit['revision']
-            revision_mat_level = {
-                'revision': revision,
-                'is_rfc': hit['rfc'] if 'rfc' in hit else False,
-            }
-            if revision_mat_level not in revisions:
-                revisions.append(revision_mat_level)
-        return revisions, organization
-    except IndexError:
+    if revision is None:
+        hits = app_config.opensearch_manager.get_sorted_module_revisions(OpenSearchIndices.YINDEX, module_name)
+    else:
+        module = {'name': module_name, 'revision': revision}
+        hits = app_config.opensearch_manager.get_module_by_name_revision(OpenSearchIndices.YINDEX, module)
+
+    if not hits:
         name_rev = f'{module_name}@{revision}' if revision else module_name
         bp.logger.warning(f'Failed to get revisions and organization for {name_rev}')
         if warnings:
             return {'warning': f'Failed to find module {name_rev}'}
         abort(404, f'Failed to get revisions and organization for {name_rev}')
 
+    organization = hits[0]['_source']['organization']
+    revisions = []
+    for hit in hits:
+        hit = hit['_source']
+        revision = hit['revision']
+        revision_mat_level = {
+            'revision': revision,
+            'is_rfc': hit['rfc'] if 'rfc' in hit else False,
+        }
+        if revision_mat_level not in revisions:
+            revisions.append(revision_mat_level)
+    return revisions, organization
+
 
 def get_latest_module_revision(module_name: str) -> str:
     """

gunicorn.conf.py

@@ -1,3 +1,5 @@
+import logging
+
 bind = 'unix:/var/run/yang/yang-catalog.sock'
 # umask = os.umask('007')
 
@@ -19,3 +21,13 @@
 # change log format
 access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
 worker_class = 'gevent'
+
+
+class HealthcheckFilter(logging.Filter):
+    def filter(self, record):
+        return record.getMessage().find('Amazon-Route53-Health-Check-Service') == -1
+
+
+def on_starting(server):
+    server.log.access_log.addFilter(HealthcheckFilter())
+    server.log.error_log.addFilter(HealthcheckFilter())

opensearch_indexing/process-drafts.py

@@ -73,12 +73,12 @@ def main(script_config: ScriptConfig = DEFAULT_SCRIPT_CONFIG.copy()) -> list[Job
         logger.info(f'Indexing draft {draft_name} - draft {i} out of {len(drafts)}')
 
         try:
+            # it takes an hour to do this with a request for each draft
+            # would be much more efficient to cache it in a file
             if not opensearch_manager.document_exists(OpenSearchIndices.DRAFTS, draft):
                 opensearch_manager.index_module(OpenSearchIndices.DRAFTS, draft)
                 logger.info(f'added {draft_name} to index')
                 done += 1
-            else:
-                logger.info(f'skipping - {draft_name} is already in index')
         except Exception:
             logger.exception(f'Problem while processing draft {draft_name}')
     logger.info('Job finished successfully')

opensearch_indexing/process_changed_mods.py

@@ -73,7 +73,6 @@ def start_processing_changed_mods(self):
         self.delete_cache = self._load_delete_cache(self.delete_cache_path)
 
         if not self.changes_cache and not self.delete_cache:
-            self.logger.info('No new modules are added or removed. Exiting script!!!')
             os.unlink(self.lock_file)
             os.unlink(self.lock_file_cron)
             sys.exit()

parseAndPopulate/resolvers/generated_from.py

@@ -1,6 +1,5 @@
 import logging
 import typing as t
-from asyncio.log import logger
 
 from parseAndPopulate.resolvers.resolver import Resolver
 
@@ -20,7 +19,6 @@ def __init__(self, logger: logging.Logger, name: str, namespace: t.Optional[str]
         self.namespace = namespace
 
     def resolve(self) -> str:
-        logger.debug('Resolving generated from')
         if self.namespace and ':smi' in self.namespace:
             return 'mib'
         if 'cisco' in self.name.lower():

parseAndPopulate/resolvers/module_type.py

@@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None:
         self.logger = logger
 
     def resolve(self) -> t.Optional[str]:
-        self.logger.debug('Resolving module type')
         try:
             module_type = self.parsed_yang.keyword
             if module_type in ALLOWED:

parseAndPopulate/resolvers/namespace.py

@@ -30,7 +30,6 @@ def __init__(
         self.belongs_to = belongs_to
 
     def resolve(self) -> str:
-        self.logger.debug('Resolving namespace')
         module_type = self.parsed_yang.keyword
         if module_type == 'submodule':
             return self._resolve_submodule_namespace()
@@ -39,25 +38,24 @@ def resolve(self) -> str:
     def _resolve_submodule_namespace(self) -> str:
         """If the model is a submodule, then it is necessary to get a namespace from its parent."""
         if not self.belongs_to:
-            self.logger.error('Belongs to not defined - unable to resolve namespace')
+            self.logger.error(f'Belongs to not defined - unable to resolve namespace - {self.name_revision}')
             return MISSING_ELEMENT
 
-        self.logger.debug('Getting parent namespace - {} is a submodule'.format(self.name_revision))
         yang_file = get_yang(self.belongs_to)
         if yang_file is None:
-            self.logger.error('Parent module not found - unable to resolve namespace')
+            self.logger.error(f'Parent module not found - unable to resolve namespace - {self.name_revision}')
             return MISSING_ELEMENT
 
         try:
             parsed_yang_parent = yangParser.parse(os.path.abspath(yang_file))
             return parsed_yang_parent.search(self.property_name)[0].arg
         except IndexError:
-            self.logger.error('Cannot parse out {} property'.format(self.property_name))
+            self.logger.error(f'Cannot parse out {self.property_name} property - {self.name_revision}')
             return MISSING_ELEMENT
 
     def _resolve_module_namespace(self) -> str:
         try:
             return self.parsed_yang.search(self.property_name)[0].arg
         except IndexError:
-            self.logger.error('Cannot parse out {} property'.format(self.property_name))
+            self.logger.error(f'Cannot parse out {self.property_name} property -  - {self.name_revision}')
             return MISSING_ELEMENT

parseAndPopulate/resolvers/organization.py

@@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger, namespace: t.
         self.namespace = namespace
 
     def resolve(self) -> str:
-        self.logger.debug('Resolving organization')
         try:
             parsed_organization = self.parsed_yang.search('organization')[0].arg.lower()
             for possible_organization in ORGANIZATIONS:

parseAndPopulate/resolvers/prefix.py

@@ -31,7 +31,6 @@ def __init__(
 
     def resolve(self) -> t.Optional[str]:
         module_type = self.parsed_yang.keyword
-        self.logger.debug('Resolving prefix of {}'.format(module_type))
         if module_type == 'submodule':
             return self._resolve_submodule_prefix()
         return self._resolve_module_prefix()
@@ -42,17 +41,16 @@ def _resolve_submodule_prefix(self) -> t.Optional[str]:
             self.logger.error('Belongs to not defined - unable to resolve namespace')
             return DEFAULT
 
-        self.logger.debug('Getting parent namespace - {} is a submodule'.format(self.name_revision))
         yang_file = get_yang(self.belongs_to)
         if yang_file is None:
-            self.logger.error('Parent module not found - unable to resolve namespace')
+            self.logger.error(f'Parent module not found - unable to resolve namespace - {self.name_revision}')
             return DEFAULT
 
         try:
             parsed_yang_parent = yangParser.parse(os.path.abspath(yang_file))
             return parsed_yang_parent.search(self.property_name)[0].arg
         except IndexError:
-            self.logger.error('Cannot parse out {} property'.format(self.property_name))
+            self.logger.error(f'Cannot parse out {self.property_name} property - {self.name_revision}')
             return DEFAULT
 
     def _resolve_module_prefix(self) -> t.Optional[str]:

parseAndPopulate/resolvers/revision.py

@@ -21,11 +21,10 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None:
         self.logger = logger
 
     def resolve(self) -> str:
-        self.logger.debug('Resolving revision')
         try:
             revision = self.parsed_yang.search('revision')[0].arg
         except IndexError:
-            self.logger.exception('Error while resolving revision')
+            self.logger.exception(f'Error while resolving revision - {self.parsed_yang.arg}')
             revision = DEFAULT
 
         return validate_revision(revision)

parseAndPopulate/resolvers/semantic_version.py

@@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None:
         self.logger = logger
 
     def resolve(self) -> t.Optional[str]:
-        self.logger.debug('Resolving semantic version')
         # cisco specific modules - semver defined is inside revision - cisco-semver:module-version
         try:
             parsed_semver = self.parsed_yang.search('revision')[0].search(('cisco-semver', 'module-version'))[0].arg

parseAndPopulate/resolvers/submodule.py

@@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger, domain_prefix
         self.domain_prefix = domain_prefix
 
     def resolve(self) -> t.Tuple[list, list]:
-        self.logger.debug('Resolving submodules')
         submodules = []
         dependencies = []
         parsed_submodules = self.parsed_yang.search('include')

parseAndPopulate/resolvers/yang_version.py

@@ -19,7 +19,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None:
         self.logger = logger
 
     def resolve(self) -> str:
-        self.logger.debug('Resolving yang version')
         try:
             yang_version = self.parsed_yang.search('yang-version')[0].arg
         except IndexError:

yangcatalog-rotate

@@ -1,6 +1,6 @@
 /var/yang/logs/*.log {
-    rotate 7
-    daily
+    rotate 52
+    weekly
     compress
     missingok
     copytruncate
@@ -9,8 +9,8 @@
 }
 
 /var/yang/logs/*/*.log {
-    rotate 7
-    daily
+    rotate 52
+    weekly
     compress
     missingok
     copytruncate
@@ -19,8 +19,8 @@
 }
 
 /var/yang/logs/*.access {
-    rotate 7
-    daily
+    rotate 52
+    weekly
     compress
     missingok
     copytruncate