From 2bdb2153a1135da8ec822874f5327f41cf189b05 Mon Sep 17 00:00:00 2001 From: richardzilincikPantheon Date: Tue, 27 Jun 2023 11:42:06 +0200 Subject: [PATCH] chore: reduce log pollution and increase retention period --- api/my_flask.py | 2 +- api/views/health_check.py | 3 -- api/views/yang_search/yang_search.py | 40 +++++++++---------- gunicorn.conf.py | 11 +++++ opensearch_indexing/process-drafts.py | 4 +- opensearch_indexing/process_changed_mods.py | 1 - parseAndPopulate/resolvers/generated_from.py | 2 - parseAndPopulate/resolvers/module_type.py | 1 - parseAndPopulate/resolvers/namespace.py | 10 ++--- parseAndPopulate/resolvers/organization.py | 1 - parseAndPopulate/resolvers/prefix.py | 6 +-- parseAndPopulate/resolvers/revision.py | 3 +- .../resolvers/semantic_version.py | 1 - parseAndPopulate/resolvers/submodule.py | 1 - parseAndPopulate/resolvers/yang_version.py | 1 - yangcatalog-rotate | 12 +++--- 16 files changed, 47 insertions(+), 52 deletions(-) diff --git a/api/my_flask.py b/api/my_flask.py index ddf3b8f9..a404ddb2 100644 --- a/api/my_flask.py +++ b/api/my_flask.py @@ -126,7 +126,7 @@ def process_response(self, response): def preprocess_request(self): super().preprocess_request() g.special_id = 0 - if 'admin/' not in request.path or 'api/job/' not in request.path: + if not any(i in request.path for i in ('admin/', 'api/job', 'healthcheck/')): self.logger.info(request.path) client_ip = request.remote_addr data = MatomoTrackerData(self.config.m_matomo_api_url, self.config.m_matomo_site_id) diff --git a/api/views/health_check.py b/api/views/health_check.py index 57653801..4a52ce02 100644 --- a/api/views/health_check.py +++ b/api/views/health_check.py @@ -90,11 +90,9 @@ def health_check_opensearch(): try: # try to ping OpenSearch if app_config.opensearch_manager.ping(): - bp.logger.info('Successfully connected to OpenSearch') # get health of cluster health = app_config.opensearch_manager.cluster_health() health_status = health.get('status') - bp.logger.info('Health status of cluster: {}'.format(health_status)) # get list of indices indices = app_config.opensearch_manager.get_indices() if len(indices) > 0: @@ -162,7 +160,6 @@ def health_check_redis(): try: result = app_config.redis.ping() if result: - bp.logger.info('Redis ping responsed successfully') response = {'info': 'Success'} else: bp.logger.error('Redis ping failed to respond') diff --git a/api/views/yang_search/yang_search.py b/api/views/yang_search/yang_search.py index 41bfa620..fa13bd8f 100644 --- a/api/views/yang_search/yang_search.py +++ b/api/views/yang_search/yang_search.py @@ -610,32 +610,32 @@ def get_modules_revision_organization(module_name: str, revision: t.Optional[str :param warnings (bool) Whether return with warnings or not :return: tuple (list of revisions and organization) of specified module name """ - try: - if revision is None: - hits = app_config.opensearch_manager.get_sorted_module_revisions(OpenSearchIndices.YINDEX, module_name) - else: - module = {'name': module_name, 'revision': revision} - hits = app_config.opensearch_manager.get_module_by_name_revision(OpenSearchIndices.YINDEX, module) - - organization = hits[0]['_source']['organization'] - revisions = [] - for hit in hits: - hit = hit['_source'] - revision = hit['revision'] - revision_mat_level = { - 'revision': revision, - 'is_rfc': hit['rfc'] if 'rfc' in hit else False, - } - if revision_mat_level not in revisions: - revisions.append(revision_mat_level) - return revisions, organization - except IndexError: + if revision is None: + hits = app_config.opensearch_manager.get_sorted_module_revisions(OpenSearchIndices.YINDEX, module_name) + else: + module = {'name': module_name, 'revision': revision} + hits = app_config.opensearch_manager.get_module_by_name_revision(OpenSearchIndices.YINDEX, module) + + if not hits: name_rev = f'{module_name}@{revision}' if revision else module_name bp.logger.warning(f'Failed to get revisions and organization for {name_rev}') if warnings: return {'warning': f'Failed to find module {name_rev}'} abort(404, f'Failed to get revisions and organization for {name_rev}') + organization = hits[0]['_source']['organization'] + revisions = [] + for hit in hits: + hit = hit['_source'] + revision = hit['revision'] + revision_mat_level = { + 'revision': revision, + 'is_rfc': hit['rfc'] if 'rfc' in hit else False, + } + if revision_mat_level not in revisions: + revisions.append(revision_mat_level) + return revisions, organization + def get_latest_module_revision(module_name: str) -> str: """ diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 9713e712..3ec0efdb 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -1,3 +1,5 @@ +import logging + bind = 'unix:/var/run/yang/yang-catalog.sock' # umask = os.umask('007') @@ -19,3 +21,12 @@ # change log format access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' worker_class = 'gevent' + + +class HealthcheckFilter(logging.Filter): + def filter(self, record): + return record.getMessage().find('Amazon-Route53-Health-Check-Service') == -1 + + +def on_starting(server): + server.log.access_log.addFilter(HealthcheckFilter()) diff --git a/opensearch_indexing/process-drafts.py b/opensearch_indexing/process-drafts.py index bc3ae8ee..c049f188 100644 --- a/opensearch_indexing/process-drafts.py +++ b/opensearch_indexing/process-drafts.py @@ -73,12 +73,12 @@ def main(script_config: ScriptConfig = DEFAULT_SCRIPT_CONFIG.copy()) -> list[Job logger.info(f'Indexing draft {draft_name} - draft {i} out of {len(drafts)}') try: + # it takes an hour to do this with a request for each draft + # would be much more efficient to cache it in a file if not opensearch_manager.document_exists(OpenSearchIndices.DRAFTS, draft): opensearch_manager.index_module(OpenSearchIndices.DRAFTS, draft) logger.info(f'added {draft_name} to index') done += 1 - else: - logger.info(f'skipping - {draft_name} is already in index') except Exception: logger.exception(f'Problem while processing draft {draft_name}') logger.info('Job finished successfully') diff --git a/opensearch_indexing/process_changed_mods.py b/opensearch_indexing/process_changed_mods.py index cb3922ca..4a5eeb1e 100755 --- a/opensearch_indexing/process_changed_mods.py +++ b/opensearch_indexing/process_changed_mods.py @@ -73,7 +73,6 @@ def start_processing_changed_mods(self): self.delete_cache = self._load_delete_cache(self.delete_cache_path) if not self.changes_cache and not self.delete_cache: - self.logger.info('No new modules are added or removed. Exiting script!!!') os.unlink(self.lock_file) os.unlink(self.lock_file_cron) sys.exit() diff --git a/parseAndPopulate/resolvers/generated_from.py b/parseAndPopulate/resolvers/generated_from.py index dca3c319..f9219dc2 100644 --- a/parseAndPopulate/resolvers/generated_from.py +++ b/parseAndPopulate/resolvers/generated_from.py @@ -1,6 +1,5 @@ import logging import typing as t -from asyncio.log import logger from parseAndPopulate.resolvers.resolver import Resolver @@ -20,7 +19,6 @@ def __init__(self, logger: logging.Logger, name: str, namespace: t.Optional[str] self.namespace = namespace def resolve(self) -> str: - logger.debug('Resolving generated from') if self.namespace and ':smi' in self.namespace: return 'mib' if 'cisco' in self.name.lower(): diff --git a/parseAndPopulate/resolvers/module_type.py b/parseAndPopulate/resolvers/module_type.py index 76981fd9..03cce578 100644 --- a/parseAndPopulate/resolvers/module_type.py +++ b/parseAndPopulate/resolvers/module_type.py @@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None: self.logger = logger def resolve(self) -> t.Optional[str]: - self.logger.debug('Resolving module type') try: module_type = self.parsed_yang.keyword if module_type in ALLOWED: diff --git a/parseAndPopulate/resolvers/namespace.py b/parseAndPopulate/resolvers/namespace.py index 282de5ca..93fa77a9 100644 --- a/parseAndPopulate/resolvers/namespace.py +++ b/parseAndPopulate/resolvers/namespace.py @@ -30,7 +30,6 @@ def __init__( self.belongs_to = belongs_to def resolve(self) -> str: - self.logger.debug('Resolving namespace') module_type = self.parsed_yang.keyword if module_type == 'submodule': return self._resolve_submodule_namespace() @@ -39,25 +38,24 @@ def resolve(self) -> str: def _resolve_submodule_namespace(self) -> str: """If the model is a submodule, then it is necessary to get a namespace from its parent.""" if not self.belongs_to: - self.logger.error('Belongs to not defined - unable to resolve namespace') + self.logger.error(f'Belongs to not defined - unable to resolve namespace - {self.name_revision}') return MISSING_ELEMENT - self.logger.debug('Getting parent namespace - {} is a submodule'.format(self.name_revision)) yang_file = get_yang(self.belongs_to) if yang_file is None: - self.logger.error('Parent module not found - unable to resolve namespace') + self.logger.error(f'Parent module not found - unable to resolve namespace - {self.name_revision}') return MISSING_ELEMENT try: parsed_yang_parent = yangParser.parse(os.path.abspath(yang_file)) return parsed_yang_parent.search(self.property_name)[0].arg except IndexError: - self.logger.error('Cannot parse out {} property'.format(self.property_name)) + self.logger.error(f'Cannot parse out {self.property_name} property - {self.name_revision}') return MISSING_ELEMENT def _resolve_module_namespace(self) -> str: try: return self.parsed_yang.search(self.property_name)[0].arg except IndexError: - self.logger.error('Cannot parse out {} property'.format(self.property_name)) + self.logger.error(f'Cannot parse out {self.property_name} property - - {self.name_revision}') return MISSING_ELEMENT diff --git a/parseAndPopulate/resolvers/organization.py b/parseAndPopulate/resolvers/organization.py index 143bb288..e1f1d491 100644 --- a/parseAndPopulate/resolvers/organization.py +++ b/parseAndPopulate/resolvers/organization.py @@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger, namespace: t. self.namespace = namespace def resolve(self) -> str: - self.logger.debug('Resolving organization') try: parsed_organization = self.parsed_yang.search('organization')[0].arg.lower() for possible_organization in ORGANIZATIONS: diff --git a/parseAndPopulate/resolvers/prefix.py b/parseAndPopulate/resolvers/prefix.py index d0c01813..202b8ebb 100644 --- a/parseAndPopulate/resolvers/prefix.py +++ b/parseAndPopulate/resolvers/prefix.py @@ -31,7 +31,6 @@ def __init__( def resolve(self) -> t.Optional[str]: module_type = self.parsed_yang.keyword - self.logger.debug('Resolving prefix of {}'.format(module_type)) if module_type == 'submodule': return self._resolve_submodule_prefix() return self._resolve_module_prefix() @@ -42,17 +41,16 @@ def _resolve_submodule_prefix(self) -> t.Optional[str]: self.logger.error('Belongs to not defined - unable to resolve namespace') return DEFAULT - self.logger.debug('Getting parent namespace - {} is a submodule'.format(self.name_revision)) yang_file = get_yang(self.belongs_to) if yang_file is None: - self.logger.error('Parent module not found - unable to resolve namespace') + self.logger.error(f'Parent module not found - unable to resolve namespace - {self.name_revision}') return DEFAULT try: parsed_yang_parent = yangParser.parse(os.path.abspath(yang_file)) return parsed_yang_parent.search(self.property_name)[0].arg except IndexError: - self.logger.error('Cannot parse out {} property'.format(self.property_name)) + self.logger.error(f'Cannot parse out {self.property_name} property - {self.name_revision}') return DEFAULT def _resolve_module_prefix(self) -> t.Optional[str]: diff --git a/parseAndPopulate/resolvers/revision.py b/parseAndPopulate/resolvers/revision.py index 17bc7d73..57f56145 100644 --- a/parseAndPopulate/resolvers/revision.py +++ b/parseAndPopulate/resolvers/revision.py @@ -21,11 +21,10 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None: self.logger = logger def resolve(self) -> str: - self.logger.debug('Resolving revision') try: revision = self.parsed_yang.search('revision')[0].arg except IndexError: - self.logger.exception('Error while resolving revision') + self.logger.exception(f'Error while resolving revision - {self.parsed_yang.arg}') revision = DEFAULT return validate_revision(revision) diff --git a/parseAndPopulate/resolvers/semantic_version.py b/parseAndPopulate/resolvers/semantic_version.py index d98b5f52..61f37b6b 100644 --- a/parseAndPopulate/resolvers/semantic_version.py +++ b/parseAndPopulate/resolvers/semantic_version.py @@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None: self.logger = logger def resolve(self) -> t.Optional[str]: - self.logger.debug('Resolving semantic version') # cisco specific modules - semver defined is inside revision - cisco-semver:module-version try: parsed_semver = self.parsed_yang.search('revision')[0].search(('cisco-semver', 'module-version'))[0].arg diff --git a/parseAndPopulate/resolvers/submodule.py b/parseAndPopulate/resolvers/submodule.py index d10e29e7..7d91ff2e 100644 --- a/parseAndPopulate/resolvers/submodule.py +++ b/parseAndPopulate/resolvers/submodule.py @@ -21,7 +21,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger, domain_prefix self.domain_prefix = domain_prefix def resolve(self) -> t.Tuple[list, list]: - self.logger.debug('Resolving submodules') submodules = [] dependencies = [] parsed_submodules = self.parsed_yang.search('include') diff --git a/parseAndPopulate/resolvers/yang_version.py b/parseAndPopulate/resolvers/yang_version.py index cc7852b1..652b4e1e 100644 --- a/parseAndPopulate/resolvers/yang_version.py +++ b/parseAndPopulate/resolvers/yang_version.py @@ -19,7 +19,6 @@ def __init__(self, parsed_yang: Statement, logger: logging.Logger) -> None: self.logger = logger def resolve(self) -> str: - self.logger.debug('Resolving yang version') try: yang_version = self.parsed_yang.search('yang-version')[0].arg except IndexError: diff --git a/yangcatalog-rotate b/yangcatalog-rotate index d6a33659..673ef89f 100644 --- a/yangcatalog-rotate +++ b/yangcatalog-rotate @@ -1,6 +1,6 @@ /var/yang/logs/*.log { - rotate 7 - daily + rotate 52 + weekly compress missingok copytruncate @@ -9,8 +9,8 @@ } /var/yang/logs/*/*.log { - rotate 7 - daily + rotate 52 + weekly compress missingok copytruncate @@ -19,8 +19,8 @@ } /var/yang/logs/*.access { - rotate 7 - daily + rotate 52 + weekly compress missingok copytruncate