chore: elasticsearch -> opensearch

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         python-version: ['3.10']
         redis-version: [6]
-        elasticsearch-version: [7.10.1]
+        opensearch-version: [2.7.0]
       max-parallel: 1
 
     steps:
@@ -65,13 +65,10 @@ jobs:
           sudo sysctl -w vm.swappiness=1
           sudo sysctl -w vm.max_map_count=262144
 
-      - name: Setup Elasticsearch ${{ matrix.elasticsearch-version }}
-        uses: getong/elasticsearch-action@v1.2
+      - name: Setup OpenSearch ${{ matrix.opensearch-version }}
+        uses: ankane/setup-opensearch@v1
         with:
-          elasticsearch version: ${{ matrix.elasticsearch-version }}
-          host port: 9200
-          container port: 9200
-          discovery type: 'single-node'
+          opensearch-version: ${{ matrix.elasticsearch-version }}
 
       - name: Prepare environment
         run: |

.gitignore

@@ -10,7 +10,7 @@ api/standard
 **/*.html
 !statistics/template/stats.html
 !api/template/*.html
-!elasticsearchIndexing/json/**/*
+!opensearch_indexing/json/**/*
 !api/views/yang_search/json/*
 /venv
 yang.egg-info

.vscode/launch.json

@@ -125,7 +125,7 @@
       "name": "process_changed_mods",
       "type": "python",
       "request": "launch",
-      "program": "${workspaceFolder}/elasticsearchIndexing/process_changed_mods.py",
+      "program": "${workspaceFolder}/opensearch_indexing/process_changed_mods.py",
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}",
       "justMyCode": false

Dockerfile

@@ -12,7 +12,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONUNBUFFERED=1
 
 ENV VIRTUAL_ENV=/backend
 ENV BACKEND=/backend
-ENV PYANG_PLUGINPATH="$BACKEND/elasticsearchIndexing/pyang_plugin"
+ENV PYANG_PLUGINPATH="$BACKEND/opensearch_indexing/pyang_plugin"
 
 #Install Cron
 RUN apt-get -y update && apt-get -y install libv8-dev cron gunicorn logrotate curl mydumper rsync vim pcregrep

api/globalConfig.py

@@ -44,7 +44,9 @@ def load_config(self):
         self.credentials = (
             config.get('Secrets-Section', 'confd-credentials', fallback='admin admin').strip('"').split(' ')
         )
-        self.elk_credentials = config.get('Secrets-Section', 'elk-secret', fallback='').strip('"').split(' ')
+        self.opensearch_credentials = (
+            config.get('Secrets-Section', 'opensearch-secret', fallback='').strip('"').split(' ')
+        )
         self.cache_dir = config.get('Directory-Section', 'cache', fallback='tests/resources/cache')
         self.save_requests = config.get('Directory-Section', 'save-requests', fallback='/var/yang/test-requests')
         self.save_file_dir = config.get('Directory-Section', 'save-file-dir', fallback='/var/yang/all_modules')
@@ -69,9 +71,9 @@ def load_config(self):
             'yang-models-dir',
             fallback='tests/resources/yangmodels/yang',
         )
-        self.es_host = config.get('DB-Section', 'es-host', fallback='localhost')
-        self.es_port = config.get('DB-Section', 'es-port', fallback='9200')
-        self.es_aws = config.get('DB-Section', 'es-aws', fallback=False)
+        self.opensearch_host = config.get('DB-Section', 'opensearch-host', fallback='localhost')
+        self.opensearch_port = config.get('DB-Section', 'opensearch-port', fallback='9200')
+        self.opensearch_aws = config.get('DB-Section', 'opensearch-aws', fallback=False)
         self.redis_host = config.get('DB-Section', 'redis-host', fallback='localhost')
         self.redis_port = config.get('DB-Section', 'redis-port', fallback='6379')
         self.json_ytree = config.get('Directory-Section', 'json-ytree', fallback='/var/yang/ytrees')
@@ -81,7 +83,7 @@ def load_config(self):
             fallback='http://localhost/api',
         )
         self.domain_prefix = config.get('Web-Section', 'domain-prefix', fallback='http://localhost')
-        self.es_aws = self.es_aws == 'True'
+        self.opensearch_aws = self.opensearch_aws == 'True'
 
         self.LOGGER = log.get_logger('api.yc_gc', '{}/yang.log'.format(self.logs_dir))
 

api/my_flask.py

@@ -20,8 +20,8 @@
 
 import api.authentication.auth as auth
 from api.matomo_tracker import MatomoTrackerData, get_headers_dict, record_analytic
-from elasticsearchIndexing.es_manager import ESManager
 from jobs.celery import celery_app
+from opensearch_indexing.opensearch_manager import OpenSearchManager
 from redisConnections.redis_user_notifications_connection import RedisUserNotificationsConnection
 from redisConnections.redis_users_connection import RedisUsersConnection
 from redisConnections.redisConnection import RedisConnection
@@ -92,9 +92,9 @@ def setup_logger(self):
             os.chmod(file_name_path, 0o664)
 
     def post_config_load(self):
-        self.config['S-ELK-CREDENTIALS'] = self.config.s_elk_secret.strip('"').split()
+        self.config['S-OPENSEARCH-CREDENTIALS'] = self.config.s_opensearch_secret.strip('"').split()
         self.config['S-CONFD-CREDENTIALS'] = self.config.s_confd_credentials.strip('"').split()
-        self.config['ES-MANAGER'] = ESManager()
+        self.config['OPENSEARCH-MANAGER'] = OpenSearchManager()
 
         celery_app.load_config()
         self.config['CELERY-APP'] = celery_app

api/views/README.md

@@ -42,7 +42,7 @@
 
 ### [health_check.py](https://github.com/YangCatalog/backend/blob/master/api/views/health_check.py)
 * `/api/admin/healthcheck/services-list - ['GET']`
-* `/api/admin/healthcheck/elk - ['GET']`
+* `/api/admin/healthcheck/opensearch - ['GET']`
 * `/api/admin/healthcheck/confd - ['GET']`
 * `/api/admin/healthcheck/redis - ['GET']`
 * `/api/admin/healthcheck/nginx - ['GET']`

api/views/admin.py

@@ -569,8 +569,8 @@ def get_script_names():
         'pull_local',
         'statistics',
         'recovery',
-        'elk_recovery',
-        'elk_fill',
+        'opensearch_recovery',
+        'opensearch_fill',
         'redis_users_recovery',
         'resolve_expiration',
         'reviseSemver',
@@ -592,7 +592,7 @@ def get_module_name(script_name):
         return 'ietfYangDraftPull'
     elif script_name in ('ietf_push', 'iana_push'):
         return 'automatic_push'
-    elif script_name in ('recovery', 'elk_recovery', 'elk_fill', 'redis_users_recovery'):
+    elif script_name in ('recovery', 'opensearch_recovery', 'opensearch_fill', 'redis_users_recovery'):
         return 'recovery'
     elif script_name == 'statistics':
         return 'statistic'

api/views/health_check.py

@@ -57,7 +57,7 @@ def set_config():
 def get_services_list():
     response_body = []
     service_endpoints = [
-        'elk',
+        'opensearch',
         'confd-admin',
         'redis-admin',
         'yang-search-admin',
@@ -68,7 +68,7 @@ def get_services_list():
         'celery',
     ]
     service_names = [
-        'Elasticsearch',
+        'OpenSearch',
         'ConfD',
         'Redis',
         'YANG search',
@@ -84,24 +84,24 @@ def get_services_list():
     return make_response(jsonify(response_body), 200)
 
 
-@bp.route('/elk', methods=['GET'])
-def health_check_elk():
-    service_name = 'Elasticsearch'
+@bp.route('/opensearch', methods=['GET'])
+def health_check_opensearch():
+    service_name = 'OpenSearch'
     try:
-        # try to ping Elasticsearch
-        if app_config.es_manager.ping():
-            bp.logger.info('Successfully connected to Elasticsearch')
+        # try to ping OpenSearch
+        if app_config.opensearch_manager.ping():
+            bp.logger.info('Successfully connected to OpenSearch')
             # get health of cluster
-            health = app_config.es_manager.cluster_health()
+            health = app_config.opensearch_manager.cluster_health()
             health_status = health.get('status')
             bp.logger.info('Health status of cluster: {}'.format(health_status))
             # get list of indices
-            indices = app_config.es_manager.get_indices()
+            indices = app_config.opensearch_manager.get_indices()
             if len(indices) > 0:
                 return make_response(
                     jsonify(
                         {
-                            'info': 'Elasticsearch is running',
+                            'info': 'OpenSearch is running',
                             'status': 'running',
                             'message': 'Cluster status: {}'.format(health_status),
                         },
@@ -112,27 +112,27 @@ def health_check_elk():
                 return make_response(
                     jsonify(
                         {
-                            'info': 'Elasticsearch is running',
+                            'info': 'OpenSearch is running',
                             'status': 'problem',
                             'message': 'Cluster status: {} Number of indices: {}'.format(health_status, len(indices)),
                         },
                     ),
                     200,
                 )
         else:
-            bp.logger.info('Cannot connect to Elasticsearch database')
+            bp.logger.info('Cannot connect to OpenSearch database')
             return make_response(
                 jsonify(
                     {
-                        'info': 'Not OK - Elasticsearch is not running',
+                        'info': 'Not OK - OpenSearch is not running',
                         'status': 'down',
-                        'error': 'Cannot ping Elasticsearch',
+                        'error': 'Cannot ping OpenSearch',
                     },
                 ),
                 200,
             )
     except Exception as err:
-        bp.logger.error('Cannot connect to Elasticsearch database. Error: {}'.format(err))
+        bp.logger.error('Cannot connect to OpenSearch database. Error: {}'.format(err))
         return make_response(jsonify(error_response(service_name, err)), 200)
 
 

api/views/redis_search.py

@@ -506,7 +506,7 @@ def filter_using_api(res_row, payload):
                         reject = True
                         break
                 else:
-                    # Module key has different value then serached for then reject
+                    # Module key has different value then searched for then reject
                     values = value.split(',')
                     reject = True
                     for val in values:

api/views/yang_search/grep_search.py

@@ -7,8 +7,8 @@
 
 from api.cache.api_cache import cache
 from api.views.yang_search.constants import GREP_SEARCH_CACHE_TIMEOUT
-from elasticsearchIndexing.es_manager import ESManager
-from elasticsearchIndexing.models.es_indices import ESIndices
+from opensearch_indexing.models.opensearch_indices import OpenSearchIndices
+from opensearch_indexing.opensearch_manager import OpenSearchManager
 from utility import log
 from utility.create_config import create_config
 from utility.staticVariables import ORGANIZATIONS
@@ -23,8 +23,8 @@ class GrepSearch:
     def __init__(
         self,
         config: ConfigParser = create_config(),
-        es_manager: ESManager = ESManager(),
-        modules_es_index: ESIndices = ESIndices.AUTOCOMPLETE,
+        opensearch_manager: OpenSearchManager = OpenSearchManager(),
+        modules_es_index: OpenSearchIndices = OpenSearchIndices.AUTOCOMPLETE,
         starting_cursor: int = 0,
     ):
         self.previous_cursor = 0
@@ -40,10 +40,10 @@ def __init__(
             self.query = json.load(query_file)
 
         log_file_path = os.path.join(config.get('Directory-Section', 'logs'), 'yang.log')
-        self.logger = log.get_logger('yc-elasticsearch', log_file_path)
+        self.logger = log.get_logger('yc-opensearch', log_file_path)
 
         self.modules_es_index = modules_es_index
-        self._es_manager = es_manager
+        self._opensearch_manager = opensearch_manager
         self._searched_modules_amount = 0
 
     def search(
@@ -250,7 +250,7 @@ def _construct_db_query(
 
     def _get_results_from_db(self, already_found_results: t.Optional[list[dict]] = None) -> list[dict]:
         response = already_found_results or []
-        es_response = self._es_manager.generic_search(
+        es_response = self._opensearch_manager.generic_search(
             index=self.modules_es_index,
             query=self.query,
             response_size=None,

api/views/yang_search/elk_search.py → api/views/yang_search/opensearch_query.py

@@ -20,12 +20,12 @@
 import json
 import os
 
-from elasticsearch import ConnectionTimeout
+from opensearchpy import ConnectionTimeout
 
 import api.views.yang_search.search_params as sp
 from api.views.yang_search.response_row import ResponseRow
-from elasticsearchIndexing.es_manager import ESManager
-from elasticsearchIndexing.models.es_indices import ESIndices
+from opensearch_indexing.models.opensearch_indices import OpenSearchIndices
+from opensearch_indexing.opensearch_manager import OpenSearchManager
 from redisConnections.redisConnection import RedisConnection
 from utility import log
 from utility.staticVariables import OUTPUT_COLUMNS
@@ -34,44 +34,45 @@
 RESERVED_CHARACTERS = ['"', '<']
 
 
-class ElkSearch:
+class OpenSearchQuery:
     """
-    Serves distinctly for yangcatalog search. This class will create a query that is sent to elasticsearch
+    Serves distinctly for yangcatalog search. This class will create a query that is sent to OpenSearch
     which returns output that needs to be processed. We process this into a list which is displayed as rows
     in a grid of yangcatalog search.
     """
 
     def __init__(
         self,
         logs_dir: str,
-        es_manager: ESManager,
+        opensearch_manager: OpenSearchManager,
         redis_connection: RedisConnection,
         search_params: sp.SearchParams,
     ) -> None:
         """
-        Initialization of search under Elasticsearch engine. We need to prepare a query
-        that will be used to search in Elasticsearch.
+        Initialization of search under OpenSearch engine. We need to prepare a query
+        that will be used to search in OpenSearch.
 
         Arguments:
             :param searched_term    (str) String that we are searching for
             :param logs_dir         (str) Directory to log files
-            :param es_manager       (ESManager) Elasticsearch manager
+            :param opensearch_manager       (OpenSearchManager) OpenSearch manager
             :param redis_connection (RedisConnection) Redis connection to modules db (db=1)
             :param search_params    (SearchParams) Contains search parameters
         """
         self._search_params = search_params
         search_query_path = os.path.join(os.environ['BACKEND'], 'api/views/yang_search/json/search.json')
         with open(search_query_path, encoding='utf-8') as reader:
             self.query: dict = json.load(reader)
-        self._es_manager = es_manager
+        self._opensearch_manager = opensearch_manager
         self._redis_connection = redis_connection
         self._latest_revisions = {}
         self._remove_columns = list(set(OUTPUT_COLUMNS) - set(self._search_params.output_columns))
         self._row_hashes = set()
         self._missing_modules = []
         self.timeout = False
         log_file_path = os.path.join(logs_dir, 'yang.log')
-        self.logger = log.get_logger('yc-elasticsearch', log_file_path)
+        self.logger = log.get_logger('yc-opensearch', log_file_path)
+        self._construct_query()
 
     def alerts(self):
         """
@@ -83,9 +84,9 @@ def alerts(self):
             alerts.append(f'Module {missing} metadata does not exist in yangcatalog')
         return alerts
 
-    def construct_query(self):
+    def _construct_query(self):
         """
-        Create a json query that is then sent to Elasticsearch.
+        Create a json query that is then sent to OpenSearch.
         Changes being made while creating query:
         - statement is a list of schema types. It is one or more or all of ['typedef', 'grouping', 'feature',
         'identity', 'extension', 'rpc', 'container', 'list', 'leaf-list', 'leaf', 'notification', 'action']
@@ -182,13 +183,13 @@ def _retrieve_results(self, latest_revisions: bool) -> list[dict]:
             query.pop('aggs')
         try:
             self.logger.debug(json.dumps(query, indent=2))
-            response = self._es_manager.generic_search(
-                ESIndices.YINDEX,
+            response = self._opensearch_manager.generic_search(
+                OpenSearchIndices.YINDEX,
                 query,
                 response_size=RESPONSE_SIZE,
             )
         except ConnectionTimeout:
-            self.logger.exception('Error while searching in Elasticsearch')
+            self.logger.exception('Error while searching in OpenSearch')
             self.timeout = True
             return []
         hits = response['hits']['hits']
@@ -216,7 +217,7 @@ def _process_hits(self, hits: list) -> list[dict]:
             module_data = self._redis_connection.get_module(module_key)
             module_data = json.loads(module_data)
             if not module_data:
-                self.logger.error(f'Failed to get module from Redis, but found in Elasticsearch: {module_key}')
+                self.logger.error(f'Failed to get module from Redis, but found in OpenSearch: {module_key}')
                 reject.add(module_key)
                 self._missing_modules.append(module_key)
                 continue
@@ -240,7 +241,7 @@ def _process_hits(self, hits: list) -> list[dict]:
                 self._row_hashes.add(row_hash)
             response_rows.append(row.output_row)
 
-        self.logger.debug(f'ElkSearch finished with length {len(response_rows)}')
+        self.logger.debug(f'OpenSearch finished with length {len(response_rows)}')
         return response_rows
 
     def _rejects_mibs_or_versions(self, module_data: dict) -> bool: