chore: elasticsearch -> opensearch

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         python-version: ['3.10']
         redis-version: [6]
-        elasticsearch-version: [7.10.1]
+        opensearch-version: [2.7.0]
       max-parallel: 1
 
     steps:
@@ -65,13 +65,10 @@ jobs:
           sudo sysctl -w vm.swappiness=1
           sudo sysctl -w vm.max_map_count=262144
 
-      - name: Setup Elasticsearch ${{ matrix.elasticsearch-version }}
-        uses: getong/elasticsearch-action@v1.2
+      - name: Setup OpenSearch ${{ matrix.opensearch-version }}
+        uses: ankane/setup-opensearch@v1
         with:
-          elasticsearch version: ${{ matrix.elasticsearch-version }}
-          host port: 9200
-          container port: 9200
-          discovery type: 'single-node'
+          opensearch-version: ${{ matrix.opensearch-version }}
 
       - name: Prepare environment
         run: |
@@ -97,7 +94,7 @@ jobs:
       - name: Test with pytest
         env:
           TOKEN: ${{ secrets.TOKEN }}
-          PYANG_PLUGINPATH: $PWD/elasticsearchIndexing/pyang_plugin
+          PYANG_PLUGINPATH: $PWD/opensearch_indexing/pyang_plugin
         run: |
           export TOKEN
           export TEST_REPO=/var/yang/tmp/test/YangModels/yang

.gitignore

@@ -10,7 +10,7 @@ api/standard
 **/*.html
 !statistics/template/stats.html
 !api/template/*.html
-!elasticsearchIndexing/json/**/*
+!opensearch_indexing/json/**/*
 !api/views/yang_search/json/*
 /venv
 yang.egg-info

.vscode/launch.json

@@ -125,7 +125,7 @@
       "name": "process_changed_mods",
       "type": "python",
       "request": "launch",
-      "program": "${workspaceFolder}/elasticsearchIndexing/process_changed_mods.py",
+      "program": "${workspaceFolder}/opensearch_indexing/process_changed_mods.py",
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}",
       "justMyCode": false

Dockerfile

@@ -12,7 +12,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONUNBUFFERED=1
 
 ENV VIRTUAL_ENV=/backend
 ENV BACKEND=/backend
-ENV PYANG_PLUGINPATH="$BACKEND/elasticsearchIndexing/pyang_plugin"
+ENV PYANG_PLUGINPATH="$BACKEND/opensearch_indexing/pyang_plugin"
 
 #Install Cron
 RUN apt-get -y update && apt-get -y install libv8-dev cron gunicorn logrotate curl mydumper rsync vim pcregrep

api/globalConfig.py

@@ -44,7 +44,9 @@ def load_config(self):
         self.credentials = (
             config.get('Secrets-Section', 'confd-credentials', fallback='admin admin').strip('"').split(' ')
         )
-        self.elk_credentials = config.get('Secrets-Section', 'elk-secret', fallback='').strip('"').split(' ')
+        self.opensearch_credentials = (
+            config.get('Secrets-Section', 'opensearch-secret', fallback='').strip('"').split(' ')
+        )
         self.cache_dir = config.get('Directory-Section', 'cache', fallback='tests/resources/cache')
         self.save_requests = config.get('Directory-Section', 'save-requests', fallback='/var/yang/test-requests')
         self.save_file_dir = config.get('Directory-Section', 'save-file-dir', fallback='/var/yang/all_modules')
@@ -69,9 +71,9 @@ def load_config(self):
             'yang-models-dir',
             fallback='tests/resources/yangmodels/yang',
         )
-        self.es_host = config.get('DB-Section', 'es-host', fallback='localhost')
-        self.es_port = config.get('DB-Section', 'es-port', fallback='9200')
-        self.es_aws = config.get('DB-Section', 'es-aws', fallback=False)
+        self.opensearch_host = config.get('DB-Section', 'opensearch-host', fallback='localhost')
+        self.opensearch_port = config.get('DB-Section', 'opensearch-port', fallback='9200')
+        self.opensearch_aws = config.get('DB-Section', 'opensearch-aws', fallback=False)
         self.redis_host = config.get('DB-Section', 'redis-host', fallback='localhost')
         self.redis_port = config.get('DB-Section', 'redis-port', fallback='6379')
         self.json_ytree = config.get('Directory-Section', 'json-ytree', fallback='/var/yang/ytrees')
@@ -81,7 +83,7 @@ def load_config(self):
             fallback='http://localhost/api',
         )
         self.domain_prefix = config.get('Web-Section', 'domain-prefix', fallback='http://localhost')
-        self.es_aws = self.es_aws == 'True'
+        self.opensearch_aws = self.opensearch_aws == 'True'
 
         self.LOGGER = log.get_logger('api.yc_gc', '{}/yang.log'.format(self.logs_dir))
 

api/my_flask.py

@@ -20,8 +20,8 @@
 
 import api.authentication.auth as auth
 from api.matomo_tracker import MatomoTrackerData, get_headers_dict, record_analytic
-from elasticsearchIndexing.es_manager import ESManager
 from jobs.celery import celery_app
+from opensearch_indexing.opensearch_manager import OpenSearchManager
 from redisConnections.redis_user_notifications_connection import RedisUserNotificationsConnection
 from redisConnections.redis_users_connection import RedisUsersConnection
 from redisConnections.redisConnection import RedisConnection
@@ -92,9 +92,9 @@ def setup_logger(self):
             os.chmod(file_name_path, 0o664)
 
     def post_config_load(self):
-        self.config['S-ELK-CREDENTIALS'] = self.config.s_elk_secret.strip('"').split()
+        self.config['S-OPENSEARCH-CREDENTIALS'] = self.config.s_opensearch_secret.strip('"').split()
         self.config['S-CONFD-CREDENTIALS'] = self.config.s_confd_credentials.strip('"').split()
-        self.config['ES-MANAGER'] = ESManager()
+        self.config['OPENSEARCH-MANAGER'] = OpenSearchManager()
 
         celery_app.load_config()
         self.config['CELERY-APP'] = celery_app

api/views/README.md

@@ -42,7 +42,7 @@
 
 ### [health_check.py](https://github.com/YangCatalog/backend/blob/master/api/views/health_check.py)
 * `/api/admin/healthcheck/services-list - ['GET']`
-* `/api/admin/healthcheck/elk - ['GET']`
+* `/api/admin/healthcheck/opensearch - ['GET']`
 * `/api/admin/healthcheck/confd - ['GET']`
 * `/api/admin/healthcheck/redis - ['GET']`
 * `/api/admin/healthcheck/nginx - ['GET']`

api/views/admin.py

@@ -569,8 +569,8 @@ def get_script_names():
         'pull_local',
         'statistics',
         'recovery',
-        'elk_recovery',
-        'elk_fill',
+        'opensearch_recovery',
+        'opensearch_fill',
         'redis_users_recovery',
         'resolve_expiration',
         'reviseSemver',
@@ -592,7 +592,7 @@ def get_module_name(script_name):
         return 'ietfYangDraftPull'
     elif script_name in ('ietf_push', 'iana_push'):
         return 'automatic_push'
-    elif script_name in ('recovery', 'elk_recovery', 'elk_fill', 'redis_users_recovery'):
+    elif script_name in ('recovery', 'opensearch_recovery', 'opensearch_fill', 'redis_users_recovery'):
         return 'recovery'
     elif script_name == 'statistics':
         return 'statistic'

api/views/health_check.py

@@ -57,7 +57,7 @@ def set_config():
 def get_services_list():
     response_body = []
     service_endpoints = [
-        'elk',
+        'opensearch',
         'confd-admin',
         'redis-admin',
         'yang-search-admin',
@@ -68,7 +68,7 @@ def get_services_list():
         'celery',
     ]
     service_names = [
-        'Elasticsearch',
+        'OpenSearch',
         'ConfD',
         'Redis',
         'YANG search',
@@ -84,24 +84,24 @@ def get_services_list():
     return make_response(jsonify(response_body), 200)
 
 
-@bp.route('/elk', methods=['GET'])
-def health_check_elk():
-    service_name = 'Elasticsearch'
+@bp.route('/opensearch', methods=['GET'])
+def health_check_opensearch():
+    service_name = 'OpenSearch'
     try:
-        # try to ping Elasticsearch
-        if app_config.es_manager.ping():
-            bp.logger.info('Successfully connected to Elasticsearch')
+        # try to ping OpenSearch
+        if app_config.opensearch_manager.ping():
+            bp.logger.info('Successfully connected to OpenSearch')
             # get health of cluster
-            health = app_config.es_manager.cluster_health()
+            health = app_config.opensearch_manager.cluster_health()
             health_status = health.get('status')
             bp.logger.info('Health status of cluster: {}'.format(health_status))
             # get list of indices
-            indices = app_config.es_manager.get_indices()
+            indices = app_config.opensearch_manager.get_indices()
             if len(indices) > 0:
                 return make_response(
                     jsonify(
                         {
-                            'info': 'Elasticsearch is running',
+                            'info': 'OpenSearch is running',
                             'status': 'running',
                             'message': 'Cluster status: {}'.format(health_status),
                         },
@@ -112,27 +112,27 @@ def health_check_elk():
                 return make_response(
                     jsonify(
                         {
-                            'info': 'Elasticsearch is running',
+                            'info': 'OpenSearch is running',
                             'status': 'problem',
                             'message': 'Cluster status: {} Number of indices: {}'.format(health_status, len(indices)),
                         },
                     ),
                     200,
                 )
         else:
-            bp.logger.info('Cannot connect to Elasticsearch database')
+            bp.logger.info('Cannot connect to OpenSearch database')
             return make_response(
                 jsonify(
                     {
-                        'info': 'Not OK - Elasticsearch is not running',
+                        'info': 'Not OK - OpenSearch is not running',
                         'status': 'down',
-                        'error': 'Cannot ping Elasticsearch',
+                        'error': 'Cannot ping OpenSearch',
                     },
                 ),
                 200,
             )
     except Exception as err:
-        bp.logger.error('Cannot connect to Elasticsearch database. Error: {}'.format(err))
+        bp.logger.error('Cannot connect to OpenSearch database. Error: {}'.format(err))
         return make_response(jsonify(error_response(service_name, err)), 200)
 
 

api/views/redis_search.py

@@ -506,7 +506,7 @@ def filter_using_api(res_row, payload):
                         reject = True
                         break
                 else:
-                    # Module key has different value then serached for then reject
+                    # Module key has different value then searched for then reject
                     values = value.split(',')
                     reject = True
                     for val in values:

api/views/yang_search/grep_search.py

@@ -7,8 +7,8 @@
 
 from api.cache.api_cache import cache
 from api.views.yang_search.constants import GREP_SEARCH_CACHE_TIMEOUT
-from elasticsearchIndexing.es_manager import ESManager
-from elasticsearchIndexing.models.es_indices import ESIndices
+from opensearch_indexing.models.opensearch_indices import OpenSearchIndices
+from opensearch_indexing.opensearch_manager import OpenSearchManager
 from utility import log
 from utility.create_config import create_config
 from utility.staticVariables import ORGANIZATIONS
@@ -23,8 +23,8 @@ class GrepSearch:
     def __init__(
         self,
         config: ConfigParser = create_config(),
-        es_manager: ESManager = ESManager(),
-        modules_es_index: ESIndices = ESIndices.AUTOCOMPLETE,
+        opensearch_manager: OpenSearchManager = OpenSearchManager(),
+        modules_es_index: OpenSearchIndices = OpenSearchIndices.AUTOCOMPLETE,
         starting_cursor: int = 0,
     ):
         self.previous_cursor = 0
@@ -40,10 +40,10 @@ def __init__(
             self.query = json.load(query_file)
 
         log_file_path = os.path.join(config.get('Directory-Section', 'logs'), 'yang.log')
-        self.logger = log.get_logger('yc-elasticsearch', log_file_path)
+        self.logger = log.get_logger('yc-opensearch', log_file_path)
 
         self.modules_es_index = modules_es_index
-        self._es_manager = es_manager
+        self._opensearch_manager = opensearch_manager
         self._searched_modules_amount = 0
 
     def search(
@@ -250,7 +250,7 @@ def _construct_db_query(
 
     def _get_results_from_db(self, already_found_results: t.Optional[list[dict]] = None) -> list[dict]:
         response = already_found_results or []
-        es_response = self._es_manager.generic_search(
+        es_response = self._opensearch_manager.generic_search(
             index=self.modules_es_index,
             query=self.query,
             response_size=None,