chore: replace elasticsearch with opensearch

.env-dist

@@ -2,14 +2,14 @@ COMPOSE_PROJECT_NAME=yc
 REDIS_VOLUME=/var/yang/redis
 RABBITMQ_USER=<RABBITMQ_USER>
 RABBITMQ_PASSWORD=<RABBITMQ PASSWORD>
-ELASTICSEARCH_DATA=/var/yang/elasticsearch
-ELASTICSEARCH_LOG=/var/yang/logs/elasticsearch
+OPENSEARCH_DATA=/var/yang/opensearch
+OPENSEARCH_LOG=/var/yang/logs/opensearch
 NGINX_LOG=/var/yang/logs/nginx
 KEY_FILE=/home/yang/deployment/resources/yangcatalog.org.key
 CERT_FILE=/home/yang/deployment/resources/yangcatalog.org.crt
 CA_CERT_FILE=/home/yang/deployment/resources/yangcatalog.org.crt
-ELASTICSEARCH_ID=1016
-ELASTICSEARCH_GID=1001
+OPENSEARCH_ID=1016
+OPENSEARCH_GID=1001
 YANG_ID=1016
 YANG_GID=1001
 YANG_RESOURCES=/var/yang

README.md

@@ -28,7 +28,7 @@ Main repository to start up all the pieces of YANG Catalog
 The deployment uses several services via container images that are
 distributed by third parties, e.g. via DockerHub
 
-* Elasticsearch
+* OpenSearch
 * RabbitMQ
 * NGINX (as the base image of the frontend container, which includes static content)
 
@@ -52,21 +52,21 @@ Redis) and combine them into a functional local deployment
 of the YANG Catalog, which should be accessible on
 http://localhost
 
-Make sure that you will start the elasticsearch Dockerfile as well
-since in production environment this is not used and an AWS elasticsearch
+Make sure that you will start the OpenSearch Dockerfile as well
+since in production environment this is not used and an AWS OpenSearch
 instance is used instead
 
 ## Status
 
-### Elasticsearch
+### OpenSearch
 
-Elasticsearch instance can be started in two different ways. Locally
-you can set "es-aws" in your yangcatalog.conf file to `False` and start
-the elasticsearch manually using `docker run` command. Also, es-host
-has to be set to elasticsearch instance IP (k8s setup) or
-`yc-elasticsearch` (docker-compose setup) and port to `9200`. We use AWS elasticsearch service in production.
-This instance runs on different server and is connected to YANG Catalog server. For this the "es-aws" needs to be
-set to `True` and es-host set to whatever URL AWS provides for elasticsearch.
+OpenSearch instance can be started in two different ways. Locally
+you can set "os-aws" in your yangcatalog.conf file to `False` and start
+ OpenSearch manually using `docker run` command. Also, os-host
+has to be set to the OpenSearch's instance IP (k8s setup) or
+`yc-opensearch` (docker-compose setup) and port to `9200`. We use AWS's OpenSearch service in production.
+This instance runs on different server and is connected to the YANG Catalog server. For this the "os-aws" option needs to be
+set to `True` and os-host set to whatever URL AWS provides for OpenSearch.
 In AWS make sure that this URL is not opened for internet but is opened for
 the other instance that is running yangcatalog.org
 
@@ -137,13 +137,11 @@ This will add a prefix to these names so in this example we would have yc-backen
 
 `RABBITMQ_PASSWORD=<RABBITMQ PASSWORD>`  - rabbitmq username.
 
-`ELASTICSEARCH_DATA=/var/lib/elasticsearch` - This is only used with local elasticsearch. If AWS
-elasticsearch is used this variables can be set to anything. If local elasticsearch is used this
-variable is used to specify where elasticsearch indexed data should be saved
+`OPENSEARCH_DATA=/var/lib/opensearch` - If local OpenSearch is used this
+variable is used to specify where OpenSearch indexed data should be saved
 
-`ELASTICSEARCH_LOG=/var/log/elasticsearch`  - This is only used with local elasticsearch. If AWS
-elasticsearch is used this variables can be set to anything. If local elasticsearch is used this
-variable is used to specify where elasticsearch logs should be saved
+`OPENSEARCH_LOG=/var/log/opensearch`  - If local OpenSearch is used this
+variable is used to specify where OpenSearch logs should be saved
 
 `NGINX_LOG=/var/log/nginx` - Where do we save nginx logs
 
@@ -156,12 +154,11 @@ If None exists just use any path
 `CA_CERT_FILE=/home/yang/deployment/resources/yangcatalog.org.crt` - ca_certificate file for HTTPS protocol.
 If None exists just use any path
 
-`ELASTICSEARCH_ID=1001` - This is only used with local elasticsearch. If AWS
-elasticsearch is used this variables can be set to anything. If local elasticsearch is used this
+`OPENSEARCH_ID=1001` - If local OpenSearch is used this
 variable is used to set permissions to specific user ID. It`s safe to use same ID as YANG_ID
 
-`ELASTICSEARCH_GID=1001` - This is only used with local elasticsearch. If AWS
-elasticsearch is used to set permissions to specific user GID. It`s safe to use same GID as YANG_GID
+`OPENSEARCH_GID=1001` - If local OpenSearch is used this
+variable is used to set permissions to specific group ID. It`s safe to use same ID as YANG_GID
 
 `YANG_ID=1016` - ID created for yang user
 

docker-compose.yml

@@ -52,7 +52,7 @@ services:
     networks:
       - confd
       - backend
-      - elasticsearch
+      - opensearch
       - rabbitmq
       - frontend
       - redis
@@ -107,7 +107,7 @@ services:
     networks:
       - confd
       - backend
-      - elasticsearch
+      - opensearch
       - rabbitmq
       - frontend
       - redis
@@ -221,36 +221,36 @@ services:
       retries: 5
     restart: always
 
-  elasticsearch:
-    container_name: yc-elasticsearch
+  opensearch:
+    container_name: yc-opensearch
     build:
-      context: elasticsearch
+      context: opensearch
       dockerfile: Dockerfile
       args:
-        - ELASTICSEARCH_ID=${ELASTICSEARCH_ID}
-        - ELASTICSEARCH_GID=${ELASTICSEARCH_GID}
+        - OPENSEARCH_ID=${OPENSEARCH_ID}
+        - OPENSEARCH_GID=${OPENSEARCH_GID}
     environment:
       - bootstrap.memory_lock=true
     ulimits:
       memlock:
         soft: -1
         hard: -1
     volumes:
-      - ${ELASTICSEARCH_LOG}:/var/log/elasticsearch
-      - ${ELASTICSEARCH_DATA}:/var/lib/elasticsearch
+      - ${OPENSEARCH_LOG}:/var/log/opensearch
+      - ${OPENSEARCH_DATA}:/var/lib/opensearch
     networks:
-      - elasticsearch
+      - opensearch
     restart: always
-
-  #  kibana:
-  #     container_name: yc-kibana
-  #     depends_on:
-  #       - elasticsearch
-  #     image: kibana:7.10.1
-  #     ports:
-  #       - '5601:5601'
-  #     networks:
-  #       - elasticsearch
+  
+#  kibana:
+#     container_name: yc-kibana
+#     depends_on:
+#       - opensearch
+#     image: kibana:7.10.1
+#     ports:
+#       - '5601:5601'
+#     networks:
+#       - opensearch
 
   # matomo:
   #   container_name: yc-matomo
@@ -380,7 +380,7 @@ networks:
     driver: bridge
   backend:
     driver: bridge
-  elasticsearch:
+  opensearch:
     driver: bridge
   rabbitmq:
     driver: bridge

k8s/README.md

@@ -203,7 +203,7 @@ inet6 line
 `vim values.yaml`
 
 * Please create all volume directories (admin-webroot, docs, downloadables, main-webroot, nginx-conf, run) under YANG_VOLUMES manually.
-* Please check if `/var/yang/conf/yangcatalog.conf` file points to correct elasticsearch IP
+* Please check if `/var/yang/conf/yangcatalog.conf` file points to correct OpenSearch IP
 
 ## 2. Run Helm Chart
 
@@ -215,11 +215,11 @@ inet6 line
 
 `microk8s kubectl get jobs`
 
-## Setup Elasticsearch container (on localhost only)
+## Setup OpenSearch container (on localhost only)
 
 `sudo sysctl -w vm.max_map_count=262144`
 
-`docker exec -it <elasticsearch_container_name> sh`
+`docker exec -it <opensearch_container_name> sh`
 
 `curl -X PUT 'http://localhost:9200/_settings' -H 'Content-Type: application/json' -d '{ "index": { "blocks": { "read_only_allow_delete": "false" } } }'`
 

k8s/templates/backend/backend-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         io.kompose.network/frontend: "true"
         io.kompose.network/rabbitmq: "true"
         io.kompose.network/redis: "true"
-        io.kompose.network/elasticsearch: "true"
+        io.kompose.network/opensearch: "true"
         io.kompose.network/crontab: "true"
         io.kompose.network/tools-ieft-org: "true"
         io.kompose.network/rsync: "true"

k8s/templates/celery-worker/celery-worker-deployment.yaml

@@ -23,7 +23,7 @@ spec:
         io.kompose.network/backend: "true"
         io.kompose.network/confd: "true"
         io.kompose.network/rabbitmq: "true"
-        io.kompose.network/elasticsearch: "true"
+        io.kompose.network/opensearch: "true"
         io.kompose.network/redis: "true"
         io.kompose.network/tools-ieft-org: "true"
         io.kompose.network/rsync: "true"

k8s/templates/cluster-ingress-egress/elasticsearch-networkpolicy.yaml

@@ -1,11 +1,11 @@
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
-  name: elasticsearch
+  name: opensearch
 spec:
   podSelector:
     matchLabels:
-      io.kompose.network/elasticsearch: "true"
+      io.kompose.network/opensearch: "true"
   policyTypes:
     - Ingress
     - Egress

k8s/templates/frontend/frontend-deployment.yaml

@@ -21,7 +21,7 @@ spec:
         kompose.version: 1.21.0 (992df58d8)
       labels:
         io.kompose.network/frontend: "true"
-        io.kompose.network/elasticsearch: "true"
+        io.kompose.network/opensearch: "true"
         io.kompose.service: yc-frontend
     spec:
       securityContext:

opensearch/Dockerfile

@@ -0,0 +1,20 @@
+FROM opensearchproject/opensearch:2.7.0
+ARG OPENSEARCH_ID
+ARG OPENSEARCH_GID
+ENV OPENSEARCH_ID "$OPENSEARCH_ID"
+ENV OPENSEARCH_GID "$OPENSEARCH_GID"
+
+ENV VIRTUAL_ENV=/opensearch
+
+COPY opensearch.yml /usr/share/opensearch/config/
+COPY thesaurus.conf /usr/share/opensearch/config/
+COPY jvm.options /usr/share/opensearch/config/
+
+USER root:root
+RUN usermod -u ${OPENSEARCH_ID} opensearch
+RUN groupmod -g ${OPENSEARCH_GID} opensearch
+RUN chown -R opensearch:root /usr/share/opensearch/config/
+# Support arbitrary UIDs as per OpenShift guidelines
+USER ${OPENSEARCH_ID}:${OPENSEARCH_GID}
+
+EXPOSE 9200

elasticsearch/jvm.options → opensearch/jvm.options

@@ -19,8 +19,8 @@
 # Xms represents the initial size of total heap space
 # Xmx represents the maximum size of total heap space
 
--Xms2g
--Xmx2g
+-Xms1g
+-Xmx1g
 
 ################################################################
 ## Expert settings
@@ -77,7 +77,7 @@
 -Dlog4j.shutdownHookEnabled=false
 -Dlog4j2.disable.jmx=true
 
--Djava.io.tmpdir=${ES_TMPDIR}
+# -Djava.io.tmpdir=${OPENSERACH_TMPDIR}
 
 ## heap dumps
 
@@ -87,24 +87,24 @@
 
 # specify an alternative path for heap dumps; ensure the directory exists and
 # has sufficient space
--XX:HeapDumpPath=/var/lib/elasticsearch
+-XX:HeapDumpPath=/var/lib/opensearch
 
 # specify an alternative path for JVM fatal error logs
--XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+-XX:ErrorFile=/var/log/opensearch/hs_err_pid%p.log
 
 ## JDK 8 GC logging
 
 8:-XX:+PrintGCDetails
 8:-XX:+PrintGCDateStamps
 8:-XX:+PrintTenuringDistribution
 8:-XX:+PrintGCApplicationStoppedTime
-8:-Xloggc:/var/log/elasticsearch/gc.log
+8:-Xloggc:/var/log/opensearch/gc.log
 8:-XX:+UseGCLogFileRotation
 8:-XX:NumberOfGCLogFiles=32
 8:-XX:GCLogFileSize=64m
 
 # JDK 9+ GC logging
-9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/opensearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
 # due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
 # time/date parsing will break in an incompatible way for some date patterns and locals
 9-:-Djava.locale.providers=COMPAT