Use MongoDB for backend (#204)

Add possibility to use a MongoDB. Through the `--mongo` option for `aiida-optimade init` the AiiDA-OPTIMADE server will be initialized with a MongoDB. All relevant fields will be added to each document, representing the relevant StructureData or CifData Node. The tests have been updated to reflect the new possibility, along with relevant code in the code base, where needed. The tests can be run with MongoDB by setting the env var PYTEST_OPTIMADE_CONFIG_FILE, the value being a path to the config file to be used. A new docker-compose-mongo.yml file has been created to instantiate both the AiiDA-OPTIMADE server service as well as a MongoDB 4.2 service. CI jobs have been added to test both pytest and different docker setups for the MongoDB backend.
aiidateam · Feb 16, 2021 · 729b0f0 · 729b0f0
1 parent c76b7c0
commit 729b0f0
Show file tree

Hide file tree

Showing 29 changed files with 1,054 additions and 272 deletions.
diff --git a/.docker/run.sh b/.docker/run.sh
@@ -10,7 +10,7 @@ cp -n /profiles/${AIIDA_PROFILE}.json ${AIIDA_PATH}/.aiida/config.json
 # echo -e "`/sbin/ip route|awk '/default/ { print $3 }'`\tdocker.host.internal" | tee -a /etc/hosts > /dev/null
 
 # Initialize database
-aiida-optimade --profile ${AIIDA_PROFILE} init ${FORCE_INIT}
+aiida-optimade --profile ${AIIDA_PROFILE} init ${FORCE_INIT} ${USE_MONGO}
 
 # Run (uvicorn) server
 aiida-optimade --profile ${AIIDA_PROFILE} run --host 0.0.0.0 --port 80
diff --git a/.github/mongo/ci_config.json b/.github/mongo/ci_config.json
@@ -0,0 +1,32 @@
+{
+    "page_limit": 15,
+    "page_limit_max": 500,
+    "base_url": null,
+    "mongo_uri": "mongodb://docker.host.internal:27017",
+    "use_real_mongo": true,
+    "mongo_database": "aiida_optimade",
+    "provider": {
+      "prefix": "aiida",
+      "name": "AiiDA",
+      "description": "AiiDA: Automated Interactive Infrastructure and Database for Computational Science (http://www.aiida.net)",
+      "homepage": "http://www.aiida.net",
+      "index_base_url": null
+    },
+    "implementation": {
+      "name": "aiida-optimade",
+      "version": "0.14.1",
+      "source_url": "https://github.com/aiidateam/aiida-optimade",
+      "maintainer": {"email": "casper.andersen@epfl.ch"}
+    },
+    "provider_fields": {
+      "structures": [
+        "ctime"
+      ]
+    },
+    "aliases": {
+        "structures": {
+            "type": "node_type"
+        }
+    },
+    "log_level": "debug"
+  }
diff --git a/.github/mongo/load_data.py b/.github/mongo/load_data.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import sys
+
+import bson.json_util
+from pymongo import MongoClient
+
+client = MongoClient("mongodb://localhost:27017")
+collection = client["aiida_optimade"]["structures"]
+
+with open(Path(__file__).parent.joinpath("test_structures_mongo.json")) as handle:
+    data = bson.json_util.loads(handle.read())
+
+try:
+    print(f"Inserting {len(data)} structures into {collection.full_name}")
+    collection.insert_many(data, ordered=False)
+except Exception as exc:  # pylint: disable=broad-except
+    print("An error occurred!")
+    sys.exit(exc)
+else:
+    print("Done!")
diff --git a/.github/mongo/test_structures_mongo.json b/.github/mongo/test_structures_mongo.json
@@ -0,0 +1 @@
+../../tests/static/test_structures_mongo.json
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -48,6 +48,10 @@ jobs:
         backend: ['django', 'sqlalchemy']
 
     services:
+      mongo:
+        image: mongo:4.2
+        ports:
+          - 27017:27017
       postgres:
         image: postgres:10
         env:
@@ -60,10 +64,6 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
-      rabbitmq:
-        image: rabbitmq:latest
-        ports:
-          - 5672:5672
 
     steps:
     - uses: actions/checkout@v2
@@ -87,26 +87,53 @@ jobs:
         AIIDA_TEST_BACKEND: ${{ matrix.backend }}
       run: .github/aiida/setup_aiida.sh
 
-    - name: Test with pytest
+    - name: Test with pytest (AiiDA)
       env:
         AIIDA_TEST_BACKEND: ${{ matrix.backend }}
         AIIDA_TEST_PROFILE: test_${{ matrix.backend }}
         AIIDA_PROFILE: test_${{ matrix.backend }}
       run: pytest -v --cov=./aiida_optimade/ --cov-report=xml
 
+    - name: Test with pytest (MongoDB)
+      env:
+        AIIDA_TEST_BACKEND: ${{ matrix.backend }}
+        AIIDA_TEST_PROFILE: test_${{ matrix.backend }}
+        AIIDA_PROFILE: test_${{ matrix.backend }}
+        PYTEST_OPTIMADE_CONFIG_FILE: ./tests/static/test_mongo_config.json
+        OPTIMADE_MONGO_URI: mongodb://localhost:27017
+        OPTIMADE_CI_FORCE_MONGO: 1
+      run: pytest -v --cov=./aiida_optimade/ --cov-report=xml:mongo_cov.xml
+
     - name: Upload coverage to Codecov
       if: matrix.python-version == 3.8
       uses: codecov/codecov-action@v1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        flags: pytest
+        flags: aiida
         file: ./coverage.xml
 
-  docker-image:
+    - name: Upload coverage to Codecov
+      if: matrix.python-version == 3.8
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: mongo
+        file: ./mongo_cov.xml
+
+  docker:
     runs-on: ubuntu-latest
     timeout-minutes: 14
 
+    strategy:
+      fail-fast: false
+      matrix:
+        database: ['aiida', 'mongo']
+
     services:
+      mongo:
+        image: mongo:4.2
+        ports:
+        - 27017:27017
       postgres:
         image: postgres:10
         env:
@@ -119,10 +146,6 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
-      rabbitmq:
-        image: rabbitmq:latest
-        ports:
-          - 5672:5672
 
     steps:
     - uses: actions/checkout@v2
@@ -146,15 +169,28 @@ jobs:
         pip install ${aiida_core}
         reentry scan
 
+        pip install pymongo
+
     - name: Setup up environment for AiiDA
       env:
         AIIDA_TEST_BACKEND: django
       run: .github/aiida/setup_aiida.sh
 
     - name: Load test data
-      run: verdi import --migration --non-interactive .github/aiida/optimade.aiida
+      run: |
+        if [ "${{ matrix.database }}" == "aiida" ]; then
+          verdi import --migration --non-interactive .github/aiida/optimade.aiida
+        else
+            .github/mongo/load_data.py
+        fi
+
+    - name: Build docker image
+      if: matrix.database == 'mongo'
+      run: docker-compose -f profiles/docker-compose.yml build --build-arg CONFIG_FILE=".github/mongo/ci_config.json"
 
-    - name: Build and start the Docker image
+    - name: Start the Docker image
+      env:
+        AIIDA_OPTIMADE_LOG_LEVEL: DEBUG
       run: |
         export DOCKER_HOST_IP=$(ip route | grep docker0 | awk '{print $9}')
         docker-compose -f profiles/docker-compose.yml up &
@@ -166,6 +202,77 @@ jobs:
       with:
         port: 3253
         all versioned paths: yes
+        validate unversioned path: yes
+        validator version: latest
+
+  docker-mongo:
+    runs-on: ubuntu-latest
+    timeout-minutes: 14
+
+    services:
+      postgres:
+        image: postgres:10
+        env:
+          POSTGRES_DB: test_django
+          POSTGRES_PASSWORD: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U setuptools
+
+        while IFS="" read -r line || [ -n "${line}" ]; do
+            if [[ "${line}" =~ ^aiida-core.*$ ]]; then
+                aiida_core="${line}"
+            fi
+        done < requirements.txt
+
+        pip install ${aiida_core}
+        reentry scan
+
+    - name: Setup up environment for AiiDA
+      env:
+        AIIDA_TEST_BACKEND: django
+      run: .github/aiida/setup_aiida.sh
+
+    - name: Load test data (AiiDA)
+      run: verdi import --migration --non-interactive .github/aiida/optimade.aiida
+
+    - name: Build docker image
+      run: docker-compose -f profiles/docker-compose-mongo.yml build --build-arg CONFIG_FILE=".github/mongo/ci_config.json"
+
+    - name: Start the Docker image
+      env:
+        AIIDA_OPTIMADE_LOG_LEVEL: INFO
+        USE_MONGO: '--mongo'
+      run: |
+        export DOCKER_HOST_IP=$(ip route | grep docker0 | awk '{print $9}')
+        docker-compose -f profiles/docker-compose-mongo.yml up &
+        .github/workflows/wait_for_it.sh localhost:3253 -t 360
+        # Long sleep, because the initialization is needed
+        sleep 90
+
+    - name: Test server with OPTIMADE Validator
+      uses: Materials-Consortia/optimade-validator-action@v2
+      with:
+        port: 3253
+        all versioned paths: yes
+        validate unversioned path: yes
         validator version: latest
 
   build-package:

diff --git a/Dockerfile b/Dockerfile
@@ -2,24 +2,26 @@ FROM python:3.8
 
 WORKDIR /app
 
-RUN pip install -U pip setuptools wheel
-
-# Install specific optimade version
+# Install specific optimade and aiida-core versions
 ARG OPTIMADE_TOOLS_VERSION=0.12.9
-RUN pip install optimade==${OPTIMADE_TOOLS_VERSION}
-
-# Install specific aiida-core version
 ARG AIIDA_VERSION=1.5.2
-RUN pip install aiida-core==${AIIDA_VERSION}
-RUN reentry scan
 
 # Copy repo contents
 COPY setup.py setup.json README.md requirements*.txt ./
 COPY aiida_optimade ./aiida_optimade
-RUN pip install -e .
+
+RUN pip install -U pip setuptools wheel \
+    && pip install optimade==${OPTIMADE_TOOLS_VERSION} \
+    && pip install aiida-core==${AIIDA_VERSION} \
+    && reentry scan \
+    && pip install -e .
 
 COPY .docker/run.sh ./
 
 EXPOSE 80
 
+ARG CONFIG_FILE=aiida_optimade/config.json
+COPY ${CONFIG_FILE} ./config.json
+ENV OPTIMADE_CONFIG_FILE /app/config.json
+
 CMD ["/app/run.sh"]