Skip to content

Commit

Permalink
Merge pull request #36 from aiidateam/develop
Browse files Browse the repository at this point in the history
Up to v0.4.0

Due to rebase and commit for v0.3.0, some commits will be redone.
This cannot be undone without losing pointers to the v0.3.0 published commit.
One should either merge with --ff-only from command line or make a merge commit
and then merge back into `develop` using a merge commit as well.
This update is considered from commit 12d2237.

Updates:
- OPTiMaDe spec v0.10.1.
- optimade-python-tools v0.3.1 (with grammar for v0.10.1).
- Redone HTTP middleware to handle session per request.
- Update to pydantic v1.
- Add implementation information,
  i.e., information about which version of `aiida-optimade` the server is using,
  to all `meta` responses.
- Use server.cfg.
- Fix broken Docker build, and add it to CI (@ltalirz)
  • Loading branch information
CasperWA committed Jan 19, 2020
2 parents 60c60b4 + 36c6d7c commit 35ffd08
Show file tree
Hide file tree
Showing 37 changed files with 1,252 additions and 581 deletions.
20 changes: 11 additions & 9 deletions .ci/assert_version.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
# pylint: disable=line-too-long
import os
import json
import sys
from pathlib import Path

SETUP_JSON = Path(__file__).resolve().parent.parent.joinpath("setup.json")

with open(SETUP_JSON, "r") as fp:
setup = json.load(fp)
SETUP = json.load(fp)

package_version = "v" + setup["version"]
PACKAGE_VERSION = "v" + SETUP["version"]

tag_version = os.getenv("TAG_VERSION")
tag_version = tag_version[len("refs/tags/") :]
TAG_VERSION = os.getenv("TAG_VERSION")
TAG_VERSION = TAG_VERSION[len("refs/tags/") :]

if tag_version == package_version:
print(f"The versions match: tag:'{tag_version}' == package:'{package_version}'")
exit(0)
if TAG_VERSION == PACKAGE_VERSION:
print(f"The versions match: tag:'{TAG_VERSION}' == package:'{PACKAGE_VERSION}'")
sys.exit(0)

print(
f"""The current package version '{package_version}' does not equal the tag version '{tag_version}'.
f"""The current package version '{PACKAGE_VERSION}' does not equal the tag version '{TAG_VERSION}'.
Update setup.json with new version.
Please remove the tag from both GitHub and your local repository!"""
)
exit(1)
sys.exit(1)
6 changes: 0 additions & 6 deletions .ci/optimade-version.json

This file was deleted.

51 changes: 0 additions & 51 deletions .ci/optimade_version_update.py

This file was deleted.

File renamed without changes.
6 changes: 6 additions & 0 deletions .docker/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,10 @@
set -ex
mkdir -p $AIIDA_PATH/.aiida
cp /profiles/$AIIDA_PROFILE.json $AIIDA_PATH/.aiida/config.json

# make docker.host.internal available
# see https://github.com/docker/for-linux/issues/264#issuecomment-387525409
# ltalirz: Only works for Mac, not Linux
# echo -e "`/sbin/ip route|awk '/default/ { print $3 }'`\tdocker.host.internal" | tee -a /etc/hosts > /dev/null

uvicorn aiida_optimade.main:app --host 0.0.0.0 --port 80
76 changes: 62 additions & 14 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -U setuptools
pip install flake8
- name: Lint with flake8
Expand All @@ -34,16 +33,14 @@ jobs:

strategy:
fail-fast: false
matrix:
python-version: [3.7, 3.8]

steps:
- uses: actions/checkout@v1

- name: Set up Python ${{ matrix.python-version}}
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version}}
python-version: 3.7

- name: Install dependencies
run: |
Expand All @@ -52,8 +49,7 @@ jobs:
pip install pre-commit
- name: Test with pre-commit
run: |
pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )

pytest:

Expand All @@ -62,7 +58,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.7, 3.8]
python-version: [3.6, 3.7, 3.8]
backend: ['django', 'sqlalchemy']

steps:
Expand Down Expand Up @@ -100,21 +96,21 @@ jobs:
pip install -e .[dev]
reentry scan
- name: Setup server.cfg for AiiDA OPTiMaDe server
run: cp .ci/server_template.cfg ./server.cfg

- name: Setup up environment for AiiDA
env:
AIIDA_TEST_BACKEND: ${{ matrix.backend }}
run: |
.github/workflows/setup_aiida.sh
run: .github/workflows/setup_aiida.sh

- name: Load test data
run: |
verdi import --migration --non-interactive .github/aiida/optimade.aiida
run: verdi import --migration --non-interactive .github/aiida/optimade.aiida

- name: Test with pytest
env:
AIIDA_PROFILE: test_${{ matrix.backend }}
run: |
pytest --cov=./aiida_optimade/ --cov-report=xml
run: pytest --cov=./aiida_optimade/ --cov-report=xml

- name: Upload coverage to Codecov
if: matrix.python-version == 3.7
Expand All @@ -124,3 +120,55 @@ jobs:
xml: ./coverage.xml
flags: unittests
yml: ./.codecov.yml

docker-image:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v1

# - uses: harmon758/postgresql-action@v1
# with:
# postgresql version: '11'
# postgresql db: test_django
# postgresql user: 'postgres'
# postgresql password: ''

# - name: Set up Python 3.7
# uses: actions/setup-python@v1
# with:
# python-version: 3.7

# - name: Install system dependencies
# run: |
# wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
# echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
# echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
# sudo apt update
# sudo apt install postgresql postgresql-server-dev-all postgresql-client rabbitmq-server graphviz
# sudo systemctl status rabbitmq-server.service

# # Install optimade-python-tools from github
# - name: Install python dependencies
# run: |
# cd ${GITHUB_WORKSPACE}/..
# git clone https://github.com/Materials-Consortia/optimade-python-tools
# pip install -e optimade-python-tools
# cd ${GITHUB_WORKSPACE}
# pip install -e .
# reentry scan

# - name: Setup up environment for AiiDA
# env:
# AIIDA_TEST_BACKEND: django
# run: .github/workflows/setup_aiida.sh

# - name: Load test data
# run: verdi import --migration --non-interactive .github/aiida/optimade.aiida

- name: Build the Docker image
run: docker-compose -f profiles/docker-compose.yml build
# .github/workflows/wait_for_it.sh localhost:3253 -t 120
# sleep 15
# curl http://localhost:3253/optimade/info > info.json
# grep -F "www.aiida.net" info.json || exit 1
161 changes: 161 additions & 0 deletions .github/workflows/wait_for_it.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
#!/usr/bin/env bash
# Use this script to test if a given TCP host/port are available

cmdname=$(basename $0)

echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }

usage()
{
cat << USAGE >&2
Usage:
$cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}

wait_for()
{
if [[ $TIMEOUT -gt 0 ]]; then
echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
else
echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
fi
start_ts=$(date +%s)
while :
do
(echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
result=$?
if [[ $result -eq 0 ]]; then
end_ts=$(date +%s)
echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
break
fi
sleep 1
done
return $result
}

wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $QUIET -eq 1 ]]; then
timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
else
timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
fi
PID=$!
trap "kill -INT -$PID" INT
wait $PID
RESULT=$?
if [[ $RESULT -ne 0 ]]; then
echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
fi
return $RESULT
}

# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
hostport=(${1//:/ })
HOST=${hostport[0]}
PORT=${hostport[1]}
shift 1
;;
--child)
CHILD=1
shift 1
;;
-q | --quiet)
QUIET=1
shift 1
;;
-s | --strict)
STRICT=1
shift 1
;;
-h)
HOST="$2"
if [[ $HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
HOST="${1#*=}"
shift 1
;;
-p)
PORT="$2"
if [[ $PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
PORT="${1#*=}"
shift 1
;;
-t)
TIMEOUT="$2"
if [[ $TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
CLI="$@"
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done

if [[ "$HOST" == "" || "$PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi

TIMEOUT=${TIMEOUT:-15}
STRICT=${STRICT:-0}
CHILD=${CHILD:-0}
QUIET=${QUIET:-0}

if [[ $CHILD -gt 0 ]]; then
wait_for
RESULT=$?
exit $RESULT
else
if [[ $TIMEOUT -gt 0 ]]; then
wait_for_wrapper
RESULT=$?
else
wait_for
RESULT=$?
fi
fi

if [[ $CLI != "" ]]; then
if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
echoerr "$cmdname: strict mode, refusing to execute subprocess"
exit $RESULT
fi
exec $CLI
else
exit $RESULT
fi
Loading

0 comments on commit 35ffd08

Please sign in to comment.