Merge pull request #151 from crim-ca/smoke-test-docker

smoke test docker

.travis.yml

@@ -44,6 +44,7 @@ install:
 stages:
   - check           # run linting checks and don't bother with the rest if invalid
   - test            # use default stage to run job matrix variations
+  - smoke-test      # minimal verifications that docker images work before deploying them
   - deploy-docker   # deploy to dockerhub if successful master/tag tests
 script:
   # start as required as sleep a bit to let application boot before tests
@@ -92,6 +93,11 @@ jobs:
         - cp -f tests/travis-ci/data_sources.json ./
         - make --version
         - make info
+    - stage: smoke-test
+      name: "Docker Smoke Test"
+      python: "3.7"
+      os: linux
+      script: make docker-test
     - stage: deploy-docker
       name: "Deploy Docker"
       python: "3.7"

CHANGES.rst

@@ -4,6 +4,24 @@ Changes
 `Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 ========================================================================
 
+Changes:
+--------
+
+- Add `Travis-CI` smoke test of built docker images for early detection of invalid setup or breaking code to boot them.
+- Add `Travis-CI` checks for imports. This check was not validated previously although available.
+- Adjust ``weaver.ini.example`` to reflect working demo server configuration (employed by smoke test).
+- Move ``weaver`` web application to ``weaver.app`` to reduce chances of breaking ``setup.py`` installation from import
+  errors due to ``weaver`` dependencies not yet installed. Redirect to new location makes this change transparent when
+  loaded with the usual ``weaver.ini`` configuration.
+
+Fixes:
+------
+
+- Fix base docker image to install Python 3 development dependencies in order to compile requirements with expected
+  environment Python version. Package ``python-dev`` for Python 2 was being installed instead.
+- Fix failing docker image boot due to incorrectly placed ``yaml`` import during setup installation.
+- Fix imports according to ``Makefile`` targets ``check-imports`` and ``fix-imports``.
+
 `1.8.0 <https://github.com/crim-ca/weaver/tree/1.8.0>`_ (2020-05-21)
 ========================================================================
 

Makefile

@@ -303,6 +303,9 @@ test-spec: install-dev		## run tests with custom input specification (pytest for
 	@[ "${TESTS}" ] || ( echo ">> 'TESTS' is not set"; exit 1 )
 	@bash -c "$(CONDA_CMD) pytest tests -v -m '${TESTS}' --junitxml $(APP_ROOT)/tests/results.xml"
 
+.PHONY: test-smoke
+test-smoke: docker-test    ## alias to 'docker-test' executing smoke test of built docker images
+
 .PHONY: coverage
 coverage: mkdir-reports install-dev		## run all tests using coverage analysis
 	@echo "Running coverage analysis..."
@@ -321,7 +324,7 @@ mkdir-reports:
 check: check-all	## alias for 'check-all' target
 
 .PHONY: check-all
-check-all: install-dev check-pep8 check-lint check-security check-doc8 check-links	## run every code style checks
+check-all: install-dev check-pep8 check-lint check-imports check-security check-doc8 check-links ## run all code checks
 
 .PHONY: check-pep8
 check-pep8: mkdir-reports install-dev 	## run PEP8 code style checks
@@ -415,20 +418,23 @@ docker-info:		## obtain docker image information
 .PHONY: docker-build-base
 docker-build-base:							## build the base docker image
 	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-base" -t "$(APP_NAME):base"
-	docker tag "$(APP_NAME):base" "$(DOCKER_REPO):$(APP_VERSION)"
+	docker tag "$(APP_NAME):base" "$(APP_NAME):latest"
 	docker tag "$(APP_NAME):base" "$(DOCKER_REPO):latest"
+	docker tag "$(APP_NAME):base" "$(DOCKER_REPO):$(APP_VERSION)"
 
 .PHONY: docker-build-manager
 docker-build-manager: docker-build-base		## build the manager docker image
 	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-manager" -t "$(APP_NAME):$(APP_VERSION)-manager"
-	docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):$(APP_VERSION)-manager"
+	docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(APP_NAME):latest-manager"
 	docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):latest-manager"
+	docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):$(APP_VERSION)-manager"
 
 .PHONY: docker-build-worker
 docker-build-worker: docker-build-base		## build the worker docker image
 	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-worker" -t "$(APP_NAME):$(APP_VERSION)-worker"
-	docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):$(APP_VERSION)-worker"
+	docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(APP_NAME):latest-worker"
 	docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):latest-worker"
+	docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):$(APP_VERSION)-worker"
 
 .PHONY: docker-build
 docker-build: docker-build-base docker-build-manager docker-build-worker		## build all docker images
@@ -449,7 +455,41 @@ docker-push-worker: docker-build-worker		## push the worker docker image
 	docker push "$(DOCKER_REPO):latest-worker"
 
 .PHONY: docker-push
-docker-push: docker-push-base docker-push-manager docker-push-worker	## push all docker images
+docker-push: docker-push-base docker-push-manager docker-push-worker  ## push all docker images
+
+# if compose up fails, print the logs and force stop
+# if compose up succeeds, query weaver to get frontpage response
+DOCKER_TEST_COMPOSES := -f "$(APP_ROOT)/tests/travis-ci/docker-compose.smoke-test.yml"
+.PHONY: docker-test
+docker-test: docker-build stop	## execute smoke test of the built images (validate that they boots and reply)
+	@echo "Smoke test of built application docker images"
+	docker-compose $(DOCKER_TEST_COMPOSES) up -d
+	sleep 10  ## leave some time to boot
+	curl localhost:4001 | grep "Weaver Information" || \
+		( docker-compose $(DOCKER_TEST_COMPOSES) logs weaver worker || true && \
+		  docker-compose $(DOCKER_TEST_COMPOSES) stop; exit 1 )
+	docker-compose $(DOCKER_TEST_COMPOSES) stop
+
+.PHONY: docker-stat
+docker-stat:  ## query docker-compose images status (from 'docker-test')
+	docker-compose $(DOCKER_TEST_COMPOSES) ps
+
+.PHONY: docker-clean
+docker-clean:  ## remove all built docker images (only matching current/latest versions)
+	docker-compose $(DOCKER_TEST_COMPOSES) down || true
+	docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)-manager" || true
+	docker rmi -f "$(DOCKER_REPO):latest-manager" || true
+	docker rmi -f "$(APP_NAME):$(APP_VERSION)-manager" || true
+	docker rmi -f "$(APP_NAME):latest-manager" || true
+	docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)-worker" || true
+	docker rmi -f "$(DOCKER_REPO):latest-worker" || true
+	docker rmi -f "$(APP_NAME):$(APP_VERSION)-worker" || true
+	docker rmi -f "$(APP_NAME):latest-worker" || true
+	docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)" || true
+	docker rmi -f "$(DOCKER_REPO):latest" || true
+	docker rmi -f "$(APP_NAME):$(APP_VERSION)" || true
+	docker rmi -f "$(APP_NAME):latest" || true
+	docker rmi -f "$(APP_NAME):base" || true
 
 ## --- Launchers targets --- ##
 

config/weaver.ini.example

@@ -1,3 +1,5 @@
+# NOTE: This configuration file is employed by Travis-CI smoke test to immediately identify any problematic setting.
+
 ###
 # app configuration
 # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
@@ -13,7 +15,7 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
 # mongodb
-mongodb.host = localhost
+mongodb.host = mongodb
 mongodb.port = 27017
 mongodb.db_name = weaver
 
@@ -88,16 +90,16 @@ weaver.extra_options =
 ###
 [celery]
 #USE_CELERYCONFIG = True
-BROKER_URL = mongodb://localhost:27017/celery
+BROKER_URL = mongodb://mongodb:27017/celery
 
 ###
 # wsgi server configuration
 ###
 
 [server:main]
 use = egg:gunicorn#main
-bind = localhost:4001
-workers = 1
+bind = 0.0.0.0:4001
+workers = 5
 timeout = 10
 
 ###

docker/Dockerfile-base

@@ -20,7 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		ca-certificates \
 		netbase \
 		gcc \
-		python-dev \
+		python3-dev \
 	&& rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir --upgrade pip setuptools \
     && pip install --no-cache-dir -e ${APP_DIR}

requirements-dev.txt

@@ -6,7 +6,7 @@ contextlib2; python_version < "3"
 coverage
 doc8
 flake8
-isort
+isort>=4.3.21,<5
 mock
 pluggy>=0.7
 pytest

requirements.txt

@@ -30,7 +30,7 @@ pyramid_celery
 pyramid_mako
 pytz
 pywps
-pyyaml==5.2
+pyyaml>=5.2
 requests
 requests_file
 # let cwltool define ruamel.yaml version (<=0.16.5)

setup.cfg

@@ -5,13 +5,13 @@ tag = True
 tag_name = {new_version}
 
 [bumpversion:file:CHANGES.rst]
-search = 
+search =
 	`Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 	========================================================================
-replace = 
+replace =
 	`Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 	========================================================================
-	
+
 	`{new_version} <https://github.com/crim-ca/weaver/tree/{new_version}>`_ ({now:%%Y-%%m-%%d})
 	========================================================================
 
@@ -32,12 +32,12 @@ search = LABEL version="{current_version}"
 replace = LABEL version="{new_version}"
 
 [tool:pytest]
-addopts = 
+addopts =
 	--strict
 	--tb=native
 	weaver/
 python_files = test_*.py
-markers = 
+markers =
 	testbed14: mark test as 'testbed14' validation
 	functional: mark test as functionality validation
 	online: mark test to need internet connection
@@ -50,7 +50,7 @@ lines_between_types = 0
 default_section = FIRSTPARTY
 sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
 known_first_party = weaver
-skip_glob = *pyramid.httpexceptions*
+combine_as_imports = true
 
 [bandit]
 skips = B101,B320,B410
@@ -60,7 +60,7 @@ targets = .
 [flake8]
 ignore = E126,E226,E402,F401,W504
 max-line-length = 120
-exclude = 
+exclude =
 	src,
 	.git,
 	__pycache__,
@@ -83,7 +83,7 @@ source = weaver
 omit = tests/*
 
 [coverage:report]
-exclude_lines = 
+exclude_lines =
 	pragma: no cover
 	raise AssertionError
 	raise NotImplementedError

tests/functional/test_wps_app.py

@@ -8,9 +8,9 @@
 """
 import unittest
 
-from lxml import etree
 import pyramid.testing
 import pytest
+from lxml import etree
 
 from tests.utils import (
     get_test_weaver_app,

tests/processes/test_wps_package.py

@@ -9,15 +9,13 @@
 from weaver.exceptions import PackageTypeError
 from weaver.formats import CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_NETCDF, CONTENT_TYPE_APP_XML, CONTENT_TYPE_TEXT_PLAIN
 from weaver.processes.constants import WPS_LITERAL
-from weaver.processes.wps_package import (
-    DEFAULT_FORMAT,
-    _are_different_and_set,     # noqa: W0212
-    _get_package_ordered_io,    # noqa: W0212
-    _is_cwl_array_type,         # noqa: W0212
-    _is_cwl_enum_type,          # noqa: W0212
-    _json2wps_datatype,         # noqa: W0212
-    _merge_io_formats           # noqa: W0212
-)
+from weaver.processes.wps_package import _are_different_and_set  # noqa: W0212
+from weaver.processes.wps_package import _get_package_ordered_io  # noqa: W0212
+from weaver.processes.wps_package import _is_cwl_array_type  # noqa: W0212
+from weaver.processes.wps_package import _is_cwl_enum_type  # noqa: W0212
+from weaver.processes.wps_package import _json2wps_datatype  # noqa: W0212
+from weaver.processes.wps_package import _merge_io_formats  # noqa: W0212
+from weaver.processes.wps_package import DEFAULT_FORMAT
 from weaver.utils import null
 
 

tests/test_formats.py

@@ -9,8 +9,8 @@
 
 from weaver.formats import (
     CONTENT_TYPE_ANY,
-    CONTENT_TYPE_APP_JSON,
     CONTENT_TYPE_APP_GEOJSON,
+    CONTENT_TYPE_APP_JSON,
     CONTENT_TYPE_APP_NETCDF,
     CONTENT_TYPE_APP_XML,
     CONTENT_TYPE_IMAGE_GEOTIFF,
@@ -23,7 +23,7 @@
     clean_mime_type_format,
     get_cwl_file_format,
     get_extension,
-    get_format,
+    get_format
 )
 
 

tests/test_utils.py

@@ -11,31 +11,24 @@
 import pytest
 from lxml import etree
 from pyramid.httpexceptions import (
-    HTTPOk,
-    HTTPCreated,
-    HTTPNotFound,
     HTTPConflict,
+    HTTPCreated,
     HTTPError as PyramidHTTPError,
-    HTTPInternalServerError,
     HTTPGatewayTimeout,
+    HTTPInternalServerError,
+    HTTPNotFound,
+    HTTPOk
 )
 from pywps.response.status import WPS_STATUS
-from requests.exceptions import HTTPError as RequestsHTTPError
 from requests import Response
+from requests.exceptions import HTTPError as RequestsHTTPError
 from six.moves.urllib.parse import urlparse
 
 from tests.compat import contextlib
 from tests.utils import mocked_file_response
 from weaver import status, utils
-from weaver.utils import (
-    _NullType,  # noqa: W0212
-    null,
-    fetch_file,
-    get_ssl_verify_option,
-    get_request_options,
-    make_dirs,
-    request_extra
-)
+from weaver.utils import _NullType  # noqa: W0212
+from weaver.utils import fetch_file, get_request_options, get_ssl_verify_option, make_dirs, null, request_extra
 
 
 def test_null_operators():

tests/travis-ci/docker-compose.smoke-test.yml

@@ -0,0 +1,32 @@
+version: "3.4"
+services:
+  mongodb:
+    image: mongo:3.6.0
+    container_name: smoke_test_mongodb
+    # mongodb crash with permission denied errors if the command is not overridden like this
+    command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod --bind_ip_all'
+    restart: "no"
+
+  weaver:
+    image: weaver:latest-manager
+    container_name: smoke_test_weaver
+    environment:
+      FORWARDED_ALLOW_IPS: "*"
+    links:
+      - mongodb
+    ports:
+      - "4001:4001"
+    volumes:
+      - ../../config/weaver.ini.example:/opt/local/src/weaver/config/weaver.ini
+    networks:
+      - default
+    restart: "no"
+
+  worker:
+    image: weaver:latest-worker
+    container_name: smoke_test_worker
+    links:
+      - mongodb
+    volumes:
+      - ../../config/weaver.ini.example:/opt/local/src/weaver/config/weaver.ini
+    restart: "no"