From 63115e91605e5fe3eab0225803eb44185478887b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 22 May 2020 01:18:33 -0400 Subject: [PATCH 1/5] add docker smoke test + fix docker install + move webapp to weaver.app --- .travis.yml | 6 ++ CHANGES.rst | 18 ++++++ Makefile | 50 +++++++++++++-- config/weaver.ini.example | 10 +-- docker/Dockerfile-base | 2 +- requirements.txt | 2 +- tests/test_formats.py | 2 +- tests/travis-ci/docker-compose.smoke-test.yml | 32 ++++++++++ weaver/__init__.py | 62 +++---------------- weaver/app.py | 61 ++++++++++++++++++ weaver/config.py | 5 +- weaver/processes/wps_package.py | 2 +- weaver/wps_restapi/processes/processes.py | 2 +- 13 files changed, 186 insertions(+), 68 deletions(-) create mode 100644 tests/travis-ci/docker-compose.smoke-test.yml create mode 100644 weaver/app.py diff --git a/.travis.yml b/.travis.yml index 5e08d7a29..30ad7cc1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ install: stages: - check # run linting checks and don't bother with the rest if invalid - test # use default stage to run job matrix variations + - smoke-test # minimal verifications that docker images work before deploying them - deploy-docker # deploy to dockerhub if successful master/tag tests script: # start as required as sleep a bit to let application boot before tests @@ -92,6 +93,11 @@ jobs: - cp -f tests/travis-ci/data_sources.json ./ - make --version - make info + - stage: smoke-test + name: "Docker Smoke Test" + python: "3.7" + os: linux + script: make docker-test - stage: deploy-docker name: "Deploy Docker" python: "3.7" diff --git a/CHANGES.rst b/CHANGES.rst index afb2371f8..5f41bfdef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,24 @@ Changes `Unreleased `_ (latest) ======================================================================== +Changes: +-------- + +- Add `Travis-CI` smoke test of built docker images for early detection of invalid setup or breaking code to boot them. +- Add `Travis-CI` checks for imports. This check was not validated previously although available. +- Adjust ``weaver.ini.example`` to reflect working demo server configuration (employed by smoke test). +- Move ``weaver`` web application to ``weaver.app`` to reduce chances of breaking ``setup.py`` installation from import + errors due to ``weaver`` dependencies not yet installed. Redirect to new location makes this change transparent when + loaded with the usual ``weaver.ini`` configuration. + +Fixes: +------ + +- Fix base docker image to install Python 3 development dependencies in order to compile requirements with expected + environment Python version. Package ``python-dev`` for Python 2 was being installed instead. +- Fix failing docker image boot due to incorrectly placed ``yaml`` import during setup installation. +- Fix imports according to ``Makefile`` targets ``check-imports`` and ``fix-imports``. + `1.8.0 `_ (2020-05-21) ======================================================================== diff --git a/Makefile b/Makefile index f065918dc..4ac54a2d7 100644 --- a/Makefile +++ b/Makefile @@ -303,6 +303,9 @@ test-spec: install-dev ## run tests with custom input specification (pytest for @[ "${TESTS}" ] || ( echo ">> 'TESTS' is not set"; exit 1 ) @bash -c "$(CONDA_CMD) pytest tests -v -m '${TESTS}' --junitxml $(APP_ROOT)/tests/results.xml" +.PHONY: test-smoke +test-smoke: docker-test ## alias to 'docker-test' executing smoke test of built docker images + .PHONY: coverage coverage: mkdir-reports install-dev ## run all tests using coverage analysis @echo "Running coverage analysis..." @@ -321,7 +324,7 @@ mkdir-reports: check: check-all ## alias for 'check-all' target .PHONY: check-all -check-all: install-dev check-pep8 check-lint check-security check-doc8 check-links ## run every code style checks +check-all: install-dev check-pep8 check-lint check-imports check-security check-doc8 check-links ## run all code checks .PHONY: check-pep8 check-pep8: mkdir-reports install-dev ## run PEP8 code style checks @@ -415,20 +418,23 @@ docker-info: ## obtain docker image information .PHONY: docker-build-base docker-build-base: ## build the base docker image docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-base" -t "$(APP_NAME):base" - docker tag "$(APP_NAME):base" "$(DOCKER_REPO):$(APP_VERSION)" + docker tag "$(APP_NAME):base" "$(APP_NAME):latest" docker tag "$(APP_NAME):base" "$(DOCKER_REPO):latest" + docker tag "$(APP_NAME):base" "$(DOCKER_REPO):$(APP_VERSION)" .PHONY: docker-build-manager docker-build-manager: docker-build-base ## build the manager docker image docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-manager" -t "$(APP_NAME):$(APP_VERSION)-manager" - docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):$(APP_VERSION)-manager" + docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(APP_NAME):latest-manager" docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):latest-manager" + docker tag "$(APP_NAME):$(APP_VERSION)-manager" "$(DOCKER_REPO):$(APP_VERSION)-manager" .PHONY: docker-build-worker docker-build-worker: docker-build-base ## build the worker docker image docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile-worker" -t "$(APP_NAME):$(APP_VERSION)-worker" - docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):$(APP_VERSION)-worker" + docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(APP_NAME):latest-worker" docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):latest-worker" + docker tag "$(APP_NAME):$(APP_VERSION)-worker" "$(DOCKER_REPO):$(APP_VERSION)-worker" .PHONY: docker-build docker-build: docker-build-base docker-build-manager docker-build-worker ## build all docker images @@ -449,7 +455,41 @@ docker-push-worker: docker-build-worker ## push the worker docker image docker push "$(DOCKER_REPO):latest-worker" .PHONY: docker-push -docker-push: docker-push-base docker-push-manager docker-push-worker ## push all docker images +docker-push: docker-push-base docker-push-manager docker-push-worker ## push all docker images + +# if compose up fails, print the logs and force stop +# if compose up succeeds, query weaver to get frontpage response +DOCKER_TEST_COMPOSES := -f "$(APP_ROOT)/tests/travis-ci/docker-compose.smoke-test.yml" +.PHONY: docker-test +docker-test: docker-build stop ## execute smoke test of the built images (validate that they boots and reply) + @echo "Smoke test of built application docker images" + docker-compose $(DOCKER_TEST_COMPOSES) up -d + sleep 10 ## leave some time to boot + curl localhost:4001 | grep "Weaver Information" || \ + ( docker-compose $(DOCKER_TEST_COMPOSES) logs weaver worker || true && \ + docker-compose $(DOCKER_TEST_COMPOSES) stop; exit 1 ) + docker-compose $(DOCKER_TEST_COMPOSES) stop + +.PHONY: docker-stat +docker-stat: ## query docker-compose images status (from 'docker-test') + docker-compose $(DOCKER_TEST_COMPOSES) ps + +.PHONY: docker-clean +docker-clean: ## remove all built docker images (only matching current/latest versions) + docker-compose $(DOCKER_TEST_COMPOSES) down || true + docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)-manager" || true + docker rmi -f "$(DOCKER_REPO):latest-manager" || true + docker rmi -f "$(APP_NAME):$(APP_VERSION)-manager" || true + docker rmi -f "$(APP_NAME):latest-manager" || true + docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)-worker" || true + docker rmi -f "$(DOCKER_REPO):latest-worker" || true + docker rmi -f "$(APP_NAME):$(APP_VERSION)-worker" || true + docker rmi -f "$(APP_NAME):latest-worker" || true + docker rmi -f "$(DOCKER_REPO):$(APP_VERSION)" || true + docker rmi -f "$(DOCKER_REPO):latest" || true + docker rmi -f "$(APP_NAME):$(APP_VERSION)" || true + docker rmi -f "$(APP_NAME):latest" || true + docker rmi -f "$(APP_NAME):base" || true ## --- Launchers targets --- ## diff --git a/config/weaver.ini.example b/config/weaver.ini.example index 97c2dad56..300085ff8 100644 --- a/config/weaver.ini.example +++ b/config/weaver.ini.example @@ -1,3 +1,5 @@ +# NOTE: This configuration file is employed by Travis-CI smoke test to immediately identify any problematic setting. + ### # app configuration # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html @@ -13,7 +15,7 @@ pyramid.debug_routematch = false pyramid.default_locale_name = en # mongodb -mongodb.host = localhost +mongodb.host = mongodb mongodb.port = 27017 mongodb.db_name = weaver @@ -88,7 +90,7 @@ weaver.extra_options = ### [celery] #USE_CELERYCONFIG = True -BROKER_URL = mongodb://localhost:27017/celery +BROKER_URL = mongodb://mongodb:27017/celery ### # wsgi server configuration @@ -96,8 +98,8 @@ BROKER_URL = mongodb://localhost:27017/celery [server:main] use = egg:gunicorn#main -bind = localhost:4001 -workers = 1 +bind = 0.0.0.0:4001 +workers = 5 timeout = 10 ### diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base index 6c83a1dec..6b3bd2177 100644 --- a/docker/Dockerfile-base +++ b/docker/Dockerfile-base @@ -20,7 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ netbase \ gcc \ - python-dev \ + python3-dev \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir --upgrade pip setuptools \ && pip install --no-cache-dir -e ${APP_DIR} diff --git a/requirements.txt b/requirements.txt index 244deac44..c26498b1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ pyramid_celery pyramid_mako pytz pywps -pyyaml==5.2 +pyyaml>=5.2 requests requests_file # let cwltool define ruamel.yaml version (<=0.16.5) diff --git a/tests/test_formats.py b/tests/test_formats.py index c37e4a516..0aec3f091 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -23,7 +23,7 @@ clean_mime_type_format, get_cwl_file_format, get_extension, - get_format, + get_format ) diff --git a/tests/travis-ci/docker-compose.smoke-test.yml b/tests/travis-ci/docker-compose.smoke-test.yml new file mode 100644 index 000000000..af3742ac0 --- /dev/null +++ b/tests/travis-ci/docker-compose.smoke-test.yml @@ -0,0 +1,32 @@ +version: "3.4" +services: + mongodb: + image: mongo:3.6.0 + container_name: smoke_test_mongodb + # mongodb crash with permission denied errors if the command is not overridden like this + command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod --bind_ip_all' + restart: "no" + + weaver: + image: weaver:latest-manager + container_name: smoke_test_weaver + environment: + FORWARDED_ALLOW_IPS: "*" + links: + - mongodb + ports: + - "4001:4001" + volumes: + - ../../config/weaver.ini.example:/opt/local/src/weaver/config/weaver.ini + networks: + - default + restart: "no" + + worker: + image: weaver:latest-worker + container_name: smoke_test_worker + links: + - mongodb + volumes: + - ../../config/weaver.ini.example:/opt/local/src/weaver/config/weaver.ini + restart: "no" diff --git a/weaver/__init__.py b/weaver/__init__.py index 4db45111a..740fb037d 100644 --- a/weaver/__init__.py +++ b/weaver/__init__.py @@ -1,10 +1,12 @@ import logging import os import sys -import yaml + +# NOTE: +# DO NOT IMPORT ANYTHING NOT PROVIDED BY PYTHON STANDARD LIBRARY HERE TO AVOID "setup.py" INSTALL FAILURE logging.captureWarnings(True) -LOGGER = logging.getLogger("weaver") +LOGGER = logging.getLogger(__name__) WEAVER_MODULE_DIR = os.path.abspath(os.path.dirname(__file__)) WEAVER_ROOT_DIR = os.path.abspath(os.path.dirname(WEAVER_MODULE_DIR)) @@ -13,58 +15,14 @@ sys.path.insert(0, WEAVER_MODULE_DIR) # provide standard package version location -from __meta__ import __version__ # isort:skip # noqa: E402 # pylint: disable=C0413 - -# =============================================================================================== -# DO NOT IMPORT ANYTHING NOT PROVIDED BY BASE PYTHON HERE TO AVOID "setup.py" INSTALL FAILURE -# =============================================================================================== - - -def includeme(config): - LOGGER.info("Adding weaver...") - config.include("weaver.config") - config.include("weaver.database") - config.include("weaver.wps") - config.include("weaver.wps_restapi") - config.include("weaver.processes") - config.include("weaver.tweens") +from __meta__ import __version__ # noqa: E402,F401 # isort:skip # pylint: disable=C0413 def main(global_config, **settings): - """ - Creates a Pyramid WSGI application for Weaver. - """ - LOGGER.info("Initiating weaver application") - - from weaver.config import WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, get_weaver_configuration, get_weaver_config_file - from weaver.processes.builtin import register_builtin_processes - from weaver.processes.utils import register_wps_processes_from_config - from weaver.utils import parse_extra_options, get_settings - from pyramid.config import Configurator + import weaver.app + return weaver.app.main(global_config, **settings) - # validate and fix configuration - weaver_config = get_weaver_configuration(settings) - settings.update({"weaver.configuration": weaver_config}) - # Parse extra_options and add each of them in the settings dict - settings.update(parse_extra_options(settings.get("weaver.extra_options", ""))) - - # load requests options - req_file = get_weaver_config_file(settings.get("weaver.request_options", ""), WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG) - with open(req_file, "r") as f: - settings.update({"weaver.request_options": yaml.safe_load(f)}) - - local_config = Configurator(settings=settings) - if global_config.get("__file__") is not None: - local_config.include("pyramid_celery") - local_config.configure_celery(global_config["__file__"]) - local_config.include("weaver") - - LOGGER.info("Registering builtin processes...") - register_builtin_processes(local_config) - - LOGGER.info("Registering WPS-1 processes from configuration file...") - wps_processes_file = get_settings(local_config).get("weaver.wps_processes_file") - register_wps_processes_from_config(wps_processes_file, local_config) - - return local_config.make_wsgi_app() +def includeme(config): + LOGGER.info("Adding Weaver Package") + config.include("weaver.app") diff --git a/weaver/app.py b/weaver/app.py new file mode 100644 index 000000000..d251ae8d8 --- /dev/null +++ b/weaver/app.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Weaver Web Application (``weaver-manager``). +""" + +import logging + +import yaml +from pyramid.config import Configurator + +from weaver.config import WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, get_weaver_configuration, get_weaver_config_file +from weaver.processes.builtin import register_builtin_processes +from weaver.processes.utils import register_wps_processes_from_config +from weaver.utils import parse_extra_options, get_settings + +LOGGER = logging.getLogger(__name__) + + +def includeme(config): + LOGGER.info("Adding Web Application") + config.include("weaver.config") + config.include("weaver.database") + config.include("weaver.wps") + config.include("weaver.wps_restapi") + config.include("weaver.processes") + config.include("weaver.tweens") + + +def main(global_config, **settings): + """ + Creates a Pyramid WSGI application for Weaver. + """ + LOGGER.info("Initiating weaver application") + + # validate and fix configuration + weaver_config = get_weaver_configuration(settings) + settings.update({"weaver.configuration": weaver_config}) + + # Parse extra_options and add each of them in the settings dict + settings.update(parse_extra_options(settings.get("weaver.extra_options", ""))) + + # load requests options + req_file = get_weaver_config_file(settings.get("weaver.request_options", ""), WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG) + with open(req_file, "r") as f: + settings.update({"weaver.request_options": yaml.safe_load(f)}) + + local_config = Configurator(settings=settings) + if global_config.get("__file__") is not None: + local_config.include("pyramid_celery") + local_config.configure_celery(global_config["__file__"]) + local_config.include("weaver") + + LOGGER.info("Registering builtin processes...") + register_builtin_processes(local_config) + + LOGGER.info("Registering WPS-1 processes from configuration file...") + wps_processes_file = get_settings(local_config).get("weaver.wps_processes_file") + register_wps_processes_from_config(wps_processes_file, local_config) + + return local_config.make_wsgi_app() diff --git a/weaver/config.py b/weaver/config.py index 581384ab4..4d66ceeb5 100644 --- a/weaver/config.py +++ b/weaver/config.py @@ -74,11 +74,12 @@ def get_weaver_config_file(file_path, default_config_file, generate_default_from default_path = os.path.abspath(os.path.join(WEAVER_CONFIG_DIR, default_config_file)) if file_path in [default_config_file, os.path.join(os.curdir, default_config_file)]: file_path = default_path - file_path = os.path.abspath(file_path) + if str(file_path).strip() != "": + file_path = os.path.abspath(file_path) if os.path.isfile(file_path): LOGGER.info("Resolved specified configuration file: [%s]", file_path) return file_path - LOGGER.warning("Cannot find configuration file: [%s]. Falling back to default.", file_path) + LOGGER.warning("Cannot find configuration file: [%s]. Falling back to default.", file_path or "") if os.path.isfile(default_path): LOGGER.info("Resolved default configuration file: [%s]", default_path) return default_path diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index c8523f25c..3ddce1f6e 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -56,7 +56,7 @@ CONTENT_TYPE_TEXT_PLAIN, get_cwl_file_format, get_extension, - get_format, + get_format ) from weaver.processes import opensearch from weaver.processes.constants import ( diff --git a/weaver/wps_restapi/processes/processes.py b/weaver/wps_restapi/processes/processes.py index a16c81d9d..c1cca5125 100644 --- a/weaver/wps_restapi/processes/processes.py +++ b/weaver/wps_restapi/processes/processes.py @@ -51,7 +51,7 @@ get_settings, raise_on_xml_exception, request_extra, - wait_secs, + wait_secs ) from weaver.visibility import VISIBILITY_PUBLIC, VISIBILITY_VALUES from weaver.wps import get_wps_output_dir, get_wps_output_path, get_wps_output_url, load_pywps_cfg From 3b489cf8870e7619a1c51f619a914bc7de04dd5a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 22 May 2020 01:40:58 -0400 Subject: [PATCH 2/5] fix imports --- setup.cfg | 16 ++++++++-------- tests/functional/test_wps_app.py | 2 +- tests/test_formats.py | 2 +- tests/test_utils.py | 10 +++++----- weaver/app.py | 4 ++-- weaver/formats.py | 2 +- weaver/processes/utils.py | 3 +-- weaver/processes/wps_package.py | 4 +--- weaver/processes/wps_workflow.py | 10 ++++++++-- weaver/utils.py | 2 +- weaver/wps_restapi/quotation/__init__.py | 3 +-- weaver/wps_restapi/swagger_definitions.py | 17 ++++++++++++----- 12 files changed, 42 insertions(+), 33 deletions(-) diff --git a/setup.cfg b/setup.cfg index f341b2b18..456747088 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,13 +5,13 @@ tag = True tag_name = {new_version} [bumpversion:file:CHANGES.rst] -search = +search = `Unreleased `_ (latest) ======================================================================== -replace = +replace = `Unreleased `_ (latest) ======================================================================== - + `{new_version} `_ ({now:%%Y-%%m-%%d}) ======================================================================== @@ -32,12 +32,12 @@ search = LABEL version="{current_version}" replace = LABEL version="{new_version}" [tool:pytest] -addopts = +addopts = --strict --tb=native weaver/ python_files = test_*.py -markers = +markers = testbed14: mark test as 'testbed14' validation functional: mark test as functionality validation online: mark test to need internet connection @@ -50,7 +50,7 @@ lines_between_types = 0 default_section = FIRSTPARTY sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER known_first_party = weaver -skip_glob = *pyramid.httpexceptions* +combine_as_imports = true [bandit] skips = B101,B320,B410 @@ -60,7 +60,7 @@ targets = . [flake8] ignore = E126,E226,E402,F401,W504 max-line-length = 120 -exclude = +exclude = src, .git, __pycache__, @@ -83,7 +83,7 @@ source = weaver omit = tests/* [coverage:report] -exclude_lines = +exclude_lines = pragma: no cover raise AssertionError raise NotImplementedError diff --git a/tests/functional/test_wps_app.py b/tests/functional/test_wps_app.py index ab3935fe6..7aa378585 100644 --- a/tests/functional/test_wps_app.py +++ b/tests/functional/test_wps_app.py @@ -8,9 +8,9 @@ """ import unittest -from lxml import etree import pyramid.testing import pytest +from lxml import etree from tests.utils import ( get_test_weaver_app, diff --git a/tests/test_formats.py b/tests/test_formats.py index 0aec3f091..9de00fb49 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -9,8 +9,8 @@ from weaver.formats import ( CONTENT_TYPE_ANY, - CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_GEOJSON, + CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_NETCDF, CONTENT_TYPE_APP_XML, CONTENT_TYPE_IMAGE_GEOTIFF, diff --git a/tests/test_utils.py b/tests/test_utils.py index a29c5459c..1e09a9d6b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,17 +11,17 @@ import pytest from lxml import etree from pyramid.httpexceptions import ( - HTTPOk, - HTTPCreated, - HTTPNotFound, HTTPConflict, + HTTPCreated, HTTPError as PyramidHTTPError, - HTTPInternalServerError, HTTPGatewayTimeout, + HTTPInternalServerError, + HTTPNotFound, + HTTPOk ) from pywps.response.status import WPS_STATUS -from requests.exceptions import HTTPError as RequestsHTTPError from requests import Response +from requests.exceptions import HTTPError as RequestsHTTPError from six.moves.urllib.parse import urlparse from tests.compat import contextlib diff --git a/weaver/app.py b/weaver/app.py index d251ae8d8..27b0ea9b6 100644 --- a/weaver/app.py +++ b/weaver/app.py @@ -9,10 +9,10 @@ import yaml from pyramid.config import Configurator -from weaver.config import WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, get_weaver_configuration, get_weaver_config_file +from weaver.config import WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, get_weaver_config_file, get_weaver_configuration from weaver.processes.builtin import register_builtin_processes from weaver.processes.utils import register_wps_processes_from_config -from weaver.utils import parse_extra_options, get_settings +from weaver.utils import get_settings, parse_extra_options LOGGER = logging.getLogger(__name__) diff --git a/weaver/formats.py b/weaver/formats.py index 2086c4436..efab43e86 100644 --- a/weaver/formats.py +++ b/weaver/formats.py @@ -1,8 +1,8 @@ import os from typing import TYPE_CHECKING +from pyramid.httpexceptions import HTTPNotFound, HTTPOk from pywps.inout.formats import FORMATS, Format -from pyramid.httpexceptions import HTTPOk, HTTPNotFound from requests.exceptions import ConnectionError from six.moves.urllib.error import HTTPError from six.moves.urllib.request import urlopen diff --git a/weaver/processes/utils.py b/weaver/processes/utils.py index 7b87b8c9f..1ff7911bb 100644 --- a/weaver/processes/utils.py +++ b/weaver/processes/utils.py @@ -29,8 +29,7 @@ get_weaver_configuration ) from weaver.database import get_db -from weaver.datatype import Process as ProcessDB -from weaver.datatype import Service +from weaver.datatype import Process as ProcessDB, Service from weaver.exceptions import ( InvalidIdentifierValue, PackageNotFound, diff --git a/weaver/processes/wps_package.py b/weaver/processes/wps_package.py index 3ddce1f6e..e87cd2e99 100644 --- a/weaver/processes/wps_package.py +++ b/weaver/processes/wps_package.py @@ -26,9 +26,7 @@ import six import yaml from cwltool.context import LoadingContext, RuntimeContext -from owslib.wps import ComplexData -from owslib.wps import Metadata as OwsMetadata -from owslib.wps import WebProcessingService +from owslib.wps import ComplexData, Metadata as OwsMetadata, WebProcessingService from pyramid.httpexceptions import HTTPOk, HTTPServiceUnavailable from pyramid_celery import celery_app as app from pywps import Process diff --git a/weaver/processes/wps_workflow.py b/weaver/processes/wps_workflow.py index 9fa2800dc..2e2cee98e 100644 --- a/weaver/processes/wps_workflow.py +++ b/weaver/processes/wps_workflow.py @@ -14,8 +14,14 @@ from cwltool.errors import WorkflowException from cwltool.job import JobBase, relink_initialworkdir from cwltool.pathmapper import adjustDirObjs, adjustFileObjs, get_listing, trim_listing, visit_class -from cwltool.process import Process as ProcessCWL -from cwltool.process import compute_checksums, normalizeFilesDirs, shortname, supportedProcessRequirements, uniquename +from cwltool.process import ( + Process as ProcessCWL, + compute_checksums, + normalizeFilesDirs, + shortname, + supportedProcessRequirements, + uniquename +) from cwltool.stdfsaccess import StdFsAccess from cwltool.utils import aslist, bytes2str_in_dicts, onWindows from cwltool.workflow import Workflow diff --git a/weaver/utils.py b/weaver/utils.py index 7506a6c98..8b741b743 100644 --- a/weaver/utils.py +++ b/weaver/utils.py @@ -25,8 +25,8 @@ from requests import HTTPError as RequestsHTTPError, Response from requests.structures import CaseInsensitiveDict from requests_file import FileAdapter -from urlmatch import urlmatch from six.moves.urllib.parse import ParseResult, parse_qs, urlparse, urlunsplit +from urlmatch import urlmatch from webob.headers import EnvironHeaders, ResponseHeaders from weaver.exceptions import InvalidIdentifierValue diff --git a/weaver/wps_restapi/quotation/__init__.py b/weaver/wps_restapi/quotation/__init__.py index 111e8fe2c..76d5435d8 100644 --- a/weaver/wps_restapi/quotation/__init__.py +++ b/weaver/wps_restapi/quotation/__init__.py @@ -1,8 +1,7 @@ import logging from weaver.wps_restapi import swagger_definitions as sd -from weaver.wps_restapi.quotation import bills as b -from weaver.wps_restapi.quotation import quotes as q +from weaver.wps_restapi.quotation import bills as b, quotes as q from weaver.wps_restapi.utils import OUTPUT_FORMAT_JSON LOGGER = logging.getLogger(__name__) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index e9030292a..b3210755f 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -4,11 +4,18 @@ """ # pylint: disable=C0103,invalid-name -from colander import Boolean, DateTime, Float, Integer -from colander import MappingSchema as MapSchema -from colander import OneOf, Range -from colander import SequenceSchema as SeqSchema -from colander import String, drop +from colander import ( + Boolean, + DateTime, + Float, + Integer, + MappingSchema as MapSchema, + OneOf, + Range, + SequenceSchema as SeqSchema, + String, + drop +) from cornice import Service from weaver import __meta__ From 4c05fbdab619027ca1ea66dfd873860b7f05df17 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 22 May 2020 12:46:16 -0400 Subject: [PATCH 3/5] allow isort misbehaving until issue is adressed (https://github.com/timothycrosley/isort/issues/1210) --- tests/processes/test_wps_package.py | 16 +++++++--------- tests/test_utils.py | 11 ++--------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/processes/test_wps_package.py b/tests/processes/test_wps_package.py index 0e5506364..3dbec1873 100644 --- a/tests/processes/test_wps_package.py +++ b/tests/processes/test_wps_package.py @@ -9,15 +9,13 @@ from weaver.exceptions import PackageTypeError from weaver.formats import CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_NETCDF, CONTENT_TYPE_APP_XML, CONTENT_TYPE_TEXT_PLAIN from weaver.processes.constants import WPS_LITERAL -from weaver.processes.wps_package import ( - DEFAULT_FORMAT, - _are_different_and_set, # noqa: W0212 - _get_package_ordered_io, # noqa: W0212 - _is_cwl_array_type, # noqa: W0212 - _is_cwl_enum_type, # noqa: W0212 - _json2wps_datatype, # noqa: W0212 - _merge_io_formats # noqa: W0212 -) +from weaver.processes.wps_package import _are_different_and_set # noqa: W0212 +from weaver.processes.wps_package import _get_package_ordered_io # noqa: W0212 +from weaver.processes.wps_package import _is_cwl_array_type # noqa: W0212 +from weaver.processes.wps_package import _is_cwl_enum_type # noqa: W0212 +from weaver.processes.wps_package import _json2wps_datatype # noqa: W0212 +from weaver.processes.wps_package import _merge_io_formats # noqa: W0212 +from weaver.processes.wps_package import DEFAULT_FORMAT from weaver.utils import null diff --git a/tests/test_utils.py b/tests/test_utils.py index 1e09a9d6b..a1e4619ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -27,15 +27,8 @@ from tests.compat import contextlib from tests.utils import mocked_file_response from weaver import status, utils -from weaver.utils import ( - _NullType, # noqa: W0212 - null, - fetch_file, - get_ssl_verify_option, - get_request_options, - make_dirs, - request_extra -) +from weaver.utils import _NullType # noqa: W0212 +from weaver.utils import fetch_file, get_request_options, get_ssl_verify_option, make_dirs, null, request_extra def test_null_operators(): From 66f9db90691c4ea0f019e53cd4f563f9ea6b1ca6 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 22 May 2020 12:53:12 -0400 Subject: [PATCH 4/5] patch import --- requirements-dev.txt | 2 +- weaver/wps_restapi/api.py | 2 +- weaver/wps_restapi/colander_extras.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fe91ae450..1f5bf9030 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ contextlib2; python_version < "3" coverage doc8 flake8 -isort +isort>=4.3.21,<5 mock pluggy>=0.7 pytest diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index 8e9bb7b73..e7acc496a 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -3,6 +3,7 @@ import six from cornice.service import get_services +from cornice_swagger import CorniceSwagger from pyramid.authentication import Authenticated, IAuthenticationPolicy from pyramid.exceptions import PredicateMismatch from pyramid.httpexceptions import ( @@ -21,7 +22,6 @@ from simplejson import JSONDecodeError from six.moves.urllib.parse import urlparse -from cornice_swagger import CorniceSwagger from weaver.__meta__ import __version__ as weaver_version from weaver.formats import CONTENT_TYPE_APP_JSON, CONTENT_TYPE_TEXT_PLAIN from weaver.owsexceptions import OWSException diff --git a/weaver/wps_restapi/colander_extras.py b/weaver/wps_restapi/colander_extras.py index 2d50c8c4a..f26e8cc00 100644 --- a/weaver/wps_restapi/colander_extras.py +++ b/weaver/wps_restapi/colander_extras.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING import colander - from cornice_swagger.converters import schema from cornice_swagger.converters.exceptions import NoSuchConverter From 3176c5806b84a67c711331c3e203f84bbedf76f2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 22 May 2020 14:14:43 -0400 Subject: [PATCH 5/5] minor docstring update --- weaver/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/weaver/utils.py b/weaver/utils.py index 8b741b743..b16487ab5 100644 --- a/weaver/utils.py +++ b/weaver/utils.py @@ -635,6 +635,7 @@ def request_extra(method, # type: AnyStr :param retries: Number of request retries to attempt if first attempt failed (according to allowed codes or error). :param backoff: Factor by which to multiply delays between retries. :param intervals: Explicit intervals in seconds between retries. + :param retry_after: If enabled, honor ``Retry-After`` response header of provided by a failing request attempt. :param allowed_codes: HTTP status codes that are considered valid to stop retrying (default: any non-4xx/5xx code). :param ssl_verify: Explicit parameter to disable SSL verification (overrides any settings, default: True). :param settings: Additional settings from which to retrieve configuration details for requests.