handle CWL Directory type

.github/workflows/tests.yml

@@ -68,7 +68,7 @@ jobs:
           - os: ubuntu-latest
             python-version: 3.7
             allow-failure: false
-            test-case: check-only
+            test-case: check-all
           # documentation build
           - os: ubuntu-latest
             python-version: 3.7

CHANGES.rst

@@ -12,6 +12,23 @@ Changes
 
 Changes:
 --------
+- Support `CWL` ``InlineJavascriptRequirement`` for `Process` deployment to allow successful schema validation.
+- Support `CWL` ``Directory`` type references (resolves `#466 <https://github.com/crim-ca/weaver/issues/466>`_).
+  Those references correspond to `WPS` and `OGC API - Processes` ``href``
+  using the ``Content-Type: application/directory`` Media-Type and must hava a trailing slash (``/``) character.
+- Support `S3` file or directory references using *Access Point*, *Virtual-hosted–style* and *Outposts* URLs
+  (see AWS documentation
+  `Methods for accessing a bucket <https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html>`_).
+- Apply more validation rules against expected `S3` file or directory reference formats.
+- Update documentation regarding handling of `S3` references (more formats supported) and ``Directory`` type references.
+- Support ``weaver.wps_output_context`` setting and ``X-WPS-Output-Context`` request header resolution in combination
+  with `S3` bucket location employed for storing `Job` outputs.
+- Nest every complex `Job` output (regardless if stored on local `WPS` outputs or on `S3`, and whether the output is
+  of ``File`` or ``Directory`` type) under its corresponding output ID collected from the `Process` definition to avoid
+  potential name conflicts in storage location, especially in the case of multiple output IDs that could be aggregated
+  with various files and listing of directory contents.
+- Allow ``colander.SchemaNode`` (with extensions for `OpenAPI` schema converters) to provide validation ``pattern``
+  field directly with a compiled ``re.Pattern`` object.
 - Support `CWL` definition for ``cwltool:CUDARequirement`` to request the use of a GPU, including support for using
   Docker with a GPU (resolves `#104 <https://github.com/crim-ca/weaver/issues/104>`_).
 - Support `CWL` definition for ``NetworkAccess`` to indicate whether a process requires outgoing IPv4/IPv6 network

Makefile

@@ -469,7 +469,7 @@ CHECKS := $(addprefix check-, $(CHECKS))
 # items that should not install python dev packages should be added here instead
 # they must provide their own target/only + with dependency install variants
 CHECKS_NO_PY := css md
-CHECKS_NO_PY := $(addprefix fix-, $(CHECKS_NO_PY))
+CHECKS_NO_PY := $(addprefix check-, $(CHECKS_NO_PY))
 CHECKS_ALL := $(CHECKS) $(CHECKS_NO_PY)
 
 $(CHECKS): check-%: install-dev check-%-only
@@ -482,7 +482,7 @@ mkdir-reports:
 check: check-all    ## alias for 'check-all' target
 
 .PHONY: check-only
-check-only: $(addsuffix -only, $(CHECKS))
+check-only: $(addsuffix -only, $(CHECKS_ALL))
 
 .PHONY: check-all
 check-all: install-dev $(CHECKS_ALL) 	## check all code linters

docs/_static/custom.css

@@ -48,3 +48,22 @@ div[class^="highlight"] {
     max-width: 100%;
     overflow: visible;
 }
+
+/* add missing border when row spans more than one line */
+.rst-content table.docutils td:first-child,
+.rst-content table.docutils th:first-child,
+.rst-content table.field-list td:first-child,
+.rst-content table.field-list th:first-child,
+.wy-table td:first-child,
+.wy-table th:first-child {
+    border-left-width: 1px;
+    border-right-width: 1px;
+}
+
+/* avoid mismatching background color in
+   table rows that spans multiple lines, due to
+   alternating colors on individual odd/even rows
+ */
+#table-file-type-handling tr.row-even > td[rowspan] {
+    background-color: revert;
+}

docs/examples/directory-listing-s3.json

@@ -0,0 +1,39 @@
+{
+  "ResponseMetadata": {
+    "RequestId": "vpiM5RBkJ3O68CnD5fO42d887Jh49Cf8bhA6nw7ZTHIuGRVccDQM",
+    "HTTPStatusCode": 200,
+    "HTTPHeaders": {
+      "x-amzn-requestid": "vpiM5RBkJ3O68CnD5fO42d887Jh49Cf8bhA6nw7ZTHIuGRVccDQM"
+    },
+    "RetryAttempts": 0
+  },
+  "IsTruncated": false,
+  "Contents": [
+    {
+      "Key": "dir/file.txt",
+      "LastModified": "2022-11-01T04:25:42+00:00",
+      "ETag": "\"17404a596cbd0d1e6c7d23fcd845ab82\"",
+      "Size": 4,
+      "StorageClass": "STANDARD"
+    },
+    {
+      "Key": "dir/sub/file.txt",
+      "LastModified": "2022-11-01T04:25:42+00:00",
+      "ETag": "\"17404a596cbd0d1e6c7d23fcd845ab82\"",
+      "Size": 4,
+      "StorageClass": "STANDARD"
+    },
+    {
+      "Key": "dir/sub/nested/file.txt",
+      "LastModified": "2022-11-01T04:25:42+00:00",
+      "ETag": "\"17404a596cbd0d1e6c7d23fcd845ab82\"",
+      "Size": 4,
+      "StorageClass": "STANDARD"
+    }
+  ],
+  "Name": "wps-process-test-bucket",
+  "Prefix": "dir/",
+  "MaxKeys": 1000,
+  "EncodingType": "url",
+  "KeyCount": 3
+}

docs/examples/directory-listing.html

@@ -0,0 +1,28 @@
+<html lang="en">
+<body>
+  <h1>Index of /dir/</h1>
+  <hr>
+  <table>
+  <thead>
+    <tr>
+      <th>Content</th>
+      <th>Modified</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><pre><a href="README">README</a></pre></td>
+      <td>2022-10-31 23:48</td>
+    </tr>
+    <tr>
+      <td><pre><a href="dir/">dir/</a></pre></td>
+      <td>2022-10-31 23:48</td></tr>
+    <tr>
+      <td><pre><a href="dir/file.txt">dir/file.txt</a></pre></td>
+      <td>2022-10-31 23:48</td>
+    </tr>
+  </tbody>
+  </table>
+  <hr>
+</body>
+</html>

docs/examples/directory-listing.json

@@ -0,0 +1,5 @@
+[
+  "https://example.com/base/dir/README.md",
+  "https://example.com/base/dir/nested/image.png",
+  "https://example.com/base/dir/nested/data.csv"
+]

docs/source/appendix.rst

@@ -19,11 +19,11 @@ Glossary
           queries in the context of :term:`EOImage` inputs.
 
     Application Package
-        General term that refers to *"what and how the :term:`Process` will execute"*. Application Packages provide
-        the core details about the execution methodology of the underlying operation the :term:`Process` provides, and
-        are therefore always contained within a :term:`Process` definition. This is more specifically represented
-        by a :term:`CWL` specification in the case of `Weaver` implementation, but could technically be defined by
-        another similar approach. See :ref:`Application Package` section for all relevant details.
+        General term that refers to *"what and how to execute"* the :term:`Process`. Application Packages provide the
+        core details about the execution methodology of the underlying operation that defines the :term:`Process`, and
+        are therefore always contained within a :ref:`Process Description <proc_op_describe>`. This is more specifically
+        represented by a :term:`CWL` specification in the case of `Weaver` implementation, but could technically be
+        defined by another similar approach. See the :ref:`Application Package` section for all relevant details.
 
     AWS
         Amazon Web Services

docs/source/cli.rst

@@ -161,7 +161,7 @@ A :term:`Workflow` of multiple :term:`Process` references (possibly of distinct
 .. note::
     Content definitions for :term:`CWL` :ref:`application-package` and/or the literal :term:`Process` body
     can be submitted using either a local file reference, an URL, or a literal string formatted as :term:`JSON`
-    or :temr:`YAML`. With the :ref:`Python Interface <client_commands>`, the definition can also be provided
+    or :term:`YAML`. With the :ref:`Python Interface <client_commands>`, the definition can also be provided
     with a :class:`dict` directly.
 
 Below is a sample :term:`Process` deployment using a basic Python script wrapped in a :term:`Docker` image to ensure