diff --git a/CHANGES.rst b/CHANGES.rst index 5cee12b28..a45725689 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,7 +12,10 @@ Changes Changes: -------- -- No change. +- Support `CWL` definition for ``cwltool:CUDARequirement`` to request the use of a GPU, including support for using + Docker with a GPU (resolves `#104 `_). +- Support `CWL` definition for ``NetworkAccess`` to indicate whether a process requires outgoing IPv4/IPv6 network + access. Fixes: ------ diff --git a/tests/wps_restapi/test_processes.py b/tests/wps_restapi/test_processes.py index 203f808aa..eb57464ad 100644 --- a/tests/wps_restapi/test_processes.py +++ b/tests/wps_restapi/test_processes.py @@ -995,6 +995,87 @@ def test_deploy_process_CWL_DockerRequirement_executionUnit(self): "formats": [{"default": True, "mediaType": "text/plain"}] }] + def test_deploy_process_CWL_CudaRequirement_executionUnit(self): + with contextlib.ExitStack() as stack: + stack.enter_context(mocked_wps_output(self.settings)) + cuda_requirements = { + "cudaVersionMin": "11.4", + "cudaComputeCapability": "3.0", + "cudaDeviceCountMin": 1, + "cudaDeviceCountMax": 8 + } + docker_requirement = {"dockerPull": "python:3.7-alpine"} + cwl = { + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "hints": { + "cwltool:CUDARequirement": cuda_requirements, + "DockerRequirement": docker_requirement + }, + "$namespaces": { + "cwltool": "http://commonwl.org/cwltool#" + }, + "inputs": {}, + "outputs": { + "output": { + "type": "File", + "outputBinding": { + "glob": "stdout.log" + }, + } + } + } + + p_id = "test-cuda" + body = { + "processDescription": {"process": {"id": p_id}}, + "executionUnit": [{"unit": cwl}], + "deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication", + } + desc = self.deploy_process_make_visible_and_fetch_deployed(body, p_id, assert_io=False) + pkg = self.get_application_package(p_id) + assert desc["deploymentProfile"] == "http://www.opengis.net/profiles/eoc/dockerizedApplication" + assert desc["process"]["id"] == p_id + assert pkg["hints"]["cwltool:CUDARequirement"] == cuda_requirements + assert pkg["hints"]["DockerRequirement"] == docker_requirement + + def test_deploy_process_CWL_NetworkRequirement_executionUnit(self): + with contextlib.ExitStack() as stack: + stack.enter_context(mocked_wps_output(self.settings)) + network_access_requirement = {"networkAccess": True} + docker_requirement = {"dockerPull": "python:3.7-alpine"} + for type in ["hints", "requirements"]: + cwl = { + "class": "CommandLineTool", + "cwlVersion": "v1.2", + type: { + "NetworkAccess": network_access_requirement, + "DockerRequirement": docker_requirement + }, + "inputs": {}, + "outputs": { + "output": { + "type": "File", + "outputBinding": { + "glob": "stdout.log" + }, + } + } + } + + p_id = "test-network-access-" + type + body = { + "processDescription": {"process": {"id": p_id}}, + "executionUnit": [{"unit": cwl}], + "deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication", + } + desc = self.deploy_process_make_visible_and_fetch_deployed(body, p_id, assert_io=False) + pkg = self.get_application_package(p_id) + assert desc["deploymentProfile"] == "http://www.opengis.net/profiles/eoc/dockerizedApplication" + assert desc["process"]["id"] == p_id + assert pkg[type]["NetworkAccess"] == network_access_requirement + assert pkg[type]["DockerRequirement"] == docker_requirement + @mocked_remote_server_requests_wps1([ resources.TEST_REMOTE_SERVER_URL, resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML, diff --git a/weaver/processes/constants.py b/weaver/processes/constants.py index f3fb44e47..79911edf7 100644 --- a/weaver/processes/constants.py +++ b/weaver/processes/constants.py @@ -83,14 +83,18 @@ class OpenSearchField(Constants): """ # FIXME: convert to 'Constants' class +CWL_REQUIREMENT_CUDA = "cwltool:CUDARequirement" CWL_REQUIREMENT_ENV_VAR = "EnvVarRequirement" CWL_REQUIREMENT_INIT_WORKDIR = "InitialWorkDirRequirement" +CWL_REQUIREMENT_NETWORK_ACCESS = "NetworkAccess" CWL_REQUIREMENT_RESOURCE = "ResourceRequirement" CWL_REQUIREMENT_SCATTER = "ScatterFeatureRequirement" CWL_REQUIREMENT_FEATURES = frozenset([ + CWL_REQUIREMENT_CUDA, CWL_REQUIREMENT_ENV_VAR, CWL_REQUIREMENT_INIT_WORKDIR, + CWL_REQUIREMENT_NETWORK_ACCESS, CWL_REQUIREMENT_RESOURCE, # FIXME: perform pre-check on job submit? (https://github.com/crim-ca/weaver/issues/138) CWL_REQUIREMENT_SCATTER, ]) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index c46d804c6..0df845df3 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -35,7 +35,9 @@ CWL_REQUIREMENT_APP_ESGF_CWT, CWL_REQUIREMENT_APP_OGC_API, CWL_REQUIREMENT_APP_WPS1, + CWL_REQUIREMENT_CUDA, CWL_REQUIREMENT_INIT_WORKDIR, + CWL_REQUIREMENT_NETWORK_ACCESS, OAS_COMPLEX_TYPES, OAS_DATA_TYPES, PACKAGE_ARRAY_BASE, @@ -3613,6 +3615,70 @@ class RequirementClass(ExtendedSchemaNode): description = "CWL requirement class specification." +class CudaRequirementSpecification(PermissiveMappingSchema): + cudaVersionMin = ExtendedSchemaNode( + String(), + example="11.4", + title="Cuda version minimum", + description="The minimum Cuda version required.", + validator=SemanticVersion(regex=r"^\d+\.\d+$") + ) + cudaComputeCapability = ExtendedSchemaNode( + String(), + example="3.0", + title="Cuda compute capability", + description="The compute capability supported by the GPU.", + validator=SemanticVersion(regex=r"^\d+\.\d+$") + ) + cudaDeviceCountMin = ExtendedSchemaNode( + Integer(), + example=1, + default=1, + validator=Range(min=1), + title="Cuda device count minimum", + description="The minimum amount of devices required." + ) + cudaDeviceCountMax = ExtendedSchemaNode( + Integer(), + example=8, + default=1, + validator=Range(min=1), + title="Cuda device count maximum", + description="The maximum amount of devices required." + ) + + +class CudaRequirementMap(ExtendedMappingSchema): + CudaRequirement = CudaRequirementSpecification( + name=CWL_REQUIREMENT_CUDA, + title=CWL_REQUIREMENT_CUDA + ) + + +class CudaRequirementClass(CudaRequirementSpecification): + _class = RequirementClass(example=CWL_REQUIREMENT_CUDA, validator=OneOf([CWL_REQUIREMENT_CUDA])) + + +class NetworkAccessRequirementSpecification(PermissiveMappingSchema): + networkAccess = ExtendedSchemaNode( + Boolean(), + example=True, + title="Network Access", + description="Indicate whether a process requires outgoing IPv4/IPv6 network access." + ) + + +class NetworkAccessRequirementMap(ExtendedMappingSchema): + NetworkAccessRequirement = NetworkAccessRequirementSpecification( + name=CWL_REQUIREMENT_NETWORK_ACCESS, + title=CWL_REQUIREMENT_NETWORK_ACCESS + ) + + +class NetworkAccessRequirementClass(NetworkAccessRequirementSpecification): + _class = RequirementClass(example=CWL_REQUIREMENT_NETWORK_ACCESS, validator=OneOf([CWL_REQUIREMENT_NETWORK_ACCESS])) + + class DockerRequirementSpecification(PermissiveMappingSchema): dockerPull = ExtendedSchemaNode( String(), @@ -3752,6 +3818,7 @@ class CWLRequirementsMap(AnyOfKeywordSchema): DockerRequirementMap(missing=drop), DockerGpuRequirementMap(missing=drop), InitialWorkDirRequirementMap(missing=drop), + NetworkAccessRequirementMap(missing=drop), PermissiveMappingSchema(missing=drop), ] @@ -3761,6 +3828,7 @@ class CWLRequirementsItem(OneOfKeywordSchema): DockerRequirementClass(missing=drop), DockerGpuRequirementClass(missing=drop), InitialWorkDirRequirementClass(missing=drop), + NetworkAccessRequirementClass(missing=drop), UnknownRequirementClass(missing=drop), # allows anything, must be last ] @@ -3779,9 +3847,11 @@ class CWLRequirements(OneOfKeywordSchema): class CWLHintsMap(AnyOfKeywordSchema, PermissiveMappingSchema): _any_of = [ BuiltinRequirementMap(missing=drop), + CudaRequirementMap(missing=drop), DockerRequirementMap(missing=drop), DockerGpuRequirementMap(missing=drop), InitialWorkDirRequirementMap(missing=drop), + NetworkAccessRequirementMap(missing=drop), ESGF_CWT_RequirementMap(missing=drop), OGCAPIRequirementMap(missing=drop), WPS1RequirementMap(missing=drop), @@ -3794,9 +3864,11 @@ class CWLHintsItem(OneOfKeywordSchema, PermissiveMappingSchema): discriminator = "class" _one_of = [ BuiltinRequirementClass(missing=drop), + CudaRequirementClass(missing=drop), DockerRequirementClass(missing=drop), DockerGpuRequirementClass(missing=drop), InitialWorkDirRequirementClass(missing=drop), + NetworkAccessRequirementClass(missing=drop), ESGF_CWT_RequirementClass(missing=drop), OGCAPIRequirementClass(missing=drop), WPS1RequirementClass(missing=drop),