Skip to content

Commit

Permalink
Merge pull request #494 from crim-ca/add-cuda-requirements
Browse files Browse the repository at this point in the history 
Include additional cwl requirements
  • Loading branch information
@ChaamC
ChaamC committed Nov 18, 2022
2 parents ea77e5c + 3bd76e1 commit c39d815
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 1 deletion.
5 changes: 4 additions & 1 deletion CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ Changes

Changes:
--------
- No change.
- Support `CWL` definition for ``cwltool:CUDARequirement`` to request the use of a GPU, including support for using
Docker with a GPU (resolves `#104 <https://github.com/crim-ca/weaver/issues/104>`_).
- Support `CWL` definition for ``NetworkAccess`` to indicate whether a process requires outgoing IPv4/IPv6 network
access.

Fixes:
------
Expand Down
81 changes: 81 additions & 0 deletions tests/wps_restapi/test_processes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,87 @@ def test_deploy_process_CWL_DockerRequirement_executionUnit(self):
"formats": [{"default": True, "mediaType": "text/plain"}]
}]

def test_deploy_process_CWL_CudaRequirement_executionUnit(self):
with contextlib.ExitStack() as stack:
stack.enter_context(mocked_wps_output(self.settings))
cuda_requirements = {
"cudaVersionMin": "11.4",
"cudaComputeCapability": "3.0",
"cudaDeviceCountMin": 1,
"cudaDeviceCountMax": 8
}
docker_requirement = {"dockerPull": "python:3.7-alpine"}
cwl = {
"class": "CommandLineTool",
"cwlVersion": "v1.2",
"hints": {
"cwltool:CUDARequirement": cuda_requirements,
"DockerRequirement": docker_requirement
},
"$namespaces": {
"cwltool": "http://commonwl.org/cwltool#"
},
"inputs": {},
"outputs": {
"output": {
"type": "File",
"outputBinding": {
"glob": "stdout.log"
},
}
}
}

p_id = "test-cuda"
body = {
"processDescription": {"process": {"id": p_id}},
"executionUnit": [{"unit": cwl}],
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication",
}
desc = self.deploy_process_make_visible_and_fetch_deployed(body, p_id, assert_io=False)
pkg = self.get_application_package(p_id)
assert desc["deploymentProfile"] == "http://www.opengis.net/profiles/eoc/dockerizedApplication"
assert desc["process"]["id"] == p_id
assert pkg["hints"]["cwltool:CUDARequirement"] == cuda_requirements
assert pkg["hints"]["DockerRequirement"] == docker_requirement

def test_deploy_process_CWL_NetworkRequirement_executionUnit(self):
with contextlib.ExitStack() as stack:
stack.enter_context(mocked_wps_output(self.settings))
network_access_requirement = {"networkAccess": True}
docker_requirement = {"dockerPull": "python:3.7-alpine"}
for type in ["hints", "requirements"]:
cwl = {
"class": "CommandLineTool",
"cwlVersion": "v1.2",
type: {
"NetworkAccess": network_access_requirement,
"DockerRequirement": docker_requirement
},
"inputs": {},
"outputs": {
"output": {
"type": "File",
"outputBinding": {
"glob": "stdout.log"
},
}
}
}

p_id = "test-network-access-" + type
body = {
"processDescription": {"process": {"id": p_id}},
"executionUnit": [{"unit": cwl}],
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication",
}
desc = self.deploy_process_make_visible_and_fetch_deployed(body, p_id, assert_io=False)
pkg = self.get_application_package(p_id)
assert desc["deploymentProfile"] == "http://www.opengis.net/profiles/eoc/dockerizedApplication"
assert desc["process"]["id"] == p_id
assert pkg[type]["NetworkAccess"] == network_access_requirement
assert pkg[type]["DockerRequirement"] == docker_requirement

@mocked_remote_server_requests_wps1([
resources.TEST_REMOTE_SERVER_URL,
resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,
Expand Down
4 changes: 4 additions & 0 deletions weaver/processes/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,18 @@ class OpenSearchField(Constants):
"""

# FIXME: convert to 'Constants' class
CWL_REQUIREMENT_CUDA = "cwltool:CUDARequirement"
CWL_REQUIREMENT_ENV_VAR = "EnvVarRequirement"
CWL_REQUIREMENT_INIT_WORKDIR = "InitialWorkDirRequirement"
CWL_REQUIREMENT_NETWORK_ACCESS = "NetworkAccess"
CWL_REQUIREMENT_RESOURCE = "ResourceRequirement"
CWL_REQUIREMENT_SCATTER = "ScatterFeatureRequirement"

CWL_REQUIREMENT_FEATURES = frozenset([
CWL_REQUIREMENT_CUDA,
CWL_REQUIREMENT_ENV_VAR,
CWL_REQUIREMENT_INIT_WORKDIR,
CWL_REQUIREMENT_NETWORK_ACCESS,
CWL_REQUIREMENT_RESOURCE, # FIXME: perform pre-check on job submit? (https://github.com/crim-ca/weaver/issues/138)
CWL_REQUIREMENT_SCATTER,
])
Expand Down
72 changes: 72 additions & 0 deletions weaver/wps_restapi/swagger_definitions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@
CWL_REQUIREMENT_APP_ESGF_CWT,
CWL_REQUIREMENT_APP_OGC_API,
CWL_REQUIREMENT_APP_WPS1,
CWL_REQUIREMENT_CUDA,
CWL_REQUIREMENT_INIT_WORKDIR,
CWL_REQUIREMENT_NETWORK_ACCESS,
OAS_COMPLEX_TYPES,
OAS_DATA_TYPES,
PACKAGE_ARRAY_BASE,
Expand Down Expand Up @@ -3613,6 +3615,70 @@ class RequirementClass(ExtendedSchemaNode):
description = "CWL requirement class specification."


class CudaRequirementSpecification(PermissiveMappingSchema):
cudaVersionMin = ExtendedSchemaNode(
String(),
example="11.4",
title="Cuda version minimum",
description="The minimum Cuda version required.",
validator=SemanticVersion(regex=r"^\d+\.\d+$")
)
cudaComputeCapability = ExtendedSchemaNode(
String(),
example="3.0",
title="Cuda compute capability",
description="The compute capability supported by the GPU.",
validator=SemanticVersion(regex=r"^\d+\.\d+$")
)
cudaDeviceCountMin = ExtendedSchemaNode(
Integer(),
example=1,
default=1,
validator=Range(min=1),
title="Cuda device count minimum",
description="The minimum amount of devices required."
)
cudaDeviceCountMax = ExtendedSchemaNode(
Integer(),
example=8,
default=1,
validator=Range(min=1),
title="Cuda device count maximum",
description="The maximum amount of devices required."
)


class CudaRequirementMap(ExtendedMappingSchema):
CudaRequirement = CudaRequirementSpecification(
name=CWL_REQUIREMENT_CUDA,
title=CWL_REQUIREMENT_CUDA
)


class CudaRequirementClass(CudaRequirementSpecification):
_class = RequirementClass(example=CWL_REQUIREMENT_CUDA, validator=OneOf([CWL_REQUIREMENT_CUDA]))


class NetworkAccessRequirementSpecification(PermissiveMappingSchema):
networkAccess = ExtendedSchemaNode(
Boolean(),
example=True,
title="Network Access",
description="Indicate whether a process requires outgoing IPv4/IPv6 network access."
)


class NetworkAccessRequirementMap(ExtendedMappingSchema):
NetworkAccessRequirement = NetworkAccessRequirementSpecification(
name=CWL_REQUIREMENT_NETWORK_ACCESS,
title=CWL_REQUIREMENT_NETWORK_ACCESS
)


class NetworkAccessRequirementClass(NetworkAccessRequirementSpecification):
_class = RequirementClass(example=CWL_REQUIREMENT_NETWORK_ACCESS, validator=OneOf([CWL_REQUIREMENT_NETWORK_ACCESS]))


class DockerRequirementSpecification(PermissiveMappingSchema):
dockerPull = ExtendedSchemaNode(
String(),
Expand Down Expand Up @@ -3752,6 +3818,7 @@ class CWLRequirementsMap(AnyOfKeywordSchema):
DockerRequirementMap(missing=drop),
DockerGpuRequirementMap(missing=drop),
InitialWorkDirRequirementMap(missing=drop),
NetworkAccessRequirementMap(missing=drop),
PermissiveMappingSchema(missing=drop),
]

Expand All @@ -3761,6 +3828,7 @@ class CWLRequirementsItem(OneOfKeywordSchema):
DockerRequirementClass(missing=drop),
DockerGpuRequirementClass(missing=drop),
InitialWorkDirRequirementClass(missing=drop),
NetworkAccessRequirementClass(missing=drop),
UnknownRequirementClass(missing=drop), # allows anything, must be last
]

Expand All @@ -3779,9 +3847,11 @@ class CWLRequirements(OneOfKeywordSchema):
class CWLHintsMap(AnyOfKeywordSchema, PermissiveMappingSchema):
_any_of = [
BuiltinRequirementMap(missing=drop),
CudaRequirementMap(missing=drop),
DockerRequirementMap(missing=drop),
DockerGpuRequirementMap(missing=drop),
InitialWorkDirRequirementMap(missing=drop),
NetworkAccessRequirementMap(missing=drop),
ESGF_CWT_RequirementMap(missing=drop),
OGCAPIRequirementMap(missing=drop),
WPS1RequirementMap(missing=drop),
Expand All @@ -3794,9 +3864,11 @@ class CWLHintsItem(OneOfKeywordSchema, PermissiveMappingSchema):
discriminator = "class"
_one_of = [
BuiltinRequirementClass(missing=drop),
CudaRequirementClass(missing=drop),
DockerRequirementClass(missing=drop),
DockerGpuRequirementClass(missing=drop),
InitialWorkDirRequirementClass(missing=drop),
NetworkAccessRequirementClass(missing=drop),
ESGF_CWT_RequirementClass(missing=drop),
OGCAPIRequirementClass(missing=drop),
WPS1RequirementClass(missing=drop),
Expand Down

0 comments on commit c39d815

Please sign in to comment.