From 50e71097929086beca7377ac6d909831a2ecb87c Mon Sep 17 00:00:00 2001 From: Gerald Walter Irsiegler Date: Tue, 5 Mar 2024 14:55:36 +0100 Subject: [PATCH] Fix: Fixed same level from_node resolving (#84) --- openeo_pg_parser_networkx/resolving_utils.py | 8 + pyproject.toml | 2 +- resolved_gfm_graph.json | 1 + .../data/res_tests/resolved/resolved_gfm.json | 51 +++++ tests/data/res_tests/udps/gfm.json | 196 ++++++++++++++++++ .../res_tests/unresolved/unresolved_gfm.json | 19 ++ tests/test_pg_resolving.py | 35 +++- 7 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 resolved_gfm_graph.json create mode 100644 tests/data/res_tests/resolved/resolved_gfm.json create mode 100644 tests/data/res_tests/udps/gfm.json create mode 100644 tests/data/res_tests/unresolved/unresolved_gfm.json diff --git a/openeo_pg_parser_networkx/resolving_utils.py b/openeo_pg_parser_networkx/resolving_utils.py index 465ff07..78d8ff8 100644 --- a/openeo_pg_parser_networkx/resolving_utils.py +++ b/openeo_pg_parser_networkx/resolving_utils.py @@ -185,6 +185,7 @@ def _fill_in_processes( _remap_names( process_graph=process_graph, process_replacement_id=process_replacement_id ) + _adjust_parameters( process_graph=process_graph, process_replacement_id=process_replacement_id, @@ -213,6 +214,13 @@ def _remap_names(process_graph, process_replacement_id): process_replacement_id ].pop(old_key) + for _, node in process_graph[process_replacement_id].items(): + for _, value in node['arguments'].items(): + if isinstance(value, dict) and 'from_node' in value.keys(): + value['from_node'] = next( + (t for t in name_remapping if t[1] == value['from_node']), None + )[0] + def _adjust_parameters(process_graph, process_replacement_id, arguments): ''' diff --git a/pyproject.toml b/pyproject.toml index 188a577..aef6198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openeo-pg-parser-networkx" -version = "2024.1.1" +version = "2024.3.1" description = "Parse OpenEO process graphs from JSON to traversible Python objects." authors = ["Lukas Weidenholzer ", "Sean Hoyal ", "Valentina Hutter ", "Gerald Irsiegler "] diff --git a/resolved_gfm_graph.json b/resolved_gfm_graph.json new file mode 100644 index 0000000..c3eebc4 --- /dev/null +++ b/resolved_gfm_graph.json @@ -0,0 +1 @@ +{"GFM_load1": {"process_id": "load_collection", "arguments": {"id": "GFM", "spatial_extent": {"west": 65.27044369351682, "east": 69.21281566288451, "south": 28.076233929760804, "north": 29.369117066086332}, "temporal_extent": ["2022-08-01T00:00:00Z", "2022-10-01T00:00:00Z"], "properties": {}}}, "GFM_reduce1": {"process_id": "reduce_dimension", "arguments": {"data": {"from_node": "GFM_load1"}, "reducer": {"process_graph": {"sum1": {"process_id": "sum", "arguments": {"data": {"from_parameter": "data"}}, "result": true}}}, "dimension": "time"}}, "GFM_save2": {"process_id": "save_result", "arguments": {"format": "GTIFF", "data": {"from_node": "GFM_reduce1"}}, "result": true}} diff --git a/tests/data/res_tests/resolved/resolved_gfm.json b/tests/data/res_tests/resolved/resolved_gfm.json new file mode 100644 index 0000000..af6ba82 --- /dev/null +++ b/tests/data/res_tests/resolved/resolved_gfm.json @@ -0,0 +1,51 @@ +{ + "GFM_load1": { + "process_id": "load_collection", + "arguments": { + "id": "GFM", + "spatial_extent": { + "west": 65.27044369351682, + "east": 69.21281566288451, + "south": 28.076233929760804, + "north": 29.369117066086332 + }, + "temporal_extent": [ + "2022-08-01T00:00:00Z", + "2022-10-01T00:00:00Z" + ], + "properties": {} + } + }, + "GFM_reduce1": { + "process_id": "reduce_dimension", + "arguments": { + "data": { + "from_node": "GFM_load1" + }, + "reducer": { + "process_graph": { + "sum1": { + "process_id": "sum", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": true + } + } + }, + "dimension": "time" + } + }, + "GFM_save2": { + "process_id": "save_result", + "arguments": { + "format": "GTIFF", + "data": { + "from_node": "GFM_reduce1" + } + }, + "result": true + } +} diff --git a/tests/data/res_tests/udps/gfm.json b/tests/data/res_tests/udps/gfm.json new file mode 100644 index 0000000..9d766a0 --- /dev/null +++ b/tests/data/res_tests/udps/gfm.json @@ -0,0 +1,196 @@ +{ + "id": "GFM", + "parameters": [ + { + "schema": { + "type": "array", + "subtype": "temporal-interval", + "title": "Single temporal interval", + "description": "Left-closed temporal interval, represented as two-element array with the following elements:\n\n1. The first element is the start of the temporal interval. The specified instance in time is **included** in the interval.\n2. The second element is the end of the temporal interval. The specified instance in time is **excluded** from the interval.\n\nThe specified temporal strings follow [RFC 3339](https://www.rfc-editor.org/rfc/rfc3339.html). Although [RFC 3339 prohibits the hour to be '24'](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.7), **this process allows the value '24' for the hour** of an end time in order to make it possible that left-closed time intervals can fully cover the day. `null` can be used to specify open intervals.", + "minItems": 2, + "maxItems": 2, + "items": { + "description": "Processes and implementations may choose to only implement a subset of the subtypes specified here. Clients must check what back-ends / processes actually support.", + "anyOf": [ + { + "type": "string", + "subtype": "date-time", + "format": "date-time", + "title": "Date with Time", + "description": "Date and time representation, as defined for `date-time` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6)." + }, + { + "type": "string", + "subtype": "date", + "format": "date", + "title": "Date only", + "description": "Date only representation, as defined for `full-date` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6). The time zone is UTC." + }, + { + "type": "string", + "subtype": "time", + "format": "time", + "title": "Time only", + "description": "Time only representation, as defined for `full-time` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6). Although [RFC 3339 prohibits the hour to be '24'](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.7), this definition allows the value '24' for the hour as end time in an interval in order to make it possible that left-closed time intervals can fully cover the day." + }, + { + "type": "string", + "subtype": "year", + "minLength": 4, + "maxLength": 4, + "pattern": "^\\d{4}$", + "title": "Year only", + "description": "Year representation, as defined for `date-fullyear` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6)." + }, + { + "type": "null" + } + ] + }, + "examples": [ + [ + "2015-01-01T00:00:00Z", + "2016-01-01T00:00:00Z" + ], + [ + "2015-01-01", + "2016-01-01" + ], + [ + "00:00:00Z", + "12:00:00Z" + ], + [ + "2015-01-01", + null + ] + ] + }, + "name": "temporal", + "description": "Scrediption" + }, + { + "schema": { + "type": "object", + "subtype": "bounding-box", + "title": "Bounding Box", + "description": "A bounding box with the required fields `west`, `south`, `east`, `north` and optionally `base`, `height`, `crs`. The `crs` is a EPSG code, a WKT2:2018 string or a PROJ definition (deprecated).", + "required": [ + "west", + "south", + "east", + "north" + ], + "properties": { + "west": { + "description": "West (lower left corner, coordinate axis 1).", + "type": "number" + }, + "south": { + "description": "South (lower left corner, coordinate axis 2).", + "type": "number" + }, + "east": { + "description": "East (upper right corner, coordinate axis 1).", + "type": "number" + }, + "north": { + "description": "North (upper right corner, coordinate axis 2).", + "type": "number" + }, + "base": { + "description": "Base (optional, lower left corner, coordinate axis 3).", + "type": [ + "number", + "null" + ] + }, + "height": { + "description": "Height (optional, upper right corner, coordinate axis 3).", + "type": [ + "number", + "null" + ] + }, + "crs": { + "description": "Coordinate reference system of the extent, specified as as [EPSG code](http://www.epsg-registry.org/), [WKT2 (ISO 19162) string](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) or [PROJ definition (deprecated)](https://proj.org/usage/quickstart.html). Defaults to `4326` (EPSG code 4326) unless the client explicitly requests a different coordinate reference system.", + "anyOf": [ + { + "type": "integer", + "subtype": "epsg-code", + "title": "EPSG Code", + "description": "Specifies details about cartographic projections as [EPSG](http://www.epsg.org) code.", + "minimum": 1000, + "examples": [ + 3857 + ] + }, + { + "type": "string", + "subtype": "wkt2-definition", + "title": "WKT2 definition", + "description": "Specifies details about cartographic projections as WKT2 string. Refers to the latest WKT2 version (currently [WKT2:2018](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) / ISO 19162:2018) unless otherwise stated by the process." + }, + { + "type": "string", + "subtype": "proj-definition", + "title": "PROJ definition", + "description": "**DEPRECATED.** Specifies details about cartographic projections as [PROJ](https://proj.org/usage/quickstart.html) definition." + } + ], + "default": 4326 + } + } + }, + "name": "spatial", + "description": "epatial sxtant" + } + ], + "process_graph": { + "load1": { + "process_id": "load_collection", + "arguments": { + "id": "GFM", + "spatial_extent": { + "from_parameter": "spatial" + }, + "temporal_extent": { + "from_parameter": "temporal" + }, + "properties": {} + } + }, + "reduce1": { + "process_id": "reduce_dimension", + "arguments": { + "data": { + "from_node": "load1" + }, + "reducer": { + "process_graph": { + "sum1": { + "process_id": "sum", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": true + } + } + }, + "dimension": "time" + } + }, + "save2": { + "process_id": "save_result", + "arguments": { + "format": "GTIFF", + "data": { + "from_node": "reduce1" + } + }, + "result": true + } + } +} diff --git a/tests/data/res_tests/unresolved/unresolved_gfm.json b/tests/data/res_tests/unresolved/unresolved_gfm.json new file mode 100644 index 0000000..2c63866 --- /dev/null +++ b/tests/data/res_tests/unresolved/unresolved_gfm.json @@ -0,0 +1,19 @@ +{ + "GFM": { + "process_id": "GFM", + "arguments": { + "temporal": [ + "2022-08-01T00:00:00Z", + "2022-10-01T00:00:00Z" + ], + "spatial": { + "west": 65.27044369351682, + "east": 69.21281566288451, + "south": 28.076233929760804, + "north": 29.369117066086332 + } + }, + "result": true, + "namespace": "user" + } +} diff --git a/tests/test_pg_resolving.py b/tests/test_pg_resolving.py index 6edd640..42cea47 100644 --- a/tests/test_pg_resolving.py +++ b/tests/test_pg_resolving.py @@ -7,6 +7,7 @@ def get_udp(process_id: str, namespace: str) -> dict: + process_id = process_id.lower() with open(f'tests/data/res_tests/udps/{process_id}.json') as f: return dict(json.load(f)) @@ -23,6 +24,8 @@ def get_predefined_process_registry(): ('apply', {}), ('load_collection', {}), ('save_result', {}), + ('sum', {}), + ('reduce_dimension', {}), ] for process_id, spec in predefined_processes_specs: @@ -34,7 +37,7 @@ def get_predefined_process_registry(): def get_full_process_registry() -> ProcessRegistry: full_process_registry = get_predefined_process_registry() - for udp in ['w_add', 'valid_load', 'nested_add']: + for udp in ['w_add', 'valid_load', 'nested_add', 'gfm']: full_process_registry['user', udp] = Process( get_udp(udp, "user"), implementation=None, namespace="user" ) @@ -58,13 +61,25 @@ def unresolved_pg() -> dict: return dict(json.loads(f.read())) +@pytest.fixture +def unresolved_gfm_pg() -> dict: + with open('tests/data/res_tests/unresolved/unresolved_gfm.json') as f: + return dict(json.loads(f.read())) + + @pytest.fixture def correctly_resolved_pg() -> dict: with open('tests/data/res_tests/resolved/resolved_complex.json') as f: return dict(json.loads(f.read())) -def test_resolve_graph_withpredefined_process_registr( +@pytest.fixture +def correctly_resolved_gfm_pg() -> dict: + with open('tests/data/res_tests/resolved/resolved_gfm.json') as f: + return dict(json.loads(f.read())) + + +def test_resolve_graph_with_predefined_process_registry( predefined_process_registry: ProcessRegistry, unresolved_pg: dict, correctly_resolved_pg: dict, @@ -132,3 +147,19 @@ def test_resolve_graph_with_none_get_udp_spec( process_registry=predefined_process_registry, get_udp_spec=lambda x, y: None, ) + + +def test_resolve_gfm_graph_with_predefined_process_registry( + predefined_process_registry: ProcessRegistry, + unresolved_gfm_pg: dict, + correctly_resolved_gfm_pg: dict, +): + resolved_pg = resolving_utils.resolve_process_graph( + process_graph=unresolved_gfm_pg, + process_registry=predefined_process_registry, + get_udp_spec=get_udp, + ) + + with open('resolved_gfm_graph.json', 'w') as f: + json.dump(resolved_pg, f) + assert correctly_resolved_gfm_pg == resolved_pg