Issue #531 start using DynamicEtlApiJobCostCalculator in job tracker

CHANGELOG.md

@@ -12,10 +12,16 @@ without compromising stable operations.
 
 ## Unreleased
 
+
+## 0.24.0
+
+- Start using `DynamicEtlApiJobCostCalculator` in job tracker. Effective ETL API selection strategy is to be configured through `EtlApiConfig`
+
 ### Bugfix
 
  - added max_processing_area_pixels custom option to sar_backscatter, avoiding going out of memory when processing too large chunks
 
+
 ## 0.23.1
 
 ### Bugfix

openeogeotrellis/_version.py

@@ -1 +1 @@
-__version__ = "0.23.1a1"
+__version__ = "0.24.0a1"

openeogeotrellis/config/config.py

@@ -105,12 +105,13 @@ class GpsBackendConfig(OpenEoBackendConfig):
     ejr_credentials_vault_path: Optional[str] = os.environ.get("OPENEO_EJR_CREDENTIALS_VAULT_PATH")
 
     # TODO: eliminate hardcoded Terrascope references
+    # TODO #531 eliminate this config favor of etl_api_config strategy below
     etl_api: Optional[str] = os.environ.get("OPENEO_ETL_API", "https://etl.terrascope.be")
     etl_source_id: str = "TerraScope/MEP"
     use_etl_api_on_sync_processing: bool = False
-    etl_dynamic_api_flag: Optional[str] = None  # TODO eliminate this temporary feature flag?
+    etl_dynamic_api_flag: Optional[str] = None  # TODO #531 eliminate this temporary feature flag?
 
-    # TODO: this config is meant to replace `etl_api` from above
+    # TODO #531 this config is meant to replace `etl_api` from above
     etl_api_config: Optional[EtlApiConfig] = None
 
     prometheus_api: Optional[str] = os.environ.get("OPENEO_PROMETHEUS_API")

openeogeotrellis/job_costs_calculator.py

@@ -96,11 +96,14 @@ def calculate_costs(self, details: CostsDetails) -> float:
 
 class DynamicEtlApiJobCostCalculator(JobCostsCalculator):
     """
-    Like EtlApiJobCostsCalculator but with an ETL API endpoint that is determined based on user or job data
+    Like EtlApiJobCostsCalculator but with an ETL API endpoint that is determined based on user or job data.
+    It basically moves determining the ETL API url (`get_etl_api()`) from constructor time
+    to per-case `calculate_costs()` call time
     """
 
     def __init__(self, cache_ttl: int = 5 * 60):
         self._request_session = requests_with_retry(total=3, backoff_factor=2)
+        # Cache of `EtlApi` instances, used in `get_etl_api()`
         self._etl_cache: Optional[TtlCache] = TtlCache(default_ttl=cache_ttl) if cache_ttl > 0 else None
 
     def calculate_costs(self, details: CostsDetails) -> float:

openeogeotrellis/job_tracker_v2.py

@@ -50,6 +50,7 @@
     noJobCostsCalculator,
     CostsDetails,
     EtlApiJobCostsCalculator,
+    DynamicEtlApiJobCostCalculator,
 )
 from openeogeotrellis.job_registry import DoubleJobRegistry, ZkJobRegistry, get_deletable_dependency_sources
 from openeogeotrellis.utils import StatsReporter, dict_merge_recursive
@@ -636,8 +637,7 @@ def main(self, *, args: Optional[List[str]] = None):
                 else:
                     raise ValueError(app_cluster)
 
-                etl_api = get_etl_api(requests_session=requests_session)
-                job_costs_calculator = EtlApiJobCostsCalculator(etl_api=etl_api)
+                job_costs_calculator: JobCostsCalculator = DynamicEtlApiJobCostCalculator()
 
                 job_tracker = JobTracker(
                     app_state_getter=app_state_getter,