diff --git a/.gitignore b/.gitignore index 95d1336..9a55f92 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ wheels/ .installed.cfg *.egg + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. @@ -81,6 +82,7 @@ celerybeat-schedule # dotenv .env +*.env* # virtualenv .venv @@ -100,7 +102,7 @@ ENV/ # mypy .mypy_cache/ -cdk.out/package.zip +cdk.out/* *.tif # vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97a9e23..a53d436 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,38 +1,44 @@ repos: - - - repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black - args: ['--safe'] - language_version: python3.7 - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: flake8 - language_version: python3.7 - args: [ - # E501 let black handle all line length decisions - # W503 black conflicts with "line break before operator" rule - # E203 black conflicts with "whitespace before ':'" rule - '--ignore=E501,W503,E203'] - - - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle - # 2.1.1 - rev: 22d3ccf6cf91ffce3b16caa946c155778f0cb20f - hooks: - - id: pydocstyle - language_version: python3.7 - args: [ - # Check for docstring presence only - '--select=D1', - # Don't require docstrings for tests - '--match=(?!test).*\.py'] - - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.770 - hooks: - - id: mypy - language_version: python3.7 - args: [--no-strict-optional, --ignore-missing-imports] \ No newline at end of file + - repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black + language_version: python3 + args: ["--safe"] + + - repo: https://github.com/PyCQA/isort + rev: 5.4.2 + hooks: + - id: isort + language_version: python3 + + - repo: https://github.com/PyCQA/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + language_version: python3 + args: [ + # E501 let black handle all line length decisions + # W503 black conflicts with "line break before operator" rule + # E203 black conflicts with "whitespace before ':'" rule + "--ignore=E501,W503,E203", + ] + + - repo: https://github.com/PyCQA/pydocstyle + rev: 5.1.1 + hooks: + - id: pydocstyle + language_version: python3 + args: [ + # Check for docstring presence only + "--select=D1", + # Don't require docstrings for tests + '--match=(?!test).*\.py', + ] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.770 + hooks: + - id: mypy + language_version: python3 + args: ["--no-strict-optional", "--ignore-missing-imports"] diff --git a/covid_api/api/api_v1/api.py b/covid_api/api/api_v1/api.py index 8294a7f..56179f1 100644 --- a/covid_api/api/api_v1/api.py +++ b/covid_api/api/api_v1/api.py @@ -1,19 +1,18 @@ """covid_api api.""" -from fastapi import APIRouter - +from covid_api.api.api_v1.endpoints import detections, datasets # isort:skip from covid_api.api.api_v1.endpoints import ( - ogc, - detections, - timelapse, - datasets, - sites, groups, - tiles, metadata, + ogc, planet, + sites, + tiles, + timelapse, ) +from fastapi import APIRouter + api_router = APIRouter() api_router.include_router(tiles.router, tags=["tiles"]) api_router.include_router(metadata.router, tags=["metadata"]) diff --git a/covid_api/api/api_v1/endpoints/datasets.py b/covid_api/api/api_v1/endpoints/datasets.py index 82dee27..b2a32e2 100644 --- a/covid_api/api/api_v1/endpoints/datasets.py +++ b/covid_api/api/api_v1/endpoints/datasets.py @@ -1,11 +1,14 @@ """Dataset endpoints.""" +from covid_api.api import utils +from covid_api.core import config +from covid_api.db.memcache import CacheLayer +from covid_api.db.static.datasets import datasets from covid_api.db.static.errors import InvalidIdentifier -from fastapi import APIRouter, HTTPException, Depends, Response - from covid_api.models.static import Datasets -from covid_api.db.static.datasets import datasets -from covid_api.db.memcache import CacheLayer -from covid_api.api import utils + +from fastapi import APIRouter, Depends, HTTPException, Response + +from starlette.requests import Request router = APIRouter() @@ -16,6 +19,7 @@ response_model=Datasets, ) def get_datasets( + request: Request, response: Response, cache_client: CacheLayer = Depends(utils.get_cache), ): @@ -29,7 +33,13 @@ def get_datasets( content = Datasets.parse_raw(content) response.headers["X-Cache"] = "HIT" if not content: - content = datasets.get_all() + scheme = request.url.scheme + host = request.headers["host"] + if config.API_VERSION_STR: + host += config.API_VERSION_STR + + content = datasets.get_all(api_url=f"{scheme}://{host}") + if cache_client and content: cache_client.set_dataset_cache(dataset_hash, content) @@ -44,6 +54,7 @@ def get_datasets( response_model=Datasets, ) def get_dataset( + request: Request, spotlight_id: str, response: Response, cache_client: CacheLayer = Depends(utils.get_cache), @@ -59,7 +70,13 @@ def get_dataset( content = Datasets.parse_raw(content) response.headers["X-Cache"] = "HIT" if not content: - content = datasets.get(spotlight_id) + scheme = request.url.scheme + host = request.headers["host"] + if config.API_VERSION_STR: + host += config.API_VERSION_STR + + content = datasets.get(spotlight_id, api_url=f"{scheme}://{host}") + if cache_client and content: cache_client.set_dataset_cache(dataset_hash, content) diff --git a/covid_api/api/api_v1/endpoints/detections.py b/covid_api/api/api_v1/endpoints/detections.py index 0e3df29..075980a 100644 --- a/covid_api/api/api_v1/endpoints/detections.py +++ b/covid_api/api/api_v1/endpoints/detections.py @@ -1,12 +1,13 @@ """ Machine Learning Detections. """ -from fastapi import APIRouter, HTTPException import json from enum import Enum from covid_api.core import config -from covid_api.models.static import Detection -from covid_api.db.utils import s3_get from covid_api.db.static.sites import SiteNames +from covid_api.db.utils import s3_get +from covid_api.models.static import Detection + +from fastapi import APIRouter, HTTPException router = APIRouter() diff --git a/covid_api/api/api_v1/endpoints/groups.py b/covid_api/api/api_v1/endpoints/groups.py index 1d436a6..2995f09 100644 --- a/covid_api/api/api_v1/endpoints/groups.py +++ b/covid_api/api/api_v1/endpoints/groups.py @@ -1,9 +1,9 @@ """Groups endpoint.""" -from fastapi import APIRouter - -from covid_api.models.static import IndicatorGroup, IndicatorGroups from covid_api.db.static.groups import groups +from covid_api.models.static import IndicatorGroup, IndicatorGroups + +from fastapi import APIRouter router = APIRouter() diff --git a/covid_api/api/api_v1/endpoints/metadata.py b/covid_api/api/api_v1/endpoints/metadata.py index 50b180a..21f8413 100644 --- a/covid_api/api/api_v1/endpoints/metadata.py +++ b/covid_api/api/api_v1/endpoints/metadata.py @@ -1,25 +1,24 @@ """API metadata.""" -from typing import Any, Dict, Optional, Union - import os import re from functools import partial +from typing import Any, Dict, Optional, Union from urllib.parse import urlencode import numpy - from rio_tiler.io import cogeo -from fastapi import APIRouter, Query -from starlette.requests import Request -from starlette.responses import Response -from starlette.concurrency import run_in_threadpool - +from covid_api.api.utils import info as cogInfo from covid_api.core import config from covid_api.models.mapbox import TileJSON from covid_api.ressources.enums import ImageType -from covid_api.api.utils import info as cogInfo + +from fastapi import APIRouter, Query + +from starlette.concurrency import run_in_threadpool +from starlette.requests import Request +from starlette.responses import Response _info = partial(run_in_threadpool, cogInfo) _bounds = partial(run_in_threadpool, cogeo.bounds) diff --git a/covid_api/api/api_v1/endpoints/ogc.py b/covid_api/api/api_v1/endpoints/ogc.py index c722f14..7bb8ebe 100644 --- a/covid_api/api/api_v1/endpoints/ogc.py +++ b/covid_api/api/api_v1/endpoints/ogc.py @@ -2,21 +2,22 @@ from urllib.parse import urlencode -from fastapi import APIRouter, Query -from starlette.requests import Request -from starlette.responses import Response -from starlette.templating import Jinja2Templates - import rasterio from rasterio import warp -from rio_tiler.mercator import get_zooms from rio_tiler import constants +from rio_tiler.mercator import get_zooms from covid_api.core import config -from covid_api.ressources.enums import ImageType from covid_api.ressources.common import mimetype +from covid_api.ressources.enums import ImageType from covid_api.ressources.responses import XMLResponse +from fastapi import APIRouter, Query + +from starlette.requests import Request +from starlette.responses import Response +from starlette.templating import Jinja2Templates + router = APIRouter() templates = Jinja2Templates(directory="covid_api/templates") diff --git a/covid_api/api/api_v1/endpoints/planet.py b/covid_api/api/api_v1/endpoints/planet.py index 81597a9..1a65314 100644 --- a/covid_api/api/api_v1/endpoints/planet.py +++ b/covid_api/api/api_v1/endpoints/planet.py @@ -1,19 +1,18 @@ """API planet mosaic tiles.""" -from typing import Any, Dict - from functools import partial - -from fastapi import APIRouter, Depends, Query, Path -from starlette.concurrency import run_in_threadpool +from typing import Any, Dict from rio_tiler.utils import render from covid_api.api import utils from covid_api.db.memcache import CacheLayer -from covid_api.ressources.enums import ImageType from covid_api.ressources.common import mimetype +from covid_api.ressources.enums import ImageType from covid_api.ressources.responses import TileResponse +from fastapi import APIRouter, Depends, Path, Query + +from starlette.concurrency import run_in_threadpool _render = partial(run_in_threadpool, render) _tile = partial(run_in_threadpool, utils.planet_mosaic_tile) diff --git a/covid_api/api/api_v1/endpoints/sites.py b/covid_api/api/api_v1/endpoints/sites.py index 459676a..ead1d6f 100644 --- a/covid_api/api/api_v1/endpoints/sites.py +++ b/covid_api/api/api_v1/endpoints/sites.py @@ -1,9 +1,9 @@ """sites endpoint.""" -from fastapi import APIRouter +from covid_api.db.static.sites import SiteNames, sites +from covid_api.models.static import Site, Sites -from covid_api.models.static import Sites, Site -from covid_api.db.static.sites import sites, SiteNames +from fastapi import APIRouter router = APIRouter() diff --git a/covid_api/api/api_v1/endpoints/tiles.py b/covid_api/api/api_v1/endpoints/tiles.py index 5d6d461..d43f6e3 100644 --- a/covid_api/api/api_v1/endpoints/tiles.py +++ b/covid_api/api/api_v1/endpoints/tiles.py @@ -1,27 +1,25 @@ """API tiles.""" -from typing import Any, Dict, Union, Optional - import re -from io import BytesIO from functools import partial +from io import BytesIO +from typing import Any, Dict, Optional, Union import numpy - -from fastapi import APIRouter, Depends, Query, Path -from starlette.concurrency import run_in_threadpool - -from rio_tiler.io import cogeo from rio_tiler.colormap import get_colormap -from rio_tiler.utils import render, geotiff_options +from rio_tiler.io import cogeo from rio_tiler.profiles import img_profiles +from rio_tiler.utils import geotiff_options, render from covid_api.api import utils from covid_api.db.memcache import CacheLayer -from covid_api.ressources.enums import ImageType from covid_api.ressources.common import drivers, mimetype +from covid_api.ressources.enums import ImageType from covid_api.ressources.responses import TileResponse +from fastapi import APIRouter, Depends, Path, Query + +from starlette.concurrency import run_in_threadpool _tile = partial(run_in_threadpool, cogeo.tile) _render = partial(run_in_threadpool, render) diff --git a/covid_api/api/api_v1/endpoints/timelapse.py b/covid_api/api/api_v1/endpoints/timelapse.py index fda4274..d050b0c 100644 --- a/covid_api/api/api_v1/endpoints/timelapse.py +++ b/covid_api/api/api_v1/endpoints/timelapse.py @@ -1,9 +1,9 @@ """API metadata.""" -from fastapi import APIRouter - -from covid_api.models.timelapse import TimelapseValue, TimelapseRequest from covid_api.api.utils import get_zonal_stat +from covid_api.models.timelapse import TimelapseRequest, TimelapseValue + +from fastapi import APIRouter router = APIRouter() diff --git a/covid_api/api/utils.py b/covid_api/api/utils.py index 9c78575..79c86c4 100644 --- a/covid_api/api/utils.py +++ b/covid_api/api/utils.py @@ -1,41 +1,38 @@ """covid_api.api.utils.""" -from typing import Any, Dict, Tuple, Optional - -from enum import Enum -import re -import time -import json +import csv import hashlib +import json import math import random -import requests +import re +import time +from enum import Enum from io import BytesIO -import csv +from typing import Any, Dict, Optional, Tuple import numpy as np -from shapely.geometry import shape, box -from rasterstats.io import bounds_window - -from starlette.requests import Request # Temporary import rasterio +import requests from rasterio import features from rasterio.io import MemoryFile from rasterio.warp import transform_bounds -from rio_tiler import constants -from rio_tiler.utils import has_alpha_band, has_mask_band -from rio_tiler.mercator import get_zooms - +from rasterstats.io import bounds_window from rio_color.operations import parse_operations from rio_color.utils import scale_dtype, to_math_type -from rio_tiler.utils import linear_rescale, _chunks +from rio_tiler import constants +from rio_tiler.mercator import get_zooms +from rio_tiler.utils import _chunks, has_alpha_band, has_mask_band, linear_rescale +from shapely.geometry import box, shape +from covid_api.core.config import INDICATOR_BUCKET, PLANET_API_KEY from covid_api.db.memcache import CacheLayer from covid_api.db.utils import s3_get from covid_api.models.timelapse import Feature -from covid_api.core.config import PLANET_API_KEY, INDICATOR_BUCKET + +from starlette.requests import Request def get_cache(request: Request) -> CacheLayer: @@ -722,14 +719,25 @@ def site_date_to_scenes(site: str, date: str): """get the scenes corresponding to detections for a given site and date""" # TODO: make this more generic # NOTE: detections folder has been broken up into `detections-plane` and `detections-ship` - site_date_to_scenes_csv = s3_get( + plane_site_date_to_scenes_csv = s3_get( INDICATOR_BUCKET, "detections-plane/detection_scenes.csv" ) - site_date_lines = site_date_to_scenes_csv.decode("utf-8").split("\n") - reader = csv.DictReader(site_date_lines) - site_date_to_scenes_dict = dict() + plane_site_date_lines = plane_site_date_to_scenes_csv.decode("utf-8").split("\n") + + ship_site_date_to_scenes_csv = s3_get( + INDICATOR_BUCKET, "detections-ship/detection_scenes.csv" + ) + ship_site_date_lines = ship_site_date_to_scenes_csv.decode("utf-8").split("\n") + + reader = list(csv.DictReader(plane_site_date_lines)) + reader.extend(list(csv.DictReader(ship_site_date_lines))) + + site_date_to_scenes_dict: dict = {} + for row in reader: - site_date_to_scenes_dict[f'{row["aoi"]}-{row["date"]}'] = row[ - "scene_id" - ].replace("'", '"') - return json.loads(site_date_to_scenes_dict[f"{site}-{date}"]) + + site_date_to_scenes_dict.setdefault(f'{row["aoi"]}-{row["date"]}', []).extend( + json.loads(row["scene_id"].replace("'", '"')) + ) + + return site_date_to_scenes_dict[f"{site}-{date}"] diff --git a/covid_api/core/config.py b/covid_api/core/config.py index 02559cb..fb9bad2 100644 --- a/covid_api/core/config.py +++ b/covid_api/core/config.py @@ -2,7 +2,6 @@ import os - API_VERSION_STR = "/v1" PROJECT_NAME = "covid_api" diff --git a/covid_api/db/memcache.py b/covid_api/db/memcache.py index 3fbb3f1..3146e7a 100644 --- a/covid_api/db/memcache.py +++ b/covid_api/db/memcache.py @@ -1,11 +1,11 @@ """covid_api.cache.memcache: memcached layer.""" -from typing import Optional, Tuple, Dict, Union +from typing import Dict, Optional, Tuple, Union from bmemcached import Client -from covid_api.ressources.enums import ImageType from covid_api.models.static import Datasets +from covid_api.ressources.enums import ImageType class CacheLayer(object): diff --git a/covid_api/db/static/datasets/__init__.py b/covid_api/db/static/datasets/__init__.py index b1426cc..97be1e4 100644 --- a/covid_api/db/static/datasets/__init__.py +++ b/covid_api/db/static/datasets/__init__.py @@ -1,14 +1,13 @@ """ covid_api static datasets """ import os import re -from typing import List, Dict, Set, Any +from copy import deepcopy +from typing import Any, Dict, List, Set -from covid_api.models.static import Datasets, Dataset from covid_api.db.static.errors import InvalidIdentifier - from covid_api.db.static.sites import sites - -from covid_api.db.utils import get_dataset_folders_by_spotlight, get_dataset_domain +from covid_api.db.utils import get_dataset_domain, get_dataset_folders_by_spotlight +from covid_api.models.static import DatasetInternal, Datasets, GeoJsonSource data_dir = os.path.join(os.path.dirname(__file__)) @@ -23,11 +22,13 @@ def __init__(self): ] self._data = { - dataset: Dataset.parse_file(os.path.join(data_dir, f"{dataset}.json")) + dataset: DatasetInternal.parse_file( + os.path.join(data_dir, f"{dataset}.json") + ) for dataset in datasets } - def get(self, spotlight_id: str) -> Datasets: + def get(self, spotlight_id: str, api_url: str) -> Datasets: """ Fetches all the datasets avilable for a given spotlight. If the spotlight_id provided is "global" then this method will return @@ -35,15 +36,11 @@ def get(self, spotlight_id: str) -> Datasets: `InvalidIdentifier` exception if the provided spotlight_id does not exist. """ - - global_datasets = self.get_global_datasets() + global_datasets = self._get_global_datasets() global_datasets = self._overload_domain(datasets=global_datasets) if spotlight_id == "global": - - return Datasets( - datasets=[dataset.dict() for dataset in global_datasets.values()] - ) + return self._prep_output(global_datasets, api_url=api_url) # Verify that the requested spotlight exists try: @@ -51,43 +48,135 @@ def get(self, spotlight_id: str) -> Datasets: except InvalidIdentifier: raise - # Append EUPorts to the spotlight ID using a pipe character so that - # the regexp will filter for either value, since certain datasets - # contain data for both the `du` and `gh` spotlights under `EUPorts` + # Append "EUPorts" to the spotlight ID's if the requested spotlight id + # was one of "du" or "gh", since certain datasets group both spotlights + # under a single value: "EUPorts". It's then necessary to search, + # and extract domain for each option ("du"/"gh" and "EUPorts") separately + + spotlight_ids = [site.id] if site.id in ["du", "gh"]: - site.id = f"{site.id}|EUPorts" + spotlight_ids.append("EUPorts") - # find all "folders" in S3 containing keys for the given spotlight - # each "folder" corresponds to a dataset. - spotlight_dataset_folders = get_dataset_folders_by_spotlight( - spotlight_id=site.id - ) + spotlight_datasets = {} - # filter the dataset items by those corresponding the folders above - # and add the datasets to the previously filtered `global` datasets - spotlight_datasets = self.filter_datasets_by_folders( - folders=spotlight_dataset_folders - ) - spotlight_datasets = self._overload_domain( - datasets=spotlight_datasets, spotlight_id=site.id - ) + for spotlight_id in spotlight_ids: + # find all "folders" in S3 containing keys for the given spotlight + # each "folder" corresponds to a dataset. + spotlight_dataset_folders = get_dataset_folders_by_spotlight( + spotlight_id=spotlight_id + ) + # filter the dataset items by those corresponding the folders above + # and add the datasets to the previously filtered `global` datasets + datasets = self._filter_datasets_by_folders( + folders=spotlight_dataset_folders + ) + + datasets = self._overload_spotlight_id( + datasets=datasets, spotlight_id=spotlight_id + ) + + datasets = self._overload_domain( + datasets=datasets, spotlight_id=spotlight_id + ) + spotlight_datasets.update(datasets) + + if spotlight_id == "tk": + spotlight_datasets["water-chlorophyll"].source.tiles = [ + tile.replace("&rescale=-100%2C100", "") + for tile in spotlight_datasets["water-chlorophyll"].source.tiles + ] # global datasets are returned for all spotlights spotlight_datasets.update(global_datasets) - return Datasets( - datasets=[dataset.dict() for dataset in spotlight_datasets.values()] - ) + return self._prep_output(spotlight_datasets, api_url=api_url) - def get_all(self) -> Datasets: + def get_all(self, api_url: str) -> Datasets: """Fetch all Datasets. Overload domain with S3 scanned domain""" self._data = self._overload_domain(datasets=self._data) - return Datasets(datasets=[dataset.dict() for dataset in self._data.values()]) + return self._prep_output(self._data, api_url=api_url) def list(self) -> List[str]: """List all datasets""" return list(self._data.keys()) + def _prep_output(self, output_datasets: dict, api_url: str): + """ + Replaces the `source` of the detections-* datasets with geojson data types and + inserts the url base of the source tile url. + The deepcopy of the the data to output is necessary to avoid modifying the + underlying objects, which would affect the result of subsequent API calls. + + Params: + ------- + output_datasets (dict): Dataset metadata objects to return to API consumer. + api_url (str): + Base url, of the form {schema}://{host}, extracted from the request, to + prepend all tile source urls with. + + Returns: + -------- + (dict) : datasets metadata object, ready to return to the API consumer + """ + output_datasets = deepcopy(output_datasets) + for dataset in output_datasets.values(): + dataset.source.tiles = [ + tile.replace("{api_url}", api_url) for tile in dataset.source.tiles + ] + + if dataset.background_source: + dataset.background_source.tiles = [ + tile.replace("{api_url}", api_url) + for tile in dataset.background_source.tiles + ] + if dataset.compare: + dataset.compare.source.tiles = [ + tile.replace("{api_url}", api_url) + for tile in dataset.compare.source.tiles + ] + if dataset.id in ["detections-ship", "detections-plane"]: + dataset.source = GeoJsonSource( + type=dataset.source.type, data=dataset.source.tiles[0] + ) + + return Datasets( + datasets=[dataset.dict() for dataset in output_datasets.values()] + ) + + @staticmethod + def _overload_spotlight_id(datasets: dict, spotlight_id: str): + """ + Returns the provided `datasets` objects with an updated value for + each dataset's `source.tiles` and `background_source.tiles` keys. + The string "{spotlightId}" in the `tiles` URL(s) is replaced with the + actual value of the spotlightId (eg: "ny", "sf", "tk") + Params: + ------ + datasets (dict): dataset metadata objects for which to overload + `source.tiles` and `background_source.tiles` keys. + spotlight_id ([dict]): spotlight id value with which to replace + "{spotlightId}" + + Returns: + ------ + dict: the `datasets` object, with an updated `source.tiles` and + `background_source.tiles` values for each dataset in the `datasets` object. + """ + for _, dataset in datasets.items(): + dataset.source.tiles = [ + url.replace("{spotlightId}", spotlight_id) + for url in dataset.source.tiles + ] + + if not dataset.background_source: + continue + + dataset.background_source.tiles = [ + url.replace("{spotlightId}", spotlight_id) + for url in dataset.background_source.tiles + ] + return datasets + @staticmethod def _overload_domain(datasets: dict, spotlight_id: str = None): """ @@ -101,7 +190,7 @@ def _overload_domain(datasets: dict, spotlight_id: str = None): ------ datasets (dict): dataset metadata objects for which to overload `domain` keys. - spotlight (Optional[dict]): spotlight to further precise `domain` + spotlight_id (Optional[str]): spotlight_id to further precise `domain` search Returns: @@ -128,7 +217,7 @@ def _overload_domain(datasets: dict, spotlight_id: str = None): return datasets - def filter_datasets_by_folders(self, folders: Set[str]) -> Dict: + def _filter_datasets_by_folders(self, folders: Set[str]) -> Dict: """ Returns all datasets corresponding to a set of folders (eg: for folders {"BMHD_30M_MONTHLY", "xco2"} this method would return the @@ -143,10 +232,14 @@ def filter_datasets_by_folders(self, folders: Set[str]) -> Dict: Dict : Metadata objects for the datasets corresponding to the folders provided. """ + # deepcopy is necessary because the spotlight and domain overriding was + # affecting the original dataset metadata items and returning the same values + # in subsequent API requests for different spotlights + return { + k: v for k, v in deepcopy(self._data).items() if v.s3_location in folders + } - return {k: v for k, v in self._data.items() if v.s3_location in folders} - - def get_global_datasets(self): + def _get_global_datasets(self): """ Returns all datasets which do not reference a specific spotlight, by filtering out datasets where the "source.tiles" value contains either diff --git a/covid_api/db/static/datasets/agriculture.json b/covid_api/db/static/datasets/agriculture.json index 9213cb4..e149122 100644 --- a/covid_api/db/static/datasets/agriculture.json +++ b/covid_api/db/static/datasets/agriculture.json @@ -3,11 +3,13 @@ "name": "Agriculture", "type": "raster-timeseries", "s3_location": "agriculture-cropmonitor", + "info": "", "time_unit": "month", + "is_periodic": true, "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/agriculture-cropmonitor/CropMonitor_{date}.tif&resampling_method=nearest&bidx=1&color_map=custom_cropmonitor" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/agriculture-cropmonitor/CropMonitor_{date}.tif&resampling_method=nearest&bidx=1&color_map=custom_cropmonitor" ] }, "exclusive_with": [ diff --git a/covid_api/db/static/datasets/co2-diff.json b/covid_api/db/static/datasets/co2-diff.json index cc900ff..56a4623 100644 --- a/covid_api/db/static/datasets/co2-diff.json +++ b/covid_api/db/static/datasets/co2-diff.json @@ -1,18 +1,20 @@ { "id": "co2-diff", "name": "Carbon Dioxide (Diff)", - "description": "This layer shows changes in carbon dioxide (CO₂) levels during coronavirus lockdowns versus previous years. Redder colors indicate increases in CO₂. Bluer colors indicate lower levels of CO₂.", "type": "raster-timeseries", "time_unit": "day", + "is_periodic": true, "s3_location": "xco2-diff", "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2-diff/xco2_15day_diff.{date}.tif&resampling_method=bilinear&bidx=1&rescale=-0.000001%2C0.000001&color_map=rdbu_r" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2-diff/xco2_16day_diff.{date}.tif&resampling_method=bilinear&bidx=1&rescale=-0.000001%2C0.000001&color_map=rdbu_r" ] }, "exclusive_with": [ + "agriculture", "no2", + "no2-diff", "co2", "gibs-population", "car-count", @@ -24,7 +26,7 @@ "detections-ship", "detections-plane" ], - "enabled": true, + "enabled": false, "swatch": { "color": "#189C54", "name": "Dark Green" diff --git a/covid_api/db/static/datasets/co2.json b/covid_api/db/static/datasets/co2.json index 8e678d9..41aeb85 100644 --- a/covid_api/db/static/datasets/co2.json +++ b/covid_api/db/static/datasets/co2.json @@ -3,15 +3,16 @@ "name": "Carbox Dioxide (Avg)", "type": "raster-timeseries", "time_unit": "day", - "description": "This layer shows the average background concentration of carbon dioxide (CO₂) in our atmosphere for 2020. Redder colors indicate more CO₂. Whiter colors indicate less CO₂.", "s3_location": "xco2-mean", + "is_periodic": true, "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2-mean/xco2_15day_mean.{date}.tif&resampling_method=bilinear&bidx=1&rescale=0.000408%2C0.000419&color_map=rdylbu_r&color_formula=gamma r {gamma}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2-mean/xco2_16day_mean.{date}.tif&resampling_method=bilinear&bidx=1&rescale=0.000408%2C0.000419&color_map=rdylbu_r&color_formula=gamma r {gamma}" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "gibs-population", @@ -33,7 +34,7 @@ "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1//{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2/xco2_15day_base.{date}.tif&resampling_method=bilinear&bidx=1&rescale=0.000408%2C0.000419&color_map=rdylbu_r&color_formula=gamma r {gamma}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/xco2/xco2_16day_base.{date}.tif&resampling_method=bilinear&bidx=1&rescale=0.000408%2C0.000419&color_map=rdylbu_r&color_formula=gamma r {gamma}" ] } }, @@ -46,13 +47,13 @@ "min": "< 408 ppm", "max": "> 419 ppm", "stops": [ - "#5D4FA2", - "#2F75BE", - "#6DC7A3", - "#D7ED96", - "#FFEA9B", - "#FA894C", - "#B11E4D" + "#313695", + "#588cbf", + "#a3d2e5", + "#e8f6e8", + "#fee89c", + "#fba55c", + "#e24932" ] }, "info": "This layer shows the average background concentration of carbon dioxide (CO₂) in our atmosphere for 2020. Redder colors indicate more CO₂. Whiter colors indicate less CO₂." diff --git a/covid_api/db/static/datasets/detections-plane.json b/covid_api/db/static/datasets/detections-plane.json index f10fe02..9831d30 100644 --- a/covid_api/db/static/datasets/detections-plane.json +++ b/covid_api/db/static/datasets/detections-plane.json @@ -2,21 +2,23 @@ "id": "detections-plane", "name": "Plane", "type": "inference-timeseries", - "description": "Planes detected each day in PlanetScope imagery are shown in red.", "s3_location": "detections-plane", + "is_periodic": false, + "time_unit": "day", "source": { "type": "geojson", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/detections/plane/{spotlightId}/{date}.geojson" + "{api_url}/detections/plane/{spotlightId}/{date}.geojson" ] }, "background_source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/planet/{spotlightId}-{date}.tif&resampling_method=nearest&bidx=1,2,3" + "{api_url}/planet/{z}/{x}/{y}?date={date}&site={spotlightId}" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/detections-ship.json b/covid_api/db/static/datasets/detections-ship.json index 8355d6f..3637b60 100644 --- a/covid_api/db/static/datasets/detections-ship.json +++ b/covid_api/db/static/datasets/detections-ship.json @@ -2,23 +2,23 @@ "id": "detections-ship", "name": "Shipping", "type": "inference-timeseries", - "description": "Ships detected each day in PlanetScope imagery are shown in red.", "s3_location": "detections-ship", "is_periodic": false, "time_unit": "day", "source": { "type": "geojson", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/detections/ship/{spotlightId}/{date}.geojson" + "{api_url}/detections/ship/{spotlightId}/{date}.geojson" ] }, "background_source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/planet/{spotlightId}-{date}.tif&resampling_method=nearest&bidx=1,2,3" + "{api_url}/planet/{z}/{x}/{y}?date={date}&site={spotlightId}" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/nightlights-hd.json b/covid_api/db/static/datasets/nightlights-hd.json index faa0d0d..a44d25f 100644 --- a/covid_api/db/static/datasets/nightlights-hd.json +++ b/covid_api/db/static/datasets/nightlights-hd.json @@ -2,16 +2,17 @@ "id": "nightlights-hd", "name": "Nightlights HD", "type": "raster-timeseries", - "description": "The High Definition Nightlights dataset is processed to eliminate light sources, including moonlight reflectance and other interferences. Darker colors indicate fewer night lights and less activity. Lighter colors indicate more night lights and more activity.", - "s3_location": "BMHD_30M_MONTHLY", + "s3_location": "bmhd_30m_monthly", + "is_periodic": true, "time_unit": "month", "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/BMHD_30M_MONTHLY/BMHD_VNP46A2_{spotlightId}_{date}_cog.tif&resampling_method=bilinear&bidx=1%2C2%2C3" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/bmhd_30m_monthly/BMHD_VNP46A2_{spotlightId}_{date}_cog.tif&resampling_method=bilinear&bidx=1%2C2%2C3" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/nightlights-viirs.json b/covid_api/db/static/datasets/nightlights-viirs.json index 6ed8f6a..51e0d67 100644 --- a/covid_api/db/static/datasets/nightlights-viirs.json +++ b/covid_api/db/static/datasets/nightlights-viirs.json @@ -3,15 +3,16 @@ "name": "Nightlights VIIRS", "type": "raster-timeseries", "time_unit": "day", - "description": "The High Definition Nightlights dataset is processed to eliminate light sources, including moonlight reflectance and other interferences. Darker colors indicate fewer night lights and less activity. Lighter colors indicate more night lights and more activity.", - "s3_location": "BM_500M_DAILY", + "s3_location": "bm_500m_daily", + "is_periodic": true, "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/BM_500M_DAILY/VNP46A2_V011_{spotlightId}_{date}_cog.tif&resampling_method=nearest&bidx=1&rescale=0%2C100&color_map=viridis" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/bm_500m_daily/VNP46A2_V011_{spotlightId}_{date}_cog.tif&resampling_method=nearest&bidx=1&rescale=0%2C100&color_map=viridis" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/no2-diff.json b/covid_api/db/static/datasets/no2-diff.json index 42d37ac..459e617 100644 --- a/covid_api/db/static/datasets/no2-diff.json +++ b/covid_api/db/static/datasets/no2-diff.json @@ -2,16 +2,16 @@ "id": "no2-diff", "name": "NO\u2082 (Diff)", "type": "raster-timeseries", - "domain": [], "time_unit": "month", + "is_periodic": false, "s3_location": "OMNO2d_HRMDifference", "source": { "type": "raster", "tiles": [ - "${config.api}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRMDifference/OMI_trno2_0.10x0.10_{date}_Col3_V4.nc.tif&resampling_method=bilinear&bidx=1&rescale=-8000000000000000%2C8000000000000000&color_map=rdbu_r&color_formula=gamma r {gamma}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRMDifference/OMI_trno2_0.10x0.10_{date}_Col3_V4.nc.tif&resampling_method=bilinear&bidx=1&rescale=-8000000000000000%2C8000000000000000&color_map=rdbu_r&color_formula=gamma r {gamma}" ] }, - "exclusiveWith": [ + "exclusive_with": [ "co2", "co2-diff", "gibs-population", @@ -30,9 +30,9 @@ "name": "Gold" }, "legend": { - "type": "gradient-adjustable", - "min": "less", - "max": "more", + "type": "gradient", + "min": "< -3", + "max": "> 3", "stops": [ "#3A88BD", "#C9E0ED", diff --git a/covid_api/db/static/datasets/no2.json b/covid_api/db/static/datasets/no2.json index b600cd1..387e8ea 100644 --- a/covid_api/db/static/datasets/no2.json +++ b/covid_api/db/static/datasets/no2.json @@ -4,10 +4,11 @@ "type": "raster-timeseries", "s3_location": "OMNO2d_HRM", "time_unit": "month", + "is_periodic": true, "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRM/OMI_trno2_0.10x0.10_{date}_Col3_V4.nc.tif&resampling_method=bilinear&bidx=1&rescale=0%2C1.5e16&color_map=custom_no2&color_formula=gamma r {gamma}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRM/OMI_trno2_0.10x0.10_{date}_Col3_V4.nc.tif&resampling_method=bilinear&bidx=1&rescale=0%2C1.5e16&color_map=custom_no2&color_formula=gamma r {gamma}" ] }, "exclusive_with": [ @@ -29,12 +30,12 @@ "enabled": true, "help": "Compare with baseline (5 previous years)", "map_label": "{date}: Base vs Mean", - "year_Diff": 2, + "year_diff": 2, "time_unit": "month_only", "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRMBaseline/OMI_trno2_0.10x0.10_Baseline_{date}_Col3_V4.nc.tif&esampling_method=bilinear&bidx=1&rescale=0%2C1.5e16&color_map=custom_no2&color_formula=gamma r {gamma}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/OMNO2d_HRMBaseline/OMI_trno2_0.10x0.10_Baseline_{date}_Col3_V4.nc.tif&esampling_method=bilinear&bidx=1&rescale=0%2C1.5e16&color_map=custom_no2&color_formula=gamma r {gamma}" ] } }, diff --git a/covid_api/db/static/datasets/population.json b/covid_api/db/static/datasets/population.json index e769dd2..c64cd5f 100644 --- a/covid_api/db/static/datasets/population.json +++ b/covid_api/db/static/datasets/population.json @@ -2,12 +2,27 @@ "id": "population", "name": "Population", "type": "raster", + "time_unit": "day", "source": { "type": "raster", "tiles": [ "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/GPW_Population_Density_2020/default/2020-05-14T00:00:00Z/GoogleMapsCompatible_Level7/{z}/{y}/{x}.png" ] }, + "exclusive_with": [ + "agriculture", + "no2", + "no2-diff", + "co2-diff", + "co2", + "car-count", + "nightlights-viirs", + "nightlights-hd", + "detection-ship", + "detection-multi", + "water-chlorophyll", + "water-spm" + ], "swatch": { "color": "#C0C0C0", "name": "Grey" diff --git a/covid_api/db/static/datasets/recovery.json b/covid_api/db/static/datasets/recovery.json new file mode 100644 index 0000000..a6a6719 --- /dev/null +++ b/covid_api/db/static/datasets/recovery.json @@ -0,0 +1,35 @@ +{ + "id": "recovery", + "name": "Recovery Proxy Map", + "description": "Recovery Proxy Maps show areas with the greatest increase in car activity shaded in orange. Darker orange indicates areas of greater change.", + "type": "raster", + "s3_location": "rpm", + "source": { + "type": "raster", + "tiles": [ + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/rpm/rpm-{spotlightId}.cog.tif&resampling_method=bilinear&bidx=1%2C2%2C3%24" + ] + }, + "paint": { + "raster_opacity": 0.9 + }, + "exclusiveWith": [ + "agriculture", + "co2", + "co2-diff", + "gibs-population", + "car-count", + "nightlights-viirs", + "nightlights-hd", + "detection-ship", + "detection-multi", + "water-chlorophyll", + "water-spm" + ], + "enabled": true, + "swatch": { + "color": "#C0C0C0", + "name": "Grey" + }, + "info": "Recovery Proxy Maps show areas with the greatest increase in car activity shaded in orange. Darker orange indicates areas of greater change." +} \ No newline at end of file diff --git a/covid_api/db/static/datasets/slowdown.json b/covid_api/db/static/datasets/slowdown.json new file mode 100644 index 0000000..1e4df50 --- /dev/null +++ b/covid_api/db/static/datasets/slowdown.json @@ -0,0 +1,34 @@ +{ + "id": "slowdown", + "name": "Slowdown Proxy Maps", + "type": "raster", + "s3_location": "slowdown_proxy_map", + "source": { + "type": "raster", + "tiles": [ + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/slowdown_proxy_map/{spotlightId}.tif&resampling_method=bilinear&bidx=1%2C2%2C3%24" + ] + }, + "paint": { + "raster_opacity": 0.9 + }, + "exclusiveWith": [ + "agriculture", + "co2", + "co2-diff", + "gibs-population", + "car-count", + "nightlights-viirs", + "nightlights-hd", + "detection-ship", + "detection-multi", + "water-chlorophyll", + "water-spm" + ], + "enabled": true, + "swatch": { + "color": "#C0C0C0", + "name": "Grey" + }, + "info": "Slowdown Proxy Maps show areas with the greatest reduction in car activity shaded in blue. Darker blues indicate areas of greater change." +} \ No newline at end of file diff --git a/covid_api/db/static/datasets/togo-ag.json b/covid_api/db/static/datasets/togo-ag.json new file mode 100644 index 0000000..92fc15a --- /dev/null +++ b/covid_api/db/static/datasets/togo-ag.json @@ -0,0 +1,37 @@ +{ + "id": "togo-ag", + "name": "Agriculture", + "type": "raster", + "description": "Dark purple colors indicate lower probability of cropland while lighter yellow colors indicate higher probability of cropland within each pixel.", + "s3_location": "Togo", + "source": { + "type": "raster", + "tiles": [ + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/Togo/togo_cropland_v7-1_cog_v2.tif&resampling_method=bilinear&bidx=1&rescale=0%2C1&color_map=inferno" + ] + }, + "enabled": true, + "exclusive_with": [], + "swatch": { + "color": "#C0C0C0", + "name": "Grey" + }, + "legend": { + "type": "gradient", + "min": "low", + "max": "high", + "stops": [ + "#000000", + "#1a0b40", + "#4b0c6b", + "#791c6d", + "#a42c60", + "#cf4446", + "#ed6825", + "#fb9b06", + "#f6d13c", + "#fbfda2" + ] + }, + "info": "Dark purple colors indicate lower probability of cropland while lighter yellow colors indicate higher probability of cropland within each pixel." +} \ No newline at end of file diff --git a/covid_api/db/static/datasets/water-chlorophyll.json b/covid_api/db/static/datasets/water-chlorophyll.json index eb17ac5..146d09f 100644 --- a/covid_api/db/static/datasets/water-chlorophyll.json +++ b/covid_api/db/static/datasets/water-chlorophyll.json @@ -2,17 +2,17 @@ "id": "water-chlorophyll", "name": "Chlorophyll", "type": "raster-timeseries", - "description": "Chlorophyll is an indicator of algae growth. Redder colors indicate increases in chlorophyll-a and worse water quality. Bluer colors indicate decreases in chlorophyll-a and improved water quality. White areas indicate no change.", "time_unit": "day", "is_periodic": false, "s3_location": "oc3_chla_anomaly", "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/oc3_chla_anomaly/anomaly-chl-{spotlightId}-{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/oc3_chla_anomaly/anomaly-chl-{spotlightId}-{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/water-spm.json b/covid_api/db/static/datasets/water-spm.json index aa893b9..c4052b7 100644 --- a/covid_api/db/static/datasets/water-spm.json +++ b/covid_api/db/static/datasets/water-spm.json @@ -2,17 +2,17 @@ "id": "water-spm", "name": "Turbidity", "type": "raster-timeseries", - "description": "Turbidity refers to the amount of sediment or particles suspended in water. Darker colors indicate more sediment and murkier water. Lighter colors indicate less sediment and clearer water.", "time_unit": "day", "s3_location": "spm_anomaly", "is_periodic": false, "source": { "type": "raster", "tiles": [ - "https://h4ymwpefng.execute-api.us-east-1.amazonaws.com/v1/{z}/{x}/{y}@1x?url=s3://covid-eo-data/spm_anomaly/anomaly-spm-{spotlightId}-{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/spm_anomaly/anomaly-spm-{spotlightId}-{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" ] }, "exclusive_with": [ + "agriculture", "no2", "co2-diff", "co2", diff --git a/covid_api/db/static/datasets/wq-gl-chl.json b/covid_api/db/static/datasets/wq-gl-chl.json new file mode 100644 index 0000000..7d86481 --- /dev/null +++ b/covid_api/db/static/datasets/wq-gl-chl.json @@ -0,0 +1,34 @@ +{ + "id": "water-wq-gl-chl", + "name": "Chlorophyll", + "type": "raster-timeseries", + "s3_location": "wq-greatlakes-chl", + "time_unit": "day", + "source": { + "type": "raster", + "tiles": [ + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/wq-greatlakes-chl/chl_anomaly_greatlakes_{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" + ] + }, + "exclusive_with": [ + "water-wq-gl-spm" + ], + "swatch": { + "color": "#154F8D", + "name": "Deep blue" + }, + "legend": { + "type": "gradient", + "min": "less", + "max": "more", + "stops": [ + "#3A88BD", + "#C9E0ED", + "#E4EEF3", + "#FDDCC9", + "#DE725B", + "#67001F" + ] + }, + "info": "Chlorophyll is an indicator of algae growth. Redder colors indicate increases in chlorophyll-a and worse water quality. Bluer colors indicate decreases in chlorophyll-a and improved water quality. White areas indicate no change." +} \ No newline at end of file diff --git a/covid_api/db/static/datasets/wq-gl-spm.json b/covid_api/db/static/datasets/wq-gl-spm.json new file mode 100644 index 0000000..904051e --- /dev/null +++ b/covid_api/db/static/datasets/wq-gl-spm.json @@ -0,0 +1,34 @@ +{ + "id": "water-wq-gl-spm", + "name": "Turbidity", + "type": "raster-timeseries", + "time_unit": "day", + "s3_location": "wq-greatlakes-sm", + "source": { + "type": "raster", + "tiles": [ + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/wq-greatlakes-sm/sm_anomaly_greatlakes_{date}.tif&resampling_method=bilinear&bidx=1&rescale=-100%2C100&color_map=rdbu_r" + ] + }, + "exclusive_with": [ + "water-wq-gl-chl" + ], + "swatch": { + "color": "#154F8D", + "name": "Deep blue" + }, + "legend": { + "type": "gradient", + "min": "less", + "max": "more", + "stops": [ + "#3A88BD", + "#C9E0ED", + "#E4EEF3", + "#FDDCC9", + "#DE725B", + "#67001F" + ] + }, + "info": "Turbidity refers to the amount of sediment or particles suspended in water. Darker colors indicate more sediment and murkier water. Lighter colors indicate less sediment and clearer water." +} \ No newline at end of file diff --git a/covid_api/db/static/groups/__init__.py b/covid_api/db/static/groups/__init__.py index 29e001f..2365c5d 100644 --- a/covid_api/db/static/groups/__init__.py +++ b/covid_api/db/static/groups/__init__.py @@ -2,8 +2,8 @@ import os from typing import List -from covid_api.models.static import IndicatorGroup, IndicatorGroups from covid_api.db.static.errors import InvalidIdentifier +from covid_api.models.static import IndicatorGroup, IndicatorGroups data_dir = os.path.join(os.path.dirname(__file__)) diff --git a/covid_api/db/static/sites/__init__.py b/covid_api/db/static/sites/__init__.py index 75aa207..4aec63b 100644 --- a/covid_api/db/static/sites/__init__.py +++ b/covid_api/db/static/sites/__init__.py @@ -1,12 +1,11 @@ """ covid_api static sites """ import os -from typing import List from enum import Enum +from typing import List -from covid_api.models.static import Site, Sites from covid_api.db.static.errors import InvalidIdentifier - -from covid_api.db.utils import get_indicators, indicator_folders, indicator_exists +from covid_api.db.utils import get_indicators, indicator_exists, indicator_folders +from covid_api.models.static import Site, Sites data_dir = os.path.join(os.path.dirname(__file__)) diff --git a/covid_api/db/utils.py b/covid_api/db/utils.py index 86cb9b6..0255d72 100644 --- a/covid_api/db/utils.py +++ b/covid_api/db/utils.py @@ -1,21 +1,21 @@ """Db tools.""" -import boto3 import csv import json import re from datetime import datetime from typing import Dict, List, Optional, Set -from covid_api.core.config import INDICATOR_BUCKET, DT_FORMAT, MT_FORMAT +import boto3 + +from covid_api.core.config import DT_FORMAT, INDICATOR_BUCKET, MT_FORMAT from covid_api.models.static import IndicatorObservation s3 = boto3.client("s3") def gather_s3_keys( - spotlight_id: Optional[str] = None, - prefix: Optional[str] = None, + spotlight_id: Optional[str] = None, prefix: Optional[str] = None, ) -> Set[str]: """ Returns a set of S3 keys. If no args are provided, the keys will represent @@ -57,9 +57,7 @@ def gather_s3_keys( key for key in keys if re.search( - rf"""[^a-zA-Z0-9]({spotlight_id})[^a-zA-Z0-9]""", - key, - re.IGNORECASE, + rf"""[^a-zA-Z0-9]({spotlight_id})[^a-zA-Z0-9]""", key, re.IGNORECASE, ) } @@ -81,9 +79,7 @@ def get_dataset_folders_by_spotlight(spotlight_id: str) -> Set[str]: def get_dataset_domain( - dataset_folder: str, - is_periodic: bool, - spotlight_id: str = None, + dataset_folder: str, is_periodic: bool, spotlight_id: str = None, ): """ Returns a domain for a given dataset as identified by a folder. If a @@ -95,7 +91,7 @@ def get_dataset_domain( ------ dataset_folder (str): dataset folder to search within time_unit (Optional[str]): time_unit from the dataset's metadata json file - spotlight (optional[Dict[str,str]]): a dictionary containing the + spotlight_id (Optional[str]): a dictionary containing the `spotlight_id` of a spotlight to restrict the domain search to. @@ -135,12 +131,12 @@ def get_dataset_domain( # Invalid date value matched continue - dates.append(date) + dates.append(date.strftime("%Y-%m-%dT%H:%M:%SZ")) if is_periodic and len(dates): return [min(dates), max(dates)] - return sorted(dates) + return sorted(set(dates)) def s3_get(bucket: str, key: str): @@ -161,9 +157,7 @@ def get_indicator_site_metadata(identifier: str, folder: str) -> Dict: def indicator_folders() -> List: """Get Indicator folders.""" response = s3.list_objects_v2( - Bucket=INDICATOR_BUCKET, - Prefix="indicators/", - Delimiter="/", + Bucket=INDICATOR_BUCKET, Prefix="indicators/", Delimiter="/", ) return [obj["Prefix"].split("/")[1] for obj in response["CommonPrefixes"]] @@ -172,8 +166,7 @@ def indicator_exists(identifier: str, indicator: str): """Check if an indicator exists for a site""" try: s3.head_object( - Bucket=INDICATOR_BUCKET, - Key=f"indicators/{indicator}/{identifier}.csv", + Bucket=INDICATOR_BUCKET, Key=f"indicators/{indicator}/{identifier}.csv", ) return True except Exception: @@ -206,9 +199,7 @@ def get_indicators(identifier) -> List: INDICATOR_BUCKET, f"indicators/{folder}/{identifier}.csv" ) indicator_lines = indicator_csv.decode("utf-8").split("\n") - reader = csv.DictReader( - indicator_lines, - ) + reader = csv.DictReader(indicator_lines,) # top level metadata is added directly to the response top_level_fields = { @@ -237,12 +228,10 @@ def get_indicators(identifier) -> List: indicator["domain"] = dict( date=[ min( - data, - key=lambda x: datetime.strptime(x["date"], DT_FORMAT), + data, key=lambda x: datetime.strptime(x["date"], DT_FORMAT), )["date"], max( - data, - key=lambda x: datetime.strptime(x["date"], DT_FORMAT), + data, key=lambda x: datetime.strptime(x["date"], DT_FORMAT), )["date"], ], indicator=[ diff --git a/covid_api/main.py b/covid_api/main.py index 2193c55..02ac04b 100644 --- a/covid_api/main.py +++ b/covid_api/main.py @@ -1,18 +1,18 @@ """covid_api app.""" from typing import Any, Dict +from covid_api import version +from covid_api.api.api_v1.api import api_router +from covid_api.core import config +from covid_api.db.memcache import CacheLayer + from fastapi import FastAPI + +from starlette.middleware.cors import CORSMiddleware +from starlette.middleware.gzip import GZipMiddleware from starlette.requests import Request from starlette.responses import HTMLResponse from starlette.templating import Jinja2Templates -from starlette.middleware.cors import CORSMiddleware -from starlette.middleware.gzip import GZipMiddleware - - -from covid_api import version -from covid_api.core import config -from covid_api.db.memcache import CacheLayer -from covid_api.api.api_v1.api import api_router templates = Jinja2Templates(directory="covid_api/templates") diff --git a/covid_api/models/static.py b/covid_api/models/static.py index 5979037..d1d858d 100644 --- a/covid_api/models/static.py +++ b/covid_api/models/static.py @@ -1,74 +1,120 @@ """Static models.""" -from typing import List, Any, Optional, Union, Dict -from pydantic import BaseModel -from pydantic.color import Color +from typing import Any, List, Optional, Union + from geojson_pydantic.features import FeatureCollection from geojson_pydantic.geometries import Polygon +from pydantic import BaseModel # , validator + +# from pydantic.color import Color + + +def to_camel(snake_str: str) -> str: + """ + Converts snake_case_string to camelCaseString + """ + first, *others = snake_str.split("_") + return "".join([first.lower(), *map(str.title, others)]) class Source(BaseModel): - """Source Model.""" + """Base Source Model""" type: str - tiles: List + + +class NonGeoJsonSource(Source): + """Source Model for all non-geojson data types""" + + tiles: List[str] + + +class GeoJsonSource(Source): + """Source Model for geojson data types""" + + data: str class Swatch(BaseModel): """Swatch Model.""" - color: Color + color: str name: str +class LabelStop(BaseModel): + """Model for Legend stops with color + label""" + + color: str + label: str + + class Legend(BaseModel): """Legend Model.""" type: str min: Optional[str] max: Optional[str] - stops: Union[List[Color], List[Dict[str, str]]] + stops: Union[List[str], List[LabelStop]] -class Dataset(BaseModel): - """Dataset Model.""" +class DatasetComparison(BaseModel): + """ Dataset `compare` Model.""" - id: str - name: str - description: str = "" - type: str - s3_location: Optional[str] - is_periodic: bool = True + enabled: bool + help: str + year_diff: int + map_label: str + source: NonGeoJsonSource time_unit: Optional[str] - domain: List = [] - source: Source - background_source: Optional[Source] - swatch: Swatch - legend: Optional[Legend] - info: str = "" -class OutputDataset(BaseModel): +class Paint(BaseModel): + """Paint Model.""" + + raster_opacity: int + + +class Dataset(BaseModel): """Dataset Model.""" id: str name: str - description: str = "" type: str - is_periodic: bool = True - time_unit: Optional[str] - domain: List = [] - source: Source - background_source: Optional[Source] + + is_periodic: bool = False + time_unit: str = "" + domain: List[str] = [] + source: Union[NonGeoJsonSource, GeoJsonSource] + background_source: Optional[Union[NonGeoJsonSource, GeoJsonSource]] + exclusive_with: List[str] = [] swatch: Swatch + compare: Optional[DatasetComparison] legend: Optional[Legend] + paint: Optional[Paint] info: str = "" +class DatasetExternal(Dataset): + """ Public facing dataset model (uses camelCase fieldnames) """ + + class Config: + """Generates alias to convert all fieldnames from snake_case to camelCase""" + + alias_generator = to_camel + allow_population_by_field_name = True + + +class DatasetInternal(Dataset): + """ Private dataset model (includes the dataset's location in s3) """ + + s3_location: Optional[str] + + class Datasets(BaseModel): """Dataset List Model.""" - datasets: List[OutputDataset] + datasets: List[DatasetExternal] class Site(BaseModel): diff --git a/covid_api/models/timelapse.py b/covid_api/models/timelapse.py index b8aadc6..8275a1d 100644 --- a/covid_api/models/timelapse.py +++ b/covid_api/models/timelapse.py @@ -1,8 +1,8 @@ """Tilelapse models.""" -from pydantic import BaseModel from geojson_pydantic.features import Feature from geojson_pydantic.geometries import Polygon +from pydantic import BaseModel class PolygonFeature(Feature): diff --git a/covid_api/ressources/responses.py b/covid_api/ressources/responses.py index 735459c..72a1032 100644 --- a/covid_api/ressources/responses.py +++ b/covid_api/ressources/responses.py @@ -1,7 +1,7 @@ """Common response models.""" -from starlette.responses import Response from starlette.background import BackgroundTask +from starlette.responses import Response class XMLResponse(Response): diff --git a/lambda/handler.py b/lambda/handler.py index 2f95c4f..69c9be8 100644 --- a/lambda/handler.py +++ b/lambda/handler.py @@ -1,6 +1,7 @@ """AWS Lambda handler.""" from mangum import Mangum + from covid_api.main import app handler = Mangum(app, enable_lifespan=False) diff --git a/setup.py b/setup.py index d016c63..9d8ee61 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ """Setup covid_api.""" -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open("README.md") as f: long_description = f.read() @@ -41,7 +41,7 @@ setup( name="covid_api", - version="0.2.3", + version="0.3.0", description=u"", long_description=long_description, long_description_content_type="text/markdown", diff --git a/ship-to-api.py b/ship-to-api.py index 5b8c9d7..325452a 100644 --- a/ship-to-api.py +++ b/ship-to-api.py @@ -1,9 +1,9 @@ """processing script for ship detection data""" -import os +import csv import json +import os import subprocess from glob import glob -import csv def get_location(loc): diff --git a/stack/app.py b/stack/app.py index 3abfcb0..9581c36 100644 --- a/stack/app.py +++ b/stack/app.py @@ -1,21 +1,16 @@ """Construct App.""" -from typing import Any, Union - import os - -from aws_cdk import ( - core, - aws_iam as iam, - aws_ec2 as ec2, - aws_ecs as ecs, - aws_ecs_patterns as ecs_patterns, - aws_lambda, - aws_apigatewayv2 as apigw, - aws_elasticache as escache, -) +from typing import Any, Union import config +from aws_cdk import aws_apigatewayv2 as apigw +from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_ecs as ecs +from aws_cdk import aws_ecs_patterns as ecs_patterns +from aws_cdk import aws_elasticache as escache +from aws_cdk import aws_iam as iam +from aws_cdk import aws_lambda, core iam_policy_statement = iam.PolicyStatement( actions=["s3:*"], resources=[f"arn:aws:s3:::{config.BUCKET}*"] diff --git a/tests/conftest.py b/tests/conftest.py index 627fd3f..f5329d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,13 @@ """``pytest`` configuration.""" import os -import pytest - -from starlette.testclient import TestClient +import pytest import rasterio from rasterio.io import DatasetReader +from starlette.testclient import TestClient + @pytest.fixture(autouse=True) def app(monkeypatch) -> TestClient: diff --git a/tests/routes/v1/test_datasets.py b/tests/routes/v1/test_datasets.py index 82c8d18..207d51c 100644 --- a/tests/routes/v1/test_datasets.py +++ b/tests/routes/v1/test_datasets.py @@ -1,9 +1,11 @@ """Test /v1/datasets endpoints""" -import boto3 import json from datetime import datetime + +import boto3 from moto import mock_s3 + from covid_api.core.config import INDICATOR_BUCKET @@ -22,15 +24,15 @@ def _setup_s3(empty=False): "oc3_chla_anomaly/anomaly-chl-tk-2020_01_29.tif", "oc3_chla_anomaly/anomaly-chl-tk-2020_02_05.tif", "oc3_chla_anomaly/anomaly-chl-tk-2020_03_02.tif", - "BM_500M_DAILY/VNP46A2_V011_be_2020_01_01_cog.tif", - "BM_500M_DAILY/VNP46A2_V011_be_2020_02_29_cog.tif", - "BM_500M_DAILY/VNP46A2_V011_be_2020_03_20_cog.tif", - "BM_500M_DAILY/VNP46A2_V011_EUPorts_2020_01_01_cog.tif", - "BM_500M_DAILY/VNP46A2_V011_EUPorts_2020_02_29_cog.tif", - "BM_500M_DAILY/VNP46A2_V011_EUPorts_2020_03_20_cog.tif", - "BMHD_30M_MONTHLY/BMHD_VNP46A2_du_202005_cog.tif", - "BMHD_30M_MONTHLY/BMHD_VNP46A2_du_202006_cog.tif", - "BMHD_30M_MONTHLY/BMHD_VNP46A2_du_202007_cog.tif", + "bm_500m_daily/VNP46A2_V011_be_2020_01_01_cog.tif", + "bm_500m_daily/VNP46A2_V011_be_2020_02_29_cog.tif", + "bm_500m_daily/VNP46A2_V011_be_2020_03_20_cog.tif", + "bm_500m_daily/VNP46A2_V011_EUPorts_2020_01_01_cog.tif", + "bm_500m_daily/VNP46A2_V011_EUPorts_2020_02_29_cog.tif", + "bm_500m_daily/VNP46A2_V011_EUPorts_2020_03_20_cog.tif", + "bmhd_30m_monthly/BMHD_VNP46A2_du_202005_cog.tif", + "bmhd_30m_monthly/BMHD_VNP46A2_du_202006_cog.tif", + "bmhd_30m_monthly/BMHD_VNP46A2_du_202007_cog.tif", "OMNO2d_HRM/OMI_trno2_0.10x0.10_200401_Col3_V4.nc.tif", "OMNO2d_HRM/OMI_trno2_0.10x0.10_200708_Col3_V4.nc.tif", "OMNO2d_HRM/OMI_trno2_0.10x0.10_200901_Col3_V4.nc.tif", @@ -44,9 +46,7 @@ def _setup_s3(empty=False): ] for key in s3_keys: s3.put_object( - Bucket=INDICATOR_BUCKET, - Key=key, - Body=b"test", + Bucket=INDICATOR_BUCKET, Key=key, Body=b"test", ) return s3 @@ -77,14 +77,12 @@ def test_datasets_monthly(app): content = json.loads(response.content) assert "datasets" in content - print(content["datasets"]) - dataset_info = [d for d in content["datasets"] if d["id"] == "co2"][0] assert dataset_info["domain"][0] == datetime.strftime( - datetime(2019, 1, 1), "%Y-%m-%dT%H:%M:%S" + datetime(2019, 1, 1), "%Y-%m-%dT%H:%M:%SZ" ) assert dataset_info["domain"][1] == datetime.strftime( - datetime(2019, 6, 1), "%Y-%m-%dT%H:%M:%S" + datetime(2019, 6, 1), "%Y-%m-%dT%H:%M:%SZ" ) @@ -101,12 +99,20 @@ def test_euports_dataset(app): dataset_info = [d for d in content["datasets"] if d["id"] == "nightlights-hd"][0] assert dataset_info["domain"][0] == datetime.strftime( - datetime(2020, 5, 1), "%Y-%m-%dT%H:%M:%S" + datetime(2020, 5, 1), "%Y-%m-%dT%H:%M:%SZ" ) assert dataset_info["domain"][1] == datetime.strftime( - datetime(2020, 7, 1), "%Y-%m-%dT%H:%M:%S" + datetime(2020, 7, 1), "%Y-%m-%dT%H:%M:%SZ" ) + assert "_du_" in dataset_info["source"]["tiles"][0] + + # Dunkirk has two different datasets under two different spotlight names: + # "du" and "EUports" - both need to be tested individually + + dataset_info = [d for d in content["datasets"] if d["id"] == "nightlights-viirs"][0] + assert "_EUPorts_" in dataset_info["source"]["tiles"][0] + @mock_s3 def test_detections_datasets(app): @@ -122,7 +128,7 @@ def test_detections_datasets(app): assert "datasets" in content dataset_info = [d for d in content["datasets"] if d["id"] == "detections-plane"][0] - assert len(dataset_info["domain"]) == 2 + assert len(dataset_info["domain"]) > 2 @mock_s3 @@ -141,12 +147,14 @@ def test_datasets_daily(app): dataset_info = [d for d in content["datasets"] if d["id"] == "water-chlorophyll"][0] assert len(dataset_info["domain"]) > 2 assert dataset_info["domain"][0] == datetime.strftime( - datetime(2020, 1, 29), "%Y-%m-%dT%H:%M:%S" + datetime(2020, 1, 29), "%Y-%m-%dT%H:%M:%SZ" ) assert dataset_info["domain"][-1] == datetime.strftime( - datetime(2020, 3, 2), "%Y-%m-%dT%H:%M:%S" + datetime(2020, 3, 2), "%Y-%m-%dT%H:%M:%SZ" ) + assert "&rescale=-100%2C100" not in dataset_info["source"]["tiles"][0] + @mock_s3 def test_global_datasets(app): diff --git a/tests/routes/v1/test_sites.py b/tests/routes/v1/test_sites.py index 4125595..400d36a 100644 --- a/tests/routes/v1/test_sites.py +++ b/tests/routes/v1/test_sites.py @@ -2,6 +2,7 @@ import boto3 from moto import mock_s3 + from covid_api.core.config import INDICATOR_BUCKET diff --git a/tests/routes/v1/test_tiles.py b/tests/routes/v1/test_tiles.py index 0c91614..5f51e06 100644 --- a/tests/routes/v1/test_tiles.py +++ b/tests/routes/v1/test_tiles.py @@ -1,12 +1,10 @@ """test /v1/tiles endpoints.""" -from typing import Dict - from io import BytesIO -from mock import patch +from typing import Dict import numpy - +from mock import patch from rasterio.io import MemoryFile from ...conftest import mock_rio diff --git a/tox.ini b/tox.ini index b841189..d6a323d 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ commands = black +# Lint [flake8] ignore = D203 exclude = .git,__pycache__,docs/source/conf.py,old,build,dist @@ -26,5 +27,12 @@ max-complexity = 14 max-line-length = 90 [mypy] -no_strict_optional = true +no_strict_optional = True ignore_missing_imports = True + +[tool:isort] +profile=black +known_first_party = covid_api +forced_separate = fastapi,starlette +known_third_party = rasterio,morecantile,rio_tiler +default_section = THIRDPARTY