Merge pull request #47 from djgagne/djgagne

Speedups to segmentation

.readthedocs.yaml

@@ -0,0 +1,39 @@
+
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "mambaforge-22.9"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+conda:
+  environment: environment.yml
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: doc/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#   install:
+#     - requirements: docs/requirements.txt

environment.yml

@@ -1,9 +1,8 @@
 name: hagelslag
 channels:
   - conda-forge
-  - defaults
 dependencies:
-  - python>=3.8
+  - python<=3.10
   - numpy
   - scipy
   - s3fs
@@ -24,6 +23,8 @@ dependencies:
   - cython
   - sphinx
   - mock
+  - dask
+  - numba
   - jasper
   - flake8
   - zarr

hagelslag/VERSION

@@ -0,0 +1 @@
+0.6

hagelslag/data/GribModelGrid.py

@@ -1,9 +1,8 @@
 import datetime
 from os.path import exists
-
+import pygrib
 import numpy as np
 import pandas as pd
-import pygrib
 from netCDF4 import Dataset
 
 

hagelslag/data/ZarrModelGrid.py

@@ -38,7 +38,7 @@ def __init__(self,
         self.end_date = pd.to_datetime(end_date)
         self.frequency = frequency
         self.valid_dates = date_range(start=self.start_date, end=self.end_date, freq=self.frequency)
-        self.forecast_hours = (self.valid_dates - self.run_date).astype("timedelta64[h]").astype(int)
+        self.forecast_hours = (self.valid_dates - self.run_date).values.astype("timedelta64[h]").astype(int)
 
     def load_data(self):
 

hagelslag/mapfiles/GRIB2Table_hrrr_2d.csv

@@ -0,0 +1,115 @@
+High Resolution Rapid Refresh (HRRR)
+CONUS 2-D Fields
+GRIB2 Table Documentation
+13 March 2013
+
+RecordNumber,TableNumber,DisciplineNumber,CategoryNumber,ParameterNumber,WGrib2Name,NCLName,FieldType,Description,Units,
+1,0,0,16,196,var,REFC_P0_L200_GLC0,Composite Reflectivity,Maximum reflectivity in model column,dBZ,
+2,0,0,3,193,var,5WAVH_P0_L3_GLC0,Echo Top,Echo top height of 18dBZ surface,m,
+3,0,0,3,194,var,UGWD_P0_L200_GLC0,VIL,Vertically integrated liquid,kg/m^2,
+4,0,0,3,195,var,VGWD_P0_L200_GLC0,Radar VIL,Radar-derived vertically integrated liquid,kg/m^2,
+5,0,0,19,0,VIS,VIS_P0_L1_GLC0,Visibility,Surface visibility,m,
+6,0,0,16,195,var,REFD_P0_L103_GLC0,Reflectivity,Reflectivity at 1 km above ground level,dBZ,
+7,8,0,3,206,var,NLGSP_P8_2L100_GLC0_1h,Maximum Updraft,Maximum updraft velocity in 1000-400 mb layer over previous hour,m/s,
+8,8,2,0,196,var,CNWAT_P8_2L100_GLC0,Maximum Downdraft,Maximum downdraft velocity in 1000-400 mb layer over previous hour,m/s,
+9,8,0,2,9,DZDT,DZDT_P8_2L104_GLC0,Mean Vertical Velocity,Mean vertical velocity in sigma 0.8-0.5 layer,m/s,
+10,0,0,3,1,PRMSL,PRMSL_P0_L101_GLC0,Pressure,Pressure reduced to mean sea level via MAPS method,Pa,
+11,8,0,1,198,var,MINRH_P8_L103_GLC0_1h,Maximum Reflectivity,Maximum reflectivity at 1 km above ground level over previous hour,dBZ,
+12,8,3,192,2,SBT124,SBT124_P8_2L103_GLC0_1h,Maximum Updraft Helicity,Maximum updraft helicity in layer 2-5 km above ground level over previous hour,m^2/s^2,
+13,8,3,192,3,SBT126,SBT125_P8_2L103_GLC0_1h,Maximum Updraft Helicity,Maximum updraft helicity in layer 1-6 km above ground level over previous hour,m^2/s^2,
+14,8,10,3,192,var,SURGE_P8_L200_GLC0_1h,Vertically Integrated Graupel,Maximum vertically integrated graupel in model column over previous hour,kg/m^2,
+15,8,2,0,206,var,RDRIP_P8_L200_GLC0_1h,Lightning Threat 1,Maximum lightning threat 1 based on vertical flux of graupel at -15C over previous hour,flashes/km^2/(5 min),
+16,8,0,1,206,var,TIPD_P8_L200_GLC0_1h,Lightning Threat 2,Maximum lightning threat 2 based on vertically integrated graupel snow and cloud ice over previous hour,flashes/km^2/(5 min),
+17,0,0,17,192,var,LTNG_P8_L200_GLC0_1h,Lightning Threat 3,Maximum lightning threat 3 based on a weighted combination of lightning threat 1 and 2 over previous hour,flashes/km^2/(5 min),
+18,0,0,7,197,var,UPHL_P8_2L103_GLC0,Updraft Helicity,Updraft helicity in layer 2-5 km above ground level,m^2/s^2,
+19,0,3,192,1,SBT123,SBT123_P8_2L103_GLC0,Updraft Helicity,Updraft helicity in layer 1-6 km above ground level,m^2/s^2,
+20,0,0,0,195,var,LRGHR_P0_L200_GLC0,Heating Rate,Large scale condensate heating rate,K/s,
+21,0,0,0,196,var,CNVHR_P0_L200_GLC0,Heating Rate,Deep convective heating rate,K/s,
+22,0,0,1,213,var,CNVMR_P0_L200_GLC0,Moistening Rate,Deep convective moistening rate,kg/kg/s,
+23,0,0,0,201,var,SHAHR_P0_L200_GLC0,Heating Rate,Shallow convective heating rate,K/s,
+24,0,0,1,214,var,SHAMR_P0_L200_GLC0,Moistening Rate,Shallow convective moistening rate,kg/kg/s,
+25,0,0,0,202,var,VDFHR_P0_L200_GLC0,Heating Rate,Vertical diffusion heating rate,K/s,
+26,0,0,2,19,WMIXE,WMIXE_P0_L103_GLC0,Wind Energy,Wind energy generation potential using 0.5*density*speed^3 at 80 m above ground level,W/m^2,
+27,0,10,1,2,UOGRD,UOGRD_P0_L103_GLC0,Wind,U-component of wind at 80 m above ground level,m/s,
+28,0,10,1,3,VOGRD,VOGRD_P0_L103_GLC0,Wind,V-component of wind at 80 m above ground level,m/s,
+29,0,0,3,0,PRES,PRES_P0_L1_GLC0,Pressure,Surface pressure,Pa,
+30,0,0,0,0,TMP,TMP_P0_L1_GLC0,Temperature,Surface (skin) temperature,K,
+31,0,2,0,2,TSOIL,TSOIL_P0_L1_GLC0,Soil Temperature,Surface (skin) soil temperature,K,
+32,0,2,0,192,var,SOILW_P0_L1_GLC0,Soil Moisture,Surface (skin) volumetric soil moisture content (0 to 0.5 cm layer below surface),fraction,
+33,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(0),Soil Temperature,Soil temperature 1 cm below surface,K,
+34,0,2,0,192,var,SOILW_P0_L106_GLC0(0),Soil Moisture,Volumetric soil moisture content 1 cm below surface (0.5 to 2.5 cm layer below surface),fraction,
+35,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(1),Soil Temperature,Soil temperature 4 cm below surface,K,
+36,0,2,0,192,var,SOILW_P0_L106_GLC0(1),Soil Moisture,Volumetric soil moisture content 4 cm below surface (2.5 to 7 cm layer below surface),fraction,
+37,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(2),Soil Temperature,Soil temperature 10 cm below surface,K,
+38,0,2,0,192,var,SOILW_P0_L106_GLC0(2),Soil Moisture,Volumetric soil moisture content 10 cm below surface (7 to 20 cm layer below surface),fraction,
+39,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(3),Soil Temperature,Soil temperature 30 cm below surface,K,
+40,0,2,0,192,var,SOILW_P0_L106_GLC0(3),Soil Moisture,Volumetric soil moisture content 30 cm below surface (20 to 45 cm layer below surface),fraction,
+41,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(4),Soil Temperature,Soil temperature 60 cm below surface,K,
+42,0,2,0,192,var,SOILW_P0_L106_GLC0(4),Soil Moisture,Volumetric soil moisture content 60 cm below surface (45 to 80 cm layer below surface),fraction,
+43,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(5),Soil Temperature,Soil temperature 100 cm below surface,K,
+44,0,2,0,192,var,SOILW_P0_L106_GLC0(5),Soil Moisture,Volumetric soil moisture content 100 cm below surface (80 to 130 cm layer below surface),fraction,
+45,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(6),Soil Temperature,Soil temperature 160 cm below surface,K,
+46,0,2,0,192,var,SOILW_P0_L106_GLC0(6),Soil Moisture,Volumetric soil moisture content 160 cm below surface (130 to 230 cm layer below surface),fraction,
+47,0,2,0,2,TSOIL,TSOIL_P0_L106_GLC0(7),Soil Temperature,Soil temperature 300 cm below surface,K,
+48,0,2,0,192,var,SOILW_P0_L106_GLC0(7),Soil Moisture,Volumetric soil moisture content 300 cm below surface (230 to 300 cm layer below surface),fraction,
+49,0,0,1,13,WEASD,WEASD_P0_L1_GLC0,Snow Water,Water equivalent of accumulated snow depth,kg/m^2,
+50,0,0,0,0,TMP,TMP_P0_L103_GLC0,Temperature,Temperature at 2 m above ground level,K,
+51,0,0,0,2,POT,POT_P0_L103_GLC0,Potential Temperature,Potential temperature at 2 m above ground level,K,
+52,0,0,1,0,SPFH,SPFH_P0_L103_GLC0,Specific Humidity,Specific humidity at 2 m above ground level,kg/kg,
+53,0,0,0,6,DPT,DPT_P0_L103_GLC0,Dewpoint,Dewpoint temperature at 2 m above ground level,K,
+54,0,0,1,1,RH,RH_P0_L103_GLC0,Relative Humidity,Relative humidity at 2 m above ground level,%,
+55.1,0,0,2,2,UGRD,UGRD_P0_L103_GLC0,Wind,U-component of wind at 10 m above ground level,m/s,
+55.2,0,0,2,3,VGRD,VGRD_P0_L103_GLC0,Wind,V-component of wind at 10 m above ground level,m/s,
+56,8,0,0,192,var,SNOHF_P8_L103_GLC0_1h,Maximum Wind,Maximum horizontal wind speed at 10 m above ground level over previous hour,m/s,
+57,8,0,1,8,APCP,APCP_P8_L1_GLC0_accHHh,Precipitation,Total precipitation accumulated since initial time,kg/m^2,
+58,8,0,1,10,ACPCP,ACPCP_P8_L1_GLC0_accHHh,Precipitation,Total convective precipitation accumulated since initial time,kg/m^2,
+59,8,0,1,9,NCPCP,NCPCP_P8_L1_GLC0_accHHh,Precipitation,Total large-scale precipitation accumulated since initial time,kg/m^2,
+60,8,0,1,13,WEASD,WEASD_P8_L1_GLC0_accHHh,Precipitation,Water equivalent of accumulated snow depth since initial time,kg/m^2,
+61,8,0,1,8,APCP,APCP_P8_L1_GLC0_acc1h,Precipitation,Total precipitation accumulated over previous hour,kg/m^2,
+62,8,0,1,10,ACPCP,ACPCP_P8_L1_GLC0_acc1h,Precipitation,Total convective precipitation accumulated over previous hour,kg/m^2,
+63,8,0,1,9,NCPCP,NCPCP_P8_L1_GLC0_acc1h,Precipitation,Total large-scale precipitation accumulated over previous hour,kg/m^2,
+64,8,0,1,13,WEASD,WEASD_P8_L1_GLC0_acc1h,Precipitation,Water equivalent of accumulated snow depth over previous hour,kg/m^2,
+65,0,0,1,195,var,CSNOW_P0_L1_GLC0,Precipitation Type,Categorical snow (yes=1 no=0),flag,
+66,0,0,1,194,var,CICEP_P0_L1_GLC0,Precipitation Type,Categorical ice pellets (yes=1 no=0),flag,
+67,0,0,1,193,var,CFRZR_P0_L1_GLC0,Precipitation Type,Categorical freezing rain (yes=1 no=0),flag,
+68,0,0,1,192,var,CRAIN_P0_L1_GLC0,Precipitation Type,Categorical rain (yes=1 no=0),flag,
+69,0,0,2,197,FRICV,FRICV_P0_L1_GLC0,Wind,Frictional velocity,m/s,
+70,0,0,0,11,SHTFL,SHTFL_P0_L1_GLC0,Heat Flux,Sensible heat net flux at surface,W/m^2,
+71,0,0,0,10,LHTFL,LHTFL_P0_L1_GLC0,Heat Flux,Latent heat net flux at surface,W/m^2,
+72,0,2,0,193,var,GFLUX_P0_L1_GLC0,Heat Flux,Ground heat flux at surface,W/m^2,
+73,0,0,7,192,var,LFTX_P0_2L100_GLC0,Lifted Index,Surface lifted index using 500 mb reference level,K,
+74,0,0,7,6,CAPE,CAPE_P0_L1_GLC0,Instability,Surface CAPE using highest theta-e parcel from lowest 10 mb,J/kg,
+75,0,0,7,7,CIN,CIN_P0_L1_GLC0,Instability,Surface CIN using highest theta-e parcel from lowest 10 mb,J/kg,
+76,0,0,1,3,PWAT,PWAT_P0_L200_GLC0,Precipitable Water,Precipitable water in model column,kg/m^2,
+77,0,0,6,3,LCDC,LCDC_P0_L214_GLC0,Cloud Cover,Low cloud cover,%,
+78,0,0,6,4,MCDC,MCDC_P0_L224_GLC0,Cloud Cover,Medium cloud cover,%,
+79,0,0,6,5,HCDC,HCDC_P0_L234_GLC0,Cloud Cover,High cloud cover,%,
+80,0,0,6,1,TCDC,TCDC_P0_L200_GLC0,Cloud Cover,Total cloud cover,%,
+81,0,0,3,5,HGT,HGT_P0_L2_GLC0,Cloud Base,Geopotential height of cloud base,gpm,
+82,0,0,3,5,HGT,HGT_P0_L3_GLC0,Cloud Top,Geopotential height of cloud top,gpm,
+83,0,0,5,193,var,ULWRF_P0_L8_GLC0,Radiation,Outgoing longwave radiation at top of atmosphere,W/m^2,
+84,0,0,4,192,var,DSWRF_P0_L1_GLC0,Radiation,Incoming shortwave radiation at the surface,W/m^2,
+85,0,0,7,8,HLCY,HLCY_P0_2L103_GLC0(0),Helicity,Storm-relative helicity for 0-3 km above ground level using Bunkers et al. 2000 storm-motion,m^2/s^2,
+86,0,0,7,8,HLCY,HLCY_P0_2L103_GLC0(1),Helicity,Storm-relative helicity for 0-1 km above ground level using Bunkers et al. 2000 storm-motion,m^2/s^2,
+87,0,0,2,194,var,USTM_P0_2L103_GLC0,Storm Motion,U-component of convective storm motion for right-moving cells using Bunkers et al. 2000,m/s,
+88,0,0,2,195,var,VSTM_P0_2L103_GLC0,Storm Motion,V-component of convective storm motion for right-moving cells using Bunkers et al. 2000,m/s,
+89,0,0,2,15,VUCSH,VUCSH_P0_2L103_GLC0(0),Wind Shear,U-component of vector wind difference between 10 m and 1 km above ground level,m/s,
+90,0,0,2,16,VVCSH,VVCSH_P0_2L103_GLC0(0),Wind Shear,V-component of vector wind difference between 10 m and 1 km above ground level,m/s,
+91,0,0,2,15,VUCSH,VUCSH_P0_2L103_GLC0(1),Wind Shear,U-component of vector wind difference between wind in surface-500 m and 5.5-6.0 km layers above ground level,m/s,
+92,0,0,2,16,VVCSH,VVCSH_P0_2L103_GLC0(1),Wind Shear,V-component of vector wind difference between wind in surface-500 m and 5.5-6.0 km layers above ground level,m/s,
+93,0,0,0,0,TMP,TMP_P0_2L108_GLC0,Temperature,Mean temperature in lowest 30 mb,K,
+94,0,0,0,2,POT,POT_P0_2L108_GLC0,Temperature,Mean potential temperature in lowest 30 mb,K,
+95,0,0,1,1,RH,RH_P0_2L108_GLC0,Relative Humidity,Mean relative humidity in lowest 30 mb,%,
+96,0,0,0,6,DPT,DPT_P0_2L108_GLC0,Dewpoint,Mean dewpoint temperature in lowest 30 mb,K,
+97,0,0,7,193,var,4LFTX_P0_2L108_GLC0,Lifted Index,Most unstable lifted index in six lowest 30 mb layers using 500 mb reference level,K,
+98,0,0,7,6,CAPE,CAPE_P0_2L108_GLC0(1),Instability,CAPE using highest average theta-e in six lowest 30 mb layers,J/kg,
+99,0,0,7,7,CIN,CIN_P0_2L108_GLC0(1),Instability,CIN using highest average theta-e in six lowest 30 mb layers,J/kg,
+100,0,0,3,196,var,HPBL_P0_L1_GLC0,Height,Planetary boundary layer height,m,
+101,0,0,3,5,HGT,HGT_P0_L5_GLC0,Height,Adiabatic condensation level above ground (LCL),gpm,
+102,0,0,7,6,CAPE,CAPE_P0_2L108_GLC0(0),Instability,Mixed-layer CAPE (MLCAPE) using average theta-e of the three lowest 30 mb layers,J/kg,
+103,0,0,7,7,CIN,CIN_P0_2L108_GLC0(0),Instability,Mixed-layer CIN (MLCIN) using average theta-e of the three lowest 30 mb layers,J/kg,
+104,0,0,7,6,CAPE,CAPE_P0_2L108_GLC0(2),Instability,Most unstable CAPE (MUCAPE) using parcel with highest theta-e in lowest 300 mb,J/kg,
+105,0,0,7,7,CIN,CIN_P0_2L108_GLC0(2),Instability,Most unstable CIN (MUCIN) using parcel with highest theta-e in lowest 300 mb,J/kg,
+106,0,0,3,200,var,PLPL_P0_2L108_GLC0,Pressure,Pressure level of most unstable parcel in lowest 300 mb,Pa,
+107,0,0,2,22,GUST,GUST_P0_L1_GLC0,Wind,Wind gust speed at 10 m above ground level,m/s,
+108,0,255,255,255,var,VAR_255_255_255_P0_L200_GLC0,Relative Humidity,Relative humidity with respect to precipitable water in model column,%,

hagelslag/mapfiles/GRIB2Table_hrrr_nat.csv

@@ -0,0 +1,20 @@
+High Resolution Rapid Refresh (HRRR)
+CONUS 3-D Native Level Fields (50 levels)
+GRIB2 Table Documentation
+25 Oct 2012
+
+RecordNumber,TableNumber,DisciplineNumber,CategoryNumber,ParameterNumber,WGrib2Name,NCLName,FieldType,Description,Units,
+1,0,0,3,0,PRES,PRES_P0_L100_GLC0,Pressure,Pressure,Pa,
+2,0,0,1,22,CLWMR,CLWMR_P0_L100_GLC0,Mixing Ratio,Cloud water mixing ratio,kg/kg,
+3,8,0,6,0,CICE,CICE_P0_L100_GLC0,Mixing Ratio,Cloud ice mixing ratio,kg/kg,
+4,8,0,1,24,RWMR,RWMR_P0_L100_GLC0,Mixing Ratio,Rain mixing ratio,kg/kg,
+5,8,0,1,25,SNMR,SNMR_P0_L100_GLC0,Mixing Ratio,Snow mixing ratio,kg/kg,
+6,0,0,1,32,GRLE,GRLE_P0_L100_GLC0,Mixing Ratio,Graupel mixing ratio,kg/kg,
+7,0,0,6,1,TCDC,TCDC_P0_L100_GLC0,Cloud Cover,Total cloud cover,%,
+8,0,0,3,5,HGT,HGT_P0_L100_GLC0,Geopotential Height,Geopotential height,gpm,
+9,0,0,0,0,TMP,TMP_P0_L100_GLC0,Temperature,Temperature,K,
+10,0,0,1,0,SPFH,SPFH_P0_L100_GLC0,Specific Humidity,Specific humidity,kg/kg,
+11.1,0,0,2,2,UGRD,UGRD_P0_L100_GLC0,Wind,U-component of wind,m/s,
+11.2,0,0,2,3,VGRD,VGRD_P0_L100_GLC0,Wind,V-component of wind,m/s,
+12,0,0,2,8,VVEL,VVEL_P0_L100_GLC0,Vertical Velocity,Vertical velocity (pressure),Pa/s,
+13,0,0,19,11,TKE,TKE_P0_L100_GLC0,Turbulent Kinetic Energy,Turbulent kinetic energy,m^2/s^2,

hagelslag/mapfiles/README

@@ -0,0 +1,6 @@
+To make VSE_mask.nc
+run python ../hagelslag/util/create_model_grid_us_mask.py -m VSE.txt -s states/states.shp -o t.nc
+and  python ../hagelslag/util/create_model_grid_us_mask.py -m VSE.txt -s mexstates/mexstates.shp -o u.nc
+and merge them together
+ncea -y max t.nc u.nc VSE_mask.nc
+

hagelslag/mapfiles/VSE.txt

@@ -0,0 +1,14 @@
+proj=lcc
+a=6370000
+b=6370000
+lat_0=30.8
+lon_0=-90.
+lat_1=30.8
+lat_2=30.8
+units=m
+dx=3000
+dy=3000
+sw_lon=258.714
+sw_lat=24.019
+ne_lon=282.843185435306
+ne_lat=36.4670218542745