Merge branch 'develop' into feature/ocnanalecenworkflow

.gitignore

@@ -156,20 +156,8 @@ sorc/ocnicepost.fd
 # Ignore scripts from externals
 #------------------------------
 # jobs symlinks
-jobs/JGFS_ATMOS_WAFS
-jobs/JGFS_ATMOS_WAFS_BLENDING
-jobs/JGFS_ATMOS_WAFS_BLENDING_0P25
-jobs/JGFS_ATMOS_WAFS_GCIP
-jobs/JGFS_ATMOS_WAFS_GRIB2
-jobs/JGFS_ATMOS_WAFS_GRIB2_0P25
 # scripts symlinks
 scripts/exemcsfc_global_sfc_prep.sh
-scripts/exgfs_atmos_wafs_blending.sh
-scripts/exgfs_atmos_wafs_blending_0p25.sh
-scripts/exgfs_atmos_wafs_gcip.sh
-scripts/exgfs_atmos_wafs_grib.sh
-scripts/exgfs_atmos_wafs_grib2.sh
-scripts/exgfs_atmos_wafs_grib2_0p25.sh
 # ush symlinks
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
@@ -186,11 +174,7 @@ ush/global_chgres_driver.sh
 ush/global_cycle.sh
 ush/global_cycle_driver.sh
 ush/jediinc2fv3.py
-ush/mkwfsgbl.sh
 ush/ufsda
-ush/wafs_blending.sh
-ush/wafs_grib2.regrid.sh
-ush/wafs_intdsk.sh
 ush/finddate.sh
 ush/make_NTC_file.pl
 ush/make_ntc_bull.pl

docs/source/components.rst

@@ -28,7 +28,7 @@ Components included as submodules:
 * **GSI Monitor** (https://github.com/NOAA-EMC/GSI-Monitor): These tools monitor the GSI package's data assimilation, detecting and reporting missing data sources, low observation counts, and high penalty values
 * **GDAS** (https://github.com/NOAA-EMC/GDASApp): Jedi based Data Assimilation system. This system is currently being developed for marine Data Assimilation and in time will replace GSI for atmospheric data assimilation as well
 * **UFS UTILS** (https://github.com/ufs-community/UFS_UTILS): Utility codes needed for UFS-weather-model
-* **wxflow** Collection of python utilities for weather workflows (https://github.com/NOAA-EMC/wxflow) 
+* **wxflow** (https://github.com/NOAA-EMC/wxflow): Collection of python utilities for weather workflows
 * **Verif global** (https://github.com/NOAA-EMC/EMC_verif-global): Verification package to evaluate GFS parallels. It uses MET and METplus. At this moment the verification package is limited to providing atmospheric metrics only
 
 .. note::
@@ -57,19 +57,20 @@ Data
 
 Observation data, also known as dump data, is prepared in production and then archived in a global dump archive (GDA) for use by users when running cycled experiments. The GDA (identified as ``$DMPDIR`` in the workflow) is available on supported platforms and the workflow system knows where to find the data.
 
-* Hera: /scratch1/NCEPDEV/global/glopara/dump
-* Orion/Hercules: /work/noaa/rstprod/dump
-* Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/dump
-* WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/dump
-* S4: /data/prod/glopara/dump
+* Hera: ``/scratch1/NCEPDEV/global/glopara/dump``
+* Orion/Hercules: ``/work/noaa/rstprod/dump``
+* Jet: ``/mnt/lfs4/HFIP/hfv3gfs/glopara/dump``
+* WCOSS2: ``/lfs/h2/emc/global/noscrub/emc.global/dump``
+* S4: ``/data/prod/glopara/dump``
 
 -----------------------------
 Global Dump Archive Structure
 -----------------------------
 
-The global dump archive (GDA) mimics the structure of its production source: ``DMPDIR/CDUMP.PDY/[CC/atmos/]FILES``
+The global dump archive (GDA) mimics the structure of its production source:
 
-The ``CDUMP`` is either gdas, gfs, or rtofs. All three contain production output for each day (``PDY``). The gdas and gfs folders are further broken into cycle (``CC``) and component (``atmos``).
+* GDAS/GFS: ``DMPDIR/gdas[gfs].PDY/CC/atmos/FILES``
+* RTOFS: ``DMPDIR/rtofs.PDY/FILES``
 
 The GDA also contains special versions of some datasets and experimental data that is being evaluated ahead of implementation into production. The following subfolder suffixes exist:
 
@@ -81,6 +82,7 @@ The GDA also contains special versions of some datasets and experimental data th
 +--------+------------------------------------------------------------------------------------------------------+
 | ur     | Un-restricted versions of restricted files in production. Produced and archived on a 48hrs delay.    |
 |        | Some restricted datasets are unrestricted. Data amounts: restricted > un-restricted > non-restricted |
+|        | Limited availability. Discontinued producing mid-2023.                                               |
 +--------+------------------------------------------------------------------------------------------------------+
 | x      | Experimental global datasets being evaluated for production. Dates and types vary depending on       |
 |        | upcoming global upgrades.                                                                            |

docs/source/conf.py

@@ -13,13 +13,14 @@
 import os
 import sys
 sys.path.insert(0, os.path.abspath('.'))
-
+from datetime import datetime
 
 # -- Project information -----------------------------------------------------
 
 project = 'Global-workflow'
-copyright = '2023, Kate Friedman, Walter Kolczynski, Rahul Mahajan, Lin Gan, Arun Chawla'
-author = 'Kate Friedman, Walter Kolczynski, Rahul Mahajan, Lin Gan, Arun Chawla'
+year = datetime.now().year
+copyright = f"2015-{year} NOAA/NWS/NCEP/EMC"
+author = 'Kate Friedman, Walter Kolczynski, Rahul Mahajan, Lin Gan, and numerous collaborators and contributors'
 
 # The full version, including alpha/beta/rc tags
 release = '0.1'

docs/source/hpc.rst

@@ -72,23 +72,23 @@ Version
 
 It is advised to use Git v2+ when available. At the time of writing this documentation the default Git clients on the different machines were as noted in the table below. It is recommended that you check the default modules before loading recommended ones:
 
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
 | Machine  | Default  | Recommended                           |
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
 | Hera     | v2.18.0  | default                               |
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
 | Hercules | v2.31.1  | default                               |
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
 | Orion    | v1.8.3.1 | **module load git/2.28.0**            |
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
 | Jet      | v2.18.0  | default                               |
-+---------+----------+---------------------------------------+
-| WCOSS2   | v2.26.2  | default or **module load git/2.29.0** |
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
+| WCOSS2   | v2.35.3  | default                               |
++----------+----------+---------------------------------------+
 | S4       | v1.8.3.1 | **module load git/2.30.0**            |
-+---------+----------+---------------------------------------+
-| AWS PW   | v1.8.3.1 | default
-+---------+----------+---------------------------------------+
++----------+----------+---------------------------------------+
+| AWS PW   | v1.8.3.1 | default                               |
++----------+----------+---------------------------------------+
 
 ^^^^^^^^^^^^^
 Output format

docs/source/index.rst

@@ -27,6 +27,10 @@ General updates: NOAA employees and affiliates can join the gfs-announce distrib
 
 GitHub updates: Users should adjust their "Watch" settings for this repo so they receive notifications as they'd like to. Find the "Watch" or "Unwatch" button towards the top right of the `authoritative global-workflow repository page <https://github.com/NOAA-EMC/global-workflow>`_ and click it to adjust how you watch the repo.
 
+=================
+Table of Contents
+=================
+
 .. toctree:: 
    :numbered:
    :maxdepth: 3

docs/source/init.rst

@@ -51,6 +51,7 @@ Cold-start atmosphere-only cycled C96 deterministic C48 enkf (80 members) ICs ar
    Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C96C48
    Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C96C48
    WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C96C48
+   AWS: https://noaa-nws-global-pds.s3.amazonaws.com/index.html#data/ICSDIR/C96C48
 
 Start date = 2021122018
 
@@ -111,6 +112,7 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5
    Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500
    WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500
    Jet: /lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500
+   AWS: https://noaa-nws-global-pds.s3.amazonaws.com/index.html#data/ICSDIR/C48C48mx500
 
 Start date = 2021032312
 
@@ -246,7 +248,7 @@ Automated Generation
 Cycled mode
 -----------
 
-Not yet supported.
+Not yet supported. See the UFS_UTILS documentation on the gdas_init utility to generate your own ICs for cycled or forecast-only mode: https://noaa-emcufs-utils.readthedocs.io/en/latest/ufs_utils.html#gdas-init
 
 .. _forecastonly-coupled:
 
@@ -319,14 +321,14 @@ Manual Generation
 
 The following information is for users needing to generate cold-start initial conditions for a cycled experiment that will run at a different resolution or layer amount than the operational GFS (C768C384L127).
 
-The ``chgres_cube`` code is available from the `UFS_UTILS repository <https://github.com/ufs-community/UFS_UTILS>`_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users should see the documentation to generation initial conditions in the UFS_UTILS repository. The ``chgres_cube`` code/scripts currently support the following GFS inputs:
+The ``chgres_cube`` code is available from the `UFS_UTILS repository <https://github.com/ufs-community/UFS_UTILS>`_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users should see the `documentation to generation initial conditions in the UFS_UTILS repository <https://noaa-emcufs-utils.readthedocs.io/en/latest/ufs_utils.html#gdas-init>`_. The ``chgres_cube`` code/scripts currently support the following GFS inputs:
 
 * pre-GFSv14
 * GFSv14
 * GFSv15
 * GFSv16
 
-See instructions in UFS_UTILS to clone, build and generate initial conditions.
+See instructions in UFS_UTILS to clone, build and generate initial conditions: https://noaa-emcufs-utils.readthedocs.io/en/latest/ufs_utils.html#gdas-init
 
 .. _warmstarts-prod:
 

docs/source/setup.rst

@@ -6,9 +6,13 @@ Experiment Setup
 
  ::
 
-   # Note: this will wipe your existing lmod environment
    source workflow/gw_setup.sh
 
+.. warning::
+   Sourcing gw_setup.sh will wipe your existing lmod environment
+
+.. note::
+   Bash shell is required to source gw_setup.sh
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Forecast-only experiment

jobs/JGFS_ATMOS_WAFS

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+########################################
+# GFS AWIPS PRODUCT GENERATION
+########################################
+date
+export PS4='$SECONDS + ' 
+set -xa
+
+export KEEPDATA=${KEEPDATA:-NO}
+
+############################################
+# Working Directory
+############################################
+export DATA=${DATA:-${DATAROOT}/${jobid:?}}
+mkdir -p $DATA
+cd $DATA
+
+############################################
+# Output for executables
+############################################
+export pgmout=OUTPUT.$$
+
+############################################
+# Load the UTILITIES module
+############################################
+#### module load prod_util
+#### module load grib_util
+
+###########################################
+# Run setpdy and initialize PDY variables
+###########################################
+export cycle=t${cyc}z 
+setpdy.sh
+. ./PDY
+
+export RERUN=${RERUN:-NO}
+
+############################################
+# Set up the NET and RUN
+############################################
+export NET=${NET:-gfs}
+export RUN=${RUN:-gfs}
+export COMPONENT=${COMPONENT:-atmos}
+
+############################################
+# Specify HOME Directory
+############################################
+export gfs_ver=${gfs_ver:-v16.3.0}
+export HOMEgfs=${HOMEgfs:-${NWROOT}/gfs.${gfs_ver}}
+export EXECgfs=$HOMEgfs/exec
+export FIXgfs=$HOMEgfs/fix/wafs
+export PARMgfs=$HOMEgfs/parm/wafs
+export USHgfs=$HOMEgfs/ush
+export SCRIPTSgfs=$HOMEgfs/scripts
+
+################################################
+# Set up the input/output directory
+################################################
+export COMIN=${COMIN:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.${PDY}/${cyc}/$COMPONENT}
+export COMOUT=${COMOUT:-$(compath.py -o $NET/$gfs_ver)/$RUN.$PDY/$cyc/$COMPONENT} 
+export PCOM=${PCOM:-$COMOUT/wmo}
+
+if [ $SENDCOM = YES ] ; then
+  mkdir -p $COMOUT $PCOM
+fi
+
+############################################
+# print current environment
+############################################
+env
+
+############################################
+# Execute the script.
+############################################
+
+${SCRIPTSgfs}/exgfs_atmos_wafs_grib.sh $fcsthrs
+export err=$?; err_chk
+
+echo "JOB $job HAS COMPLETED NORMALLY!"
+
+############################################
+# print exec output
+############################################
+if [ -e "$pgmout" ] ; then
+  cat $pgmout
+fi
+
+############################################
+# remove temporary working directory
+############################################
+if [ $KEEPDATA != YES ] ; then
+    rm -rf $DATA
+fi
+
+date