Merge remote-tracking branch 'origin/develop' into feature/remove_FIX…

…_dir_vars * origin/develop: Add unstructured grid for HR3/GFS (NOAA-EMC#2230) Move IMS remapping files from COM_OBS to FIXgdas (NOAA-EMC#2322) Set HOMEgfs for module_setup in CI driver (NOAA-EMC#2321) Remove `finddate.sh` from system (NOAA-EMC#2308) Refs NOAA-EMC#2184
KateFriedman-NOAA · Feb 16, 2024 · d460d7d · d460d7d
2 parents 0a296b1 + cf83885
commit d460d7d
Show file tree

Hide file tree

Showing 31 changed files with 158 additions and 78 deletions.
diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -16,3 +16,8 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules
diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -17,7 +17,7 @@ arguments:
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
-skip_ci_on_host:
+skip_ci_on_hosts:
   - orion
   - hera
   - hercules
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
+ - gfs: './build_all.sh -wgu'
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -8,7 +8,7 @@ set -eux
 #                     to run from within a cron job in the CI Managers account
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -20,21 +20,22 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -x
@@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -89,13 +90,13 @@ for pr in ${pr_list}; do
     sed -i "1 i\`\`\`" "${output_ci}"
     sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
-      num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
+      num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l)
       passed=0
-      for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
+      for platforms in "${HOMEgfs}"/ci/platforms/config.*; do
         machine=$(basename "${platforms}" | cut -d. -f2)
         if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
           ((passed=passed+1))
@@ -139,7 +140,7 @@ for pr in ${pr_list}; do
       } >> "${output_ci}"
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-      "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
          cancel_slurm_jobs "${pslot}"

diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -guw  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -47,12 +47,15 @@ esac
 ######################################################
 # setup runtime env for correct python install and git
 ######################################################
+HOMEgfs=${ROOT_DIR}
+export HOMEgfs
 set +x
 source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 set -x
+unset HOMEgfs
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI

diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
@@ -9,7 +9,7 @@ set -eux
 # Abstract TODO
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -18,20 +18,21 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -eux
@@ -47,7 +48,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
 fi
 if [[ -z "${pr_list}" ]]; then
   echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"

diff --git a/docs/source/clone.rst b/docs/source/clone.rst
@@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build:
    ./build_all.sh -gu
 
 
+For building with PDLIB for the wave model, use the `-w` options during build:
+
+::
+
+   ./build_all.sh -w
+
+
 Build workflow components and link workflow artifacts such as executables, etc.
 
 ::

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     if [[ "${step}" = "fcst" ]]; then
         export OMP_PLACES=cores
         export OMP_STACKSIZE=2048M
+        export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+        export FI_OFI_RXM_SAR_LIMIT=3145728
     elif [[ "${step}" = "efcs" ]]; then
         export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
         export FI_OFI_RXM_SAR_LIMIT=3145728

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -172,16 +172,16 @@ export ICERES="${OCNRES}"
 # These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export waveGRD='glo_500'
+        export waveGRD='uglo_100km'
         ;;
     "C96" | "C192")
-        export waveGRD='glo_200'
+        export waveGRD='uglo_100km'
         ;;
     "C384")
-        export waveGRD='glo_025'
+        export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='mx025'
+        export waveGRD='uglo_m1g16'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -737,15 +737,15 @@ case ${step} in
     export npe_genesis=1
     export nth_genesis=1
     export npe_node_genesis=1
-    export memory_genesis="4G"
+    export memory_genesis="10G"
     ;;
 
   "genesis_fsu")
     export wtime_genesis_fsu="00:10:00"
     export npe_genesis_fsu=1
     export nth_genesis_fsu=1
     export npe_node_genesis_fsu=1
-    export memory_genesis_fsu="4G"
+    export memory_genesis_fsu="10G"
     ;;
 
   "fit2obs")

diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic
@@ -21,16 +21,16 @@ case "${CASE}" in
     export CPL_WAVIC=workflow_C384_refactored
     ;;
   "C768")
-    export CPL_ATMIC=HR2_refactored
-    export CPL_ICEIC=HR1_refactored
-    export CPL_OCNIC=HR1_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C768
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   "C1152")
-    export CPL_ATMIC=HR2_C1152_refactored
-    export CPL_ICEIC=HR3_refactored
-    export CPL_OCNIC=HR3_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C1152
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -15,7 +15,7 @@ if (( $# <= 1 )); then
     echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072"
     echo "--mom6 500|100|025"
     echo "--cice6 500|100|025"
-    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025"
+    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025|uglo_100km|uglo_m1g16"
     echo "--gocart"
 
     exit 1
@@ -416,6 +416,14 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "mx025")
       ntasks_ww3=80
       ;;
+    "uglo_100km")
+      ntasks_ww3=40
+      nthreads_ww3=1
+      ;;
+    "uglo_m1g16")  
+      ntasks_ww3=1000
+      nthreads_ww3=1
+      ;;
     *)
       echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!"
       exit 1

diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave
@@ -79,7 +79,19 @@ case "${waveGRD}" in
     export wavepostGRD='glo_500'
     export waveuoutpGRD=${waveGRD}
     ;;
-  *)
+  "uglo_100km") 
+    #unstructured 100km grid 
+    export waveinterpGRD='glo_200'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "uglo_m1g16")
+    #unstructured m1v16 grid 
+    export waveinterpGRD='glo_15mxt'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+   *)
     echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting."
     exit 1
     ;;

diff --git a/parm/wave/at_10m_interp.inp.tmpl b/parm/wave/at_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'at_10m'
 $
  0

diff --git a/parm/wave/ep_10m_interp.inp.tmpl b/parm/wave/ep_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'ep_10m'
 $
  0

diff --git a/parm/wave/glo_15mxt_interp.inp.tmpl b/parm/wave/glo_15mxt_interp.inp.tmpl
@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_15mxt'
 $
  0

diff --git a/parm/wave/glo_200_interp.inp.tmpl b/parm/wave/glo_200_interp.inp.tmpl
@@ -0,0 +1,12 @@
+$ Input file for interpolation of GLO30m_ext Grid
+$------------------------------------------------
+$ Start Time DT NSteps
+ TIME DT NSTEPS
+$ Total number of grids 
+ 2
+$ Grid extensions 
+  'uglo_100km'
+  'glo_200'
+$
+ 0
+$
diff --git a/parm/wave/glo_30m_interp.inp.tmpl b/parm/wave/glo_30m_interp.inp.tmpl
@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_30m'
 $
  0

diff --git a/parm/wave/wc_10m_interp.inp.tmpl b/parm/wave/wc_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'wc_10m'
 $
  0

diff --git a/scripts/exgfs_wave_init.sh b/scripts/exgfs_wave_init.sh
@@ -118,6 +118,13 @@ source "${HOMEgfs}/ush/preamble.sh"
         err=2;export err;${errchk}
       fi
 
+
+      if [ -f ${FIXwave}/${grdID}.msh ]
+      then
+        cp "${FIXwave}/${grdID}.msh" "${grdID}.msh"
+      fi
+      #TO DO: how do we say "it's unstructured, and therefore need to have error check here" 
+
       [[ ! -d "${COM_WAVE_PREP}" ]] && mkdir -m 775 -p "${COM_WAVE_PREP}"
       if [ ${CFP_MP:-"NO"} = "YES" ]; then
         echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
@@ -166,7 +173,7 @@ source "${HOMEgfs}/ush/preamble.sh"
       exit=$?
     fi
 
-    if [ "$exit" != '0' ]
+    if [[ "$exit" != '0' ]]
     then
       set +x
       echo ' '
@@ -195,9 +202,9 @@ source "${HOMEgfs}/ush/preamble.sh"
       echo '********************************************** '
       echo '*** FATAL ERROR : NO MODEL DEFINITION FILE *** '
       echo '********************************************** '
-      echo "                                grdID = $grdID"
+      echo "                                grdID = ${grdID}"
       echo ' '
-      sed "s/^/$grdID.out : /g"  $grdID.out
+      sed "s/^/${grdID}.out : /g"  "${grdID}.out"
       set_trace
       err=3;export err;${errchk}
     fi