Update JEDI ATM to use .nc for obs and generalize x,y layout (NOAA-EM…

…C#2336)

# Description
The changes in this PR are twofold
1. replace `.nc4` suffix for JEDI ATM observation related files with
`.nc`
2. use templated variables to specify `{layout_x, layout_y}` for JEDI
ATM variational and local ensemble apps

The first change conforms with the Unidata recommendation that netCDF
files end with the suffix `.nc`. The second change replaces hardwired
JEDI ATM var and ens `{layout_x, layout_y}` in `config.resources` with a
more flexible approach.

Resolves NOAA-EMC#2335

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -5,10 +5,16 @@ base:
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
   ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+atmanl:
+  LAYOUT_X_ATMANL: 1
+  LAYOUT_Y_ATMANL: 1
+atmensanl:
+  LAYOUT_X_ATMENSANL: 1
+  LAYOUT_Y_ATMENSANL: 1
 esfc:
   DONST: "NO"
 nsst:
   NST_MODEL: "1"
 sfcanl:
   DONST: "NO"
-
+

parm/config/gfs/config.atmanl

@@ -13,6 +13,9 @@ export STATICB_TYPE="gsibec"
 export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/berror/staticb_${STATICB_TYPE}.yaml
 export INTERP_METHOD='barycentric'
 
+export layout_x_atmanl=@LAYOUT_X_ATMANL@
+export layout_y_atmanl=@LAYOUT_Y_ATMANL@
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 

parm/config/gfs/config.atmensanl

@@ -10,6 +10,9 @@ export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml
 export ATMENSYAML=${HOMEgfs}/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml
 export INTERP_METHOD='barycentric'
 
+export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
+export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 

parm/config/gfs/config.resources

@@ -195,9 +195,8 @@ case ${step} in
     ;;
 
   "atmanlinit")
-    # make below case dependent later
-    export layout_x=8
-    export layout_y=8
+    export layout_x=${layout_x_atmanl}
+    export layout_y=${layout_y_atmanl}
 
     export layout_gsib_x=$(( layout_x * 3 ))
     export layout_gsib_y=$(( layout_y * 2 ))
@@ -211,9 +210,8 @@ case ${step} in
     ;;
 
   "atmanlrun")
-    # make below case dependent later
-    export layout_x=8
-    export layout_y=8
+    export layout_x=${layout_x_atmanl}
+    export layout_y=${layout_y_atmanl}
 
     export wtime_atmanlrun="00:30:00"
     export npe_atmanlrun=$(( layout_x * layout_y * 6 ))
@@ -829,9 +827,8 @@ case ${step} in
     ;;
 
   "atmensanlinit")
-    # make below case dependent later
-    export layout_x=8
-    export layout_y=8
+    export layout_x=${layout_x_atmensanl}
+    export layout_y=${layout_y_atmensanl}
 
     export wtime_atmensanlinit="00:10:00"
     export npe_atmensanlinit=1
@@ -841,9 +838,8 @@ case ${step} in
     ;;
 
   "atmensanlrun")
-    # make below case dependent later
-    export layout_x=8
-    export layout_y=8
+    export layout_x=${layout_x_atmensanl}
+    export layout_y=${layout_y_atmensanl}
 
     export wtime_atmensanlrun="00:30:00"
     export npe_atmensanlrun=$(( layout_x * layout_y * 6 ))

parm/config/gfs/yaml/defaults.yaml

@@ -9,10 +9,14 @@ base:
   FHMAX_GFS: 120
 
 atmanl:
+  LAYOUT_X_ATMANL: 8
+  LAYOUT_Y_ATMANL: 8
   IO_LAYOUT_X: 1
   IO_LAYOUT_Y: 1
 
 atmensanl:
+  LAYOUT_X_ATMENSANL: 8
+  LAYOUT_Y_ATMENSANL: 8
   IO_LAYOUT_X: 1
   IO_LAYOUT_Y: 1
 

ush/python/pygfs/task/analysis.py

@@ -99,7 +99,7 @@ def get_bias_dict(self) -> Dict[str, Any]:
                 obdir = os.path.dirname(obfile)
                 basename = os.path.basename(obfile)
                 prefix = '.'.join(basename.split('.')[:-2])
-                for file in ['satbias.nc4', 'satbias_cov.nc4', 'tlapse.txt']:
+                for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']:
                     bfile = f"{prefix}.{file}"
                     copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
 
@@ -311,13 +311,13 @@ def tgz_diags(statfile: str, diagdir: str) -> None:
         Parameters
         ----------
         statfile : str | os.PathLike
-            Path to the output .tar.gz .tgz file that will contain the diag*.nc4 files e.g. atmstat.tgz
+            Path to the output .tar.gz .tgz file that will contain the diag*.nc files e.g. atmstat.tgz
         diagdir : str | os.PathLike
             Directory containing JEDI diag files
         """
 
         # get list of diag files to put in tarball
-        diags = glob.glob(os.path.join(diagdir, 'diags', 'diag*nc4'))
+        diags = glob.glob(os.path.join(diagdir, 'diags', 'diag*nc'))
 
         logger.info(f"Compressing {len(diags)} diag files to {statfile}")
 

ush/python/pygfs/task/atm_analysis.py

@@ -152,7 +152,7 @@ def finalize(self: Analysis) -> None:
         atmstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat")
 
         # get list of diag files to put in tarball
-        diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc4'))
+        diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc'))
 
         logger.info(f"Compressing {len(diags)} diag files to {atmstat}.gz")
 

ush/python/pygfs/task/atmens_analysis.py

@@ -188,7 +188,7 @@ def finalize(self: Analysis) -> None:
         atmensstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX}atmensstat")
 
         # get list of diag files to put in tarball
-        diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc4'))
+        diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc'))
 
         logger.info(f"Compressing {len(diags)} diag files to {atmensstat}.gz")