Port to Narwhal and add perf_suite (CICE-Consortium#678)

* Add narwhal intel, gnu, cray, aocc
Add perf_suite.ts

* update narwhal_cray and perf_suite

configuration/scripts/cice.batch.csh

@@ -91,7 +91,26 @@ cat >> ${jobfile} << EOFB
 #PBS -l walltime=${batchtime}
 EOFB
 
-else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*  || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang) then
+else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*  || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr* || ${ICE_MACHINE} =~ mustang*) then
+cat >> ${jobfile} << EOFB
+#PBS -N ${shortcase}
+#PBS -q ${queue}
+#PBS -A ${acct}
+#PBS -l select=${nnodes}:ncpus=${maxtpn}:mpiprocs=${taskpernode}
+#PBS -l walltime=${batchtime}
+#PBS -j oe
+#PBS -W umask=022
+###PBS -M username@domain.com
+###PBS -m be
+EOFB
+
+else if (${ICE_MACHINE} =~ narwhal*) then
+if (${runlength} <= 0) then
+  set batchtime = "00:29:59"
+  set queue = "debug"
+else
+  set queue = "standard"
+endif
 cat >> ${jobfile} << EOFB
 #PBS -N ${shortcase}
 #PBS -q ${queue}

configuration/scripts/cice.launch.csh

@@ -58,7 +58,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ onyx*) then
+else if (${ICE_MACHINE} =~ onyx* || ${ICE_MACHINE} =~ narwhal) then
 cat >> ${jobfile} << EOFR
 aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
 EOFR

configuration/scripts/machines/Macros.narwhal_aocc

@@ -0,0 +1,59 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, aocc compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DNO_R16 -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -ffixed-form
+FREEFLAGS  := -ffree-form
+FFLAGS     := -byteswapio
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fsanitize=integer-divide-by-zero,float-divide-by-zero,bounds
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -mp
+   CFLAGS += -mp
+   FFLAGS += -mp
+else
+   LDFLAGS += -nomp
+#   CFLAGS += -nomp
+   FFLAGS += -nomp
+endif
+

configuration/scripts/machines/Macros.narwhal_cray

@@ -0,0 +1,60 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, cray compiler
+#==============================================================================
+
+CPP        := ftn -e P
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -132
+FREEFLAGS  := 
+FFLAGS     := -hbyteswapio
+FFLAGS_NOOPT:= -O0
+LDFLAGS    := -hbyteswapio
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -hfp0 -g -Rbcdps -Ktrap=fp
+else
+  FFLAGS     += -O2 -hfp0     # -eo
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+else
+   LDFLAGS += -hnoomp
+#   CFLAGS += -hnoomp
+   FFLAGS += -hnoomp
+endif
+

configuration/scripts/machines/Macros.narwhal_gnu

@@ -0,0 +1,67 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS     += -O0
+endif
+
+ifeq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O0 -g -fprofile-arcs -ftest-coverage
+  CFLAGS   += -O0 -g -coverage
+  LDFLAGS  += -g -ftest-coverage -fprofile-arcs
+endif
+
+ifneq ($(ICE_BLDDEBUG), true)
+ifneq ($(ICE_COVERAGE), true)
+  FFLAGS   += -O2
+  CFLAGS   += -O2
+endif
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+

configuration/scripts/machines/Macros.narwhal_intel

@@ -0,0 +1,57 @@
+#==============================================================================
+# Macros file for NAVYDSRC narwhal, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -fcommon
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+# -mcmodel medium -shared-intel
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+

configuration/scripts/machines/env.narwhal_aocc

@@ -0,0 +1,52 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-aocc
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-nvidia
+module load PrgEnv-aocc/8.1.0
+module load cray-pals/1.0.17
+module load bct-env/0.1
+module unload aocc
+module load aocc/2.2.0.1
+module unload cray-mpich
+module load cray-mpich/8.1.5
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.7.4.4
+module load cray-hdf5/1.12.0.4
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+
+endif
+
+setenv ICE_MACHINE_MACHNAME narwhal
+setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "aocc_3.0.0-Build#78 2020_12_10 clang/flang 12.0.0, cray-mpich/8.1.9, netcdf/4.7.4.4"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 128    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "

configuration/scripts/machines/env.narwhal_cray

@@ -0,0 +1,53 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module unload PrgEnv-aocc
+module unload PrgEnv-cray
+module unload PrgEnv-gnu
+module unload PrgEnv-intel
+module unload PrgEnv-nvidia
+module load PrgEnv-cray/8.1.0
+module load cray-pals/1.0.17
+module load bct-env/0.1
+module unload cce 
+module load cce/12.0.3
+module unload cray-mpich
+module load cray-mpich/8.1.9
+
+module unload cray-hdf5
+module unload cray-hdf5-parallel
+module unload cray-netcdf-hdf5parallel
+module unload cray-parallel-netcdf
+module unload netcdf
+module load cray-netcdf/4.7.4.4
+module load cray-hdf5/1.12.0.4
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_WAIT_POLICY passive
+
+endif
+
+setenv ICE_MACHINE_MACHNAME narwhal
+setenv ICE_MACHINE_MACHINFO "Cray EX AMD EPYC 7H12"
+setenv ICE_MACHINE_ENVNAME cray
+setenv ICE_MACHINE_ENVINFO "cce 12.0.3, cray-mpich/8.1.9, netcdf/4.7.4.4"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /p/work1/projects/RASM/cice_consortium
+setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub "
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "debug"
+setenv ICE_MACHINE_TPNODE 128    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 12
+setenv ICE_MACHINE_QSTAT "qstat "