update

.cicd/Jenkinsfile

@@ -16,7 +16,8 @@ pipeline {
         choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
-        booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
+        // Uncomment the following line to re-enable comprehensive tests
+        // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
     }
 
     stages {
@@ -89,7 +90,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {
@@ -99,6 +100,10 @@ pipeline {
                     }
                 }
 
+                agent {
+                    label env.SRW_PLATFORM
+                }
+
                 environment {
                     BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
                     BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
@@ -109,12 +114,8 @@ pipeline {
                 stages {
                     // Clean the workspace, checkout the repository, and run checkout_externals
                     stage('Initialize') {
-                        agent {
-                            label env.SRW_PLATFORM
-                        }
-
                         steps {
-                            echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM}"
+                            echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
                             cleanWs()
                             checkout scm
                             sh '"${WORKSPACE}/manage_externals/checkout_externals"'
@@ -123,12 +124,8 @@ pipeline {
 
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
-                        agent {
-                            label env.SRW_PLATFORM
-                        }
-
                         steps {
-                            echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
+                            echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
                             sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"'
                         }
 
@@ -142,17 +139,18 @@ pipeline {
 
                     // Run the unified test script
                     stage('Test') {
-                        agent {
-                            label env.SRW_PLATFORM
-                        }
-
                         environment {
                             SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/expt_dirs"
                         }
 
                         steps {
-                            echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
+                            echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
+
+                            // Remove the following line to re-enable comprehensive tests
+                            sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
 
+                            // Uncomment the following block to re-enable comprehensive tests
+                            /*
                             // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
                             // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
                             script {
@@ -169,12 +167,13 @@ pipeline {
 
                                 sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
                             }
+                            */
                         }
 
                         post {
                             always {
                                 // Archive the test log files
-                                sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
+                                sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
                                 // Remove the data sets from the experiments directory to conserve disk space
                                 sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf'
                                 s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []

.cicd/scripts/srw_build.sh

@@ -25,12 +25,16 @@ else
 fi
 
 # Build and install
-cd ${workspace}/test
+cd ${workspace}/tests
+set +e
 ./build.sh ${platform} ${SRW_COMPILER}
+build_exit=$?
+set -e
 cd -
 
 # Create combined log file for upload to s3
 build_dir="${workspace}/build_${SRW_COMPILER}"
 cat ${build_dir}/log.cmake ${build_dir}/log.make \
     >${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log
 
+exit $build_exit

.cicd/scripts/srw_test.sh

@@ -28,6 +28,7 @@ fi
 # Test directories
 we2e_experiment_base_dir="${workspace}/expt_dirs"
 we2e_test_dir="${workspace}/tests/WE2E"
+nco_dir="${workspace}/nco_dirs"
 
 # Run the end-to-end tests.
 if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
@@ -37,15 +38,26 @@ else
 fi
 
 cd ${we2e_test_dir}
-./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} ${we2e_experiment_base_dir}
+./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \
+    expt_basedir=${we2e_experiment_base_dir} \
+    opsroot=${nco_dir}
 
-# Allow the tests to start before checking for status.
-# TODO: Create a parameter that sets the initial start delay.
-sleep 300
+# Run the new run_srw_tests script if the machine is Cheyenne.
+if [[ "${platform}" = "cheyenne" ]]; then
+    cd ${workspace}/ush
+    ./run_srw_tests.py -e=${we2e_experiment_base_dir}
+    cd ${we2e_test_dir}
+fi
 
 # Progress file
 progress_file="${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt"
 
+# Allow the tests to start before checking for status.
+# TODO: Create a parameter that sets the initial start delay.
+if [[ "${platform}" != "cheyenne" ]]; then
+    sleep 300
+fi
+
 # Wait for all tests to complete.
 while true; do
 
@@ -71,10 +83,15 @@ done
 # TODO: Create parameter that sets the interval for the we2e cron jobs; this
 # value should be some factor of that interval to ensure the cron jobs execute
 # before the workspace is cleaned up.
-sleep 600
+if [[ "${platform}" != "cheyenne" ]]; then
+    sleep 600
+fi
 
 # Set exit code to number of failures
 set +e
 failures=$(grep "Workflow status:  FAILURE" ${progress_file} | wc -l)
+if [[ $failures -ne 0 ]]; then
+    failures=1
+fi
 set -e
 exit ${failures}

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 52072c5
+hash = 84b28ec
 local_path = sorc/ufs-weather-model
 required = True
 

README.md

@@ -1,15 +1,18 @@
 # UFS Short-Range Weather Application
 
-The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader weather enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
+The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader Weather Enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
 
-The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.0.0) represents a snapshot of this continuously evolving system. 
+The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.1.0) represents a snapshot of this continuously evolving system. 
 
-The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.0.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
+The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.1.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.1.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
 
 For instructions on how to clone the repository, build the code, and run the workflow, see:
 https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started
 
-UFS Development Team. (2022, June 23). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.0.0). Zenodo. https://doi.org/10.5281/zenodo.6505854
+For a debugging guide for users and developers in the field of Earth System Modeling, please see:
+https://epic.noaa.gov/wp-content/uploads/2022/12/Debugging-Guide.pdf
+
+UFS Development Team. (2022, Nov. 17). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.1.0). Zenodo. https://doi.org/10.5281/zenodo.7277602
 
 [![Python unittests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml)
 [![Python functional tests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml)

devbuild.sh

@@ -33,6 +33,8 @@ OPTIONS
       does a "make clean"
   --build
       does a "make" (build only)
+  --move
+      move binaries to final location.
   --build-dir=BUILD_DIR
       build directory
   --install-dir=INSTALL_DIR
@@ -125,6 +127,7 @@ BUILD_RRFS_UTILS="off"
 # Make options
 CLEAN=false
 BUILD=false
+MOVE=false
 USE_SUB_MODULES=false #change default to true later
 
 # process required arguments
@@ -155,6 +158,7 @@ while :; do
     --continue=?*|--continue=) usage_error "$1 argument ignored." ;;
     --clean) CLEAN=true ;;
     --build) BUILD=true ;;
+    --move) MOVE=true ;;
     --build-dir=?*) BUILD_DIR=${1#*=} ;;
     --build-dir|--build-dir=) usage_error "$1 requires argument." ;;
     --install-dir=?*) INSTALL_DIR=${1#*=} ;;
@@ -245,7 +249,7 @@ fi
 
 # set MODULE_FILE for this platform/compiler combination
 MODULE_FILE="build_${PLATFORM}_${COMPILER}"
-if [ ! -f "${SRW_DIR}/modulefiles/${MODULE_FILE}" ]; then
+if [ ! -f "${SRW_DIR}/modulefiles/${MODULE_FILE}.lua" ]; then
   printf "ERROR: module file does not exist for platform/compiler\n" >&2
   printf "  MODULE_FILE=${MODULE_FILE}\n" >&2
   printf "  PLATFORM=${PLATFORM}\n" >&2
@@ -398,18 +402,31 @@ module list
 mkdir -p ${BUILD_DIR}
 cd ${BUILD_DIR}
 
-printf "... Generate CMAKE configuration ...\n"
-cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake
-
 if [ "${CLEAN}" = true ]; then
-    printf "... Clean executables ...\n"
-    make ${MAKE_SETTINGS} clean 2>&1 | tee log.make
+    if [ -f $PWD/Makefile ]; then
+       printf "... Clean executables ...\n"
+       make ${MAKE_SETTINGS} clean 2>&1 | tee log.make
+    fi
 elif [ "${BUILD}" = true ]; then
+    printf "... Generate CMAKE configuration ...\n"
+    cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake
+
     printf "... Compile executables ...\n"
     make ${MAKE_SETTINGS} build 2>&1 | tee log.make
 else
+    printf "... Generate CMAKE configuration ...\n"
+    cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake
+
     printf "... Compile and install executables ...\n"
     make ${MAKE_SETTINGS} install 2>&1 | tee log.make
+
+    if [ "${MOVE}" = true ]; then
+       if [[ ! ${SRW_DIR} -ef ${INSTALL_DIR} ]]; then
+           printf "... Moving executables to final locations ...\n"
+           mkdir -p ${SRW_DIR}/${BIN_DIR}
+           mv ${INSTALL_DIR}/${BIN_DIR}/* ${SRW_DIR}/${BIN_DIR}
+       fi
+    fi
 fi
 
 exit 0