Skip to content

Commit

Permalink
Merge branch 'jasonb5/machinefiles/updates-lc-machines-modules' (PR #…
Browse files Browse the repository at this point in the history 
…6633)

* Adds dane/ruby queue definitions
* Updates dane/ruby cmake macros
* Updates dane/ruby machine definition to use slurm and new modules
* Removes lc_slurm
  • Loading branch information
@jasonb5
jasonb5 committed Sep 24, 2024
2 parents 39d5295 + 227f736 commit 83eaa7e
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 53 deletions.
6 changes: 6 additions & 0 deletions cime_config/machines/cmake_macros/intel_dane.cmake
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/")

list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib")

set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'")
6 changes: 6 additions & 0 deletions cime_config/machines/cmake_macros/intel_ruby.cmake
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/")

list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib")
list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib")

set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'")
36 changes: 10 additions & 26 deletions cime_config/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,33 +237,17 @@
</batch_system>

<!-- for lawrence livermore computing -->
<batch_system type="lc_slurm">
<batch_query per_job_arg="-j">squeue</batch_query>
<batch_submit>sbatch</batch_submit>
<batch_cancel>scancel</batch_cancel>
<batch_directive>#SBATCH</batch_directive>
<jobid_pattern>(\d+)$</jobid_pattern>
<depend_string>--dependency=afterok:jobid</depend_string>
<depend_allow_string>--dependency=afterany:jobid</depend_allow_string>
<depend_separator>:</depend_separator>
<walltime_format>%H:%M:%S</walltime_format>
<batch_mail_flag>--mail-user</batch_mail_flag>
<batch_mail_type_flag>--mail-type</batch_mail_type_flag>
<batch_mail_type>none, all, begin, end, fail</batch_mail_type>
<directives>
<directive>--export=ALL</directive>
<directive>-p {{ job_queue }}</directive>
<directive>-J {{ job_id }}</directive>
<directive>-N {{ num_nodes }}</directive>
<directive>-n {{ total_tasks }}</directive>
<directive>-t {{ job_wallclock_time }}</directive>
<directive>-o {{ job_id }}.out</directive>
<directive>-e {{ job_id }}.err</directive>
<directive>-A {{ project }}</directive>
</directives>
<batch_system MACH="ruby" type="slurm">
<queues>
<queue walltimemax="24:00:00" nodemax="520" default="true">pbatch</queue>
<queue walltimemax="01:00:00" nodemax="12">pdebug</queue>
</queues>
</batch_system>

<batch_system MACH="dane" type="slurm">
<queues>
<queue walltimemax="01:00:00" nodemax="270" default="true">pbatch</queue>
<queue walltimemax="00:30:00">pdebug</queue>
<queue walltimemax="24:00:00" nodemax="520" default="true">pbatch</queue>
<queue walltimemax="01:00:00" nodemax="20">pdebug</queue>
</queues>
</batch_system>
<!-- for lawrence livermore computing -->
Expand Down
76 changes: 49 additions & 27 deletions cime_config/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3065,9 +3065,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/workspace/e3sm/tools/cprnc</CCSM_CPRNC>
<CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>56</MAX_MPITASKS_PER_NODE>
Expand All @@ -3076,8 +3076,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
</mpirun>
<mpirun mpilib="default">
<executable>srun</executable>
<arguments>
<arg name="pmi_layer">--mpi=pmi2</arg>
<arg name="export">--export=ALL</arg>
<arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
<arg name="thread_count">-c 1</arg>
<arg name="binding">--cpu_bind=cores</arg>
<arg name="placement">-m plane={{ tasks_per_node }}</arg>
</arguments>
</mpirun>
<module_system type="module">
<module_system type="module" allow_error="true">
<init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
<init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
Expand All @@ -3089,24 +3097,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<modules compiler="intel">
<command name="load">python/3.9.12</command>
<command name="load">git</command>
<command name="load">subversion</command>
<command name="load">cmake/3.19.2</command>
<command name="load">mkl/2022.1.0</command>
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="load">mvapich2/2.3.7</command>
<command name="load">cmake/3.19.2</command>
<command name="use --append">/usr/workspace/e3sm/install/quartz/modulefiles</command>
<command name="load">hdf5/1.12.2</command>
<command name="load">netcdf-c/4.9.0</command>
<command name="load">netcdf-fortran/4.6.0</command>
<command name="load">parallel-netcdf/1.12.3</command>
<command name="load">screamML-venv/0.0.1</command>
<command name="load">subversion</command>
<command name="use --append">/usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core</command>
<command name="load">mvapich2/2.3.7-ll7cmqm</command>
<command name="load">hdf5/1.10.7-ewjpbjd</command>
<command name="load">netcdf-c/4.4.1.1-vaxofek</command>
<command name="load">netcdf-fortran/4.4.4-3pzbx2u</command>
<command name="load">parallel-netcdf/1.11.0-tzgdala</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<env name="NETCDF_PATH">/usr/workspace/e3sm/install/quartz/netcdf-fortran/</env>
<env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
<env name="OMP_STACKSIZE">128M</env>
<env name="HDF5_USE_FILE_LOCKING">FALSE</env>
<env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg</env>
<env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn</env>
<env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2</env>
<env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5</env>
</environment_variables>
</machine>

Expand All @@ -3121,9 +3132,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/workspace/e3sm/tools/cprnc</CCSM_CPRNC>
<CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>224</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
Expand All @@ -3132,8 +3143,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
</mpirun>
<mpirun mpilib="default">
<executable>srun</executable>
<arguments>
<arg name="pmi_layer">--mpi=pmi2</arg>
<arg name="export">--export=ALL</arg>
<arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
<arg name="thread_count">-c 1</arg>
<arg name="binding">--cpu_bind=cores</arg>
<arg name="placement">-m plane={{ tasks_per_node }}</arg>
</arguments>
</mpirun>
<module_system type="module">
<module_system type="module" allow_error="true">
<init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
<init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
Expand All @@ -3145,24 +3164,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<modules compiler="intel">
<command name="load">python/3.9.12</command>
<command name="load">git</command>
<command name="load">subversion</command>
<command name="load">mkl/2022.1.0</command>
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="load">mvapich2/2.3.7</command>
<command name="load">cmake/3.19.2</command>
<command name="use --append">/usr/workspace/e3sm/install/quartz/modulefiles</command>
<command name="load">hdf5/1.12.2</command>
<command name="load">netcdf-c/4.9.0</command>
<command name="load">netcdf-fortran/4.6.0</command>
<command name="load">parallel-netcdf/1.12.3</command>
<command name="load">screamML-venv/0.0.1</command>
<command name="load">subversion</command>
<command name="use --append">/usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core</command>
<command name="load">mvapich2/2.3.7-27jao34</command>
<command name="load">hdf5/1.10.7-766kapa</command>
<command name="load">netcdf-c/4.4.1.1-2uznnlw</command>
<command name="load">netcdf-fortran/4.4.4-itpstyo</command>
<command name="load">parallel-netcdf/1.11.0-26sxm4m</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<env name="NETCDF_PATH">/usr/workspace/e3sm/install/quartz/netcdf-fortran/</env>
<env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
<env name="OMP_STACKSIZE">128M</env>
<env name="HDF5_USE_FILE_LOCKING">FALSE</env>
<env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv</env>
<env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib</env>
<env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp</env>
<env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck</env>
</environment_variables>
</machine>

Expand Down

0 comments on commit 83eaa7e

Please sign in to comment.