Adding Kripke to Benchpark (#83)

* Adding Kripke to Benchpark * adding license * adding ramble.yaml for cuda and rocm * Adding tags * Adding cuda instead of openmp * adding Kripke package.py in benchpark repo temporarily * Update license * Pulling in new Spack package, updating to point at Robert's PR * First attempt to align versions of RAJA suite components * Experiment/kripke new (#214) * OpenMP implementation of Kripke * HIP implementation of kripke * Update spack version. Pin kripke version * Delete repo/kripke/application.py * Add kripke application.py again * Remove redundant file * CUDA implementation of kripke * Upgrade cmake externals package * Update cmake external package config for LLNL-Tioga-HPECray-zen3-MI250X-Slingshot * Update lassen compiler config * Removing deprecated cmake from machine config * Removing deprecated cmake from machine config * Removing deprecated cmake from machine config * minor --------- Co-authored-by: Riyaz Haque <your-email@example.com> Co-authored-by: pearce8 <pearce8@llnl.gov> --------- Co-authored-by: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Co-authored-by: Riyaz Haque <your-email@example.com>
LLNL · May 3, 2024 · 2645909 · 2645909
1 parent 7a4f6b6
commit 2645909
Show file tree

Hide file tree

Showing 13 changed files with 509 additions and 11 deletions.
diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml
@@ -16,9 +16,9 @@ packages:
       buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
-      buildable: false
+    - spec: cmake@3.26.3
+      prefix: /usr/tce/packages/cmake/cmake-3.26.3
+    buildable: false
   gmake:
     externals:
     - spec: gmake@4.2.1

diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml
@@ -16,8 +16,8 @@ packages:
     buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    - spec: cmake@3.26.3
+      prefix: /usr/tce/packages/cmake/cmake-3.26.3
     buildable: false
   gmake:
     externals:
@@ -48,6 +48,11 @@ packages:
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/
     - spec: mvapich2@2.3.7-clang-14.0.6
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6
+  cub:
+    buildable: false
+    externals:
+    - spec: cub@11.8.0
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0/
   cuda:
     buildable: false
     externals:

diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
@@ -4,6 +4,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 compilers:
+- compiler:
+    spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1
+    paths:
+      cc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlc
+      cxx: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlC
+      f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf
+      fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++14
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/11.8.0, xl/2023.06.28-cuda-11.8.0-gcc-11.2.1]
+    environment: {}
+    extra_rpaths: []
 - compiler:
     spec: xl@16.1.1-2022.08.19-cuda-11.7.0
     paths:
@@ -29,7 +45,7 @@ compilers:
       fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
     flags:
       cflags: -g -O2
-      cxxflags: -g -O2 -std=c++11
+      cxxflags: -g -O2 -std=c++14
       fflags: -g -O2
     operating_system: rhel7
     target: ppc64le
@@ -45,7 +61,7 @@ compilers:
       fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf
     flags: # TODO: Fix spack concretization bug
       cflags: -g -O2
-      cxxflags: -g -O2 -std=c++11
+      cxxflags: -g -O2 -std=c++14
       fflags: -g -O2
     operating_system: rhel7
     target: ppc64le

diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
@@ -11,8 +11,8 @@ packages:
     buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    - spec: cmake@3.29.2
+      prefix: /usr/tce/packages/cmake/cmake-3.29.2
     buildable: false
   gmake:
     externals:
@@ -41,6 +41,11 @@ packages:
     - spec: cusparse@11.8.0
       prefix: /usr/tce/packages/cuda/cuda-11.8.0
     buildable: false
+  cub:
+    buildable: false
+    externals:
+    - spec: cub@11.8.0
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0/
   cuda:
     externals:
     - spec: cuda@10.1.243
@@ -72,6 +77,10 @@ packages:
       buildable: false
   mpi:
     externals:
+    - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
     - spec: spectrum-mpi@2022.08.19-cuda-11.8.0
       prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0
       extra_attributes:

diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
@@ -6,9 +6,9 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version}
+      spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
     default-mpi:
-      spack_spec: spectrum-mpi@2022.08.19
+      spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
     compiler-gcc:
       spack_spec: gcc@8.3.1
     compiler-clang:

diff --git a/experiments/kripke/cuda/execute_experiment.tpl b/experiments/kripke/cuda/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml
@@ -0,0 +1,54 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: '2'
+            n_nodes: '2'
+            arch: 'CUDA'
+          experiments:
+            kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '64'
+                nzz: '32'
+                npx: '2'
+                npy: '2'
+                npz: '1'
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'
diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml
@@ -0,0 +1,57 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+            set:
+              OMP_NUM_THREADS: '{omp_num_threads}'
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: ['1', '16']
+            n_nodes: '1'
+            omp_num_threads: ['32', '2']
+            arch: 'OpenMP'
+          experiments:
+            kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '32'
+                nzz: '32'
+                npx: ['1', '4']
+                npy: ['1', '2']
+                npz: ['1', '2']
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'
diff --git a/experiments/kripke/rocm/execute_experiment.tpl b/experiments/kripke/rocm/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml
@@ -0,0 +1,54 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: ['8']
+            n_nodes: '1'
+            arch: 'HIP'
+          experiments:
+            kripke_rocm_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '64'
+                nzz: '64'
+                npx: '2'
+                npy: '2'
+                npz: '2'
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+rocm{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'