From 26459096144e983ac8e9b59783f33a55602565f7 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Fri, 3 May 2024 16:37:00 -0500 Subject: [PATCH] Adding Kripke to Benchpark (#83) * Adding Kripke to Benchpark * adding license * adding ramble.yaml for cuda and rocm * Adding tags * Adding cuda instead of openmp * adding Kripke package.py in benchpark repo temporarily * Update license * Pulling in new Spack package, updating to point at Robert's PR * First attempt to align versions of RAJA suite components * Experiment/kripke new (#214) * OpenMP implementation of Kripke * HIP implementation of kripke * Update spack version. Pin kripke version * Delete repo/kripke/application.py * Add kripke application.py again * Remove redundant file * CUDA implementation of kripke * Upgrade cmake externals package * Update cmake external package config for LLNL-Tioga-HPECray-zen3-MI250X-Slingshot * Update lassen compiler config * Removing deprecated cmake from machine config * Removing deprecated cmake from machine config * Removing deprecated cmake from machine config * minor --------- Co-authored-by: Riyaz Haque Co-authored-by: pearce8 --------- Co-authored-by: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Co-authored-by: Riyaz Haque --- .../auxiliary_software_files/packages.yaml | 6 +- .../auxiliary_software_files/packages.yaml | 9 +- .../auxiliary_software_files/compilers.yaml | 20 ++- .../auxiliary_software_files/packages.yaml | 13 +- .../spack.yaml | 4 +- .../kripke/cuda/execute_experiment.tpl | 13 ++ experiments/kripke/cuda/ramble.yaml | 54 +++++++ .../kripke/openmp/execute_experiment.tpl | 13 ++ experiments/kripke/openmp/ramble.yaml | 57 +++++++ .../kripke/rocm/execute_experiment.tpl | 13 ++ experiments/kripke/rocm/ramble.yaml | 54 +++++++ repo/kripke/application.py | 118 ++++++++++++++ repo/kripke/package.py | 146 ++++++++++++++++++ 13 files changed, 509 insertions(+), 11 deletions(-) create mode 100755 experiments/kripke/cuda/execute_experiment.tpl create mode 100644 experiments/kripke/cuda/ramble.yaml create mode 100755 experiments/kripke/openmp/execute_experiment.tpl create mode 100644 experiments/kripke/openmp/ramble.yaml create mode 100755 experiments/kripke/rocm/execute_experiment.tpl create mode 100644 experiments/kripke/rocm/ramble.yaml create mode 100644 repo/kripke/application.py create mode 100644 repo/kripke/package.py diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml index 0cb4fc3b..7aa44dea 100644 --- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -16,9 +16,9 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 - buildable: false + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 + buildable: false gmake: externals: - spec: gmake@4.2.1 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index 1b64a0ee..129876c2 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -16,8 +16,8 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 buildable: false gmake: externals: @@ -48,6 +48,11 @@ packages: prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/ - spec: mvapich2@2.3.7-clang-14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6 + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: buildable: false externals: diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml index d27a632c..4e4c457a 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml @@ -4,6 +4,22 @@ # SPDX-License-Identifier: Apache-2.0 compilers: +- compiler: + spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1 + paths: + cc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlc + cxx: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlC + f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + flags: + cflags: -g -O2 + cxxflags: -g -O2 -std=c++14 + fflags: -g -O2 + operating_system: rhel7 + target: ppc64le + modules: [cuda/11.8.0, xl/2023.06.28-cuda-11.8.0-gcc-11.2.1] + environment: {} + extra_rpaths: [] - compiler: spec: xl@16.1.1-2022.08.19-cuda-11.7.0 paths: @@ -29,7 +45,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf flags: cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le @@ -45,7 +61,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf flags: # TODO: Fix spack concretization bug cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml index 61c85f62..5d26658c 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml @@ -11,8 +11,8 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 + - spec: cmake@3.29.2 + prefix: /usr/tce/packages/cmake/cmake-3.29.2 buildable: false gmake: externals: @@ -41,6 +41,11 @@ packages: - spec: cusparse@11.8.0 prefix: /usr/tce/packages/cuda/cuda-11.8.0 buildable: false + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: externals: - spec: cuda@10.1.243 @@ -72,6 +77,10 @@ packages: buildable: false mpi: externals: + - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1 + prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1 + extra_attributes: + ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm" - spec: spectrum-mpi@2022.08.19-cuda-11.8.0 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0 extra_attributes: diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml index ed1edcb6..7b2f4257 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml @@ -6,9 +6,9 @@ spack: packages: default-compiler: - spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version} + spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 default-mpi: - spack_spec: spectrum-mpi@2022.08.19 + spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 compiler-gcc: spack_spec: gcc@8.3.1 compiler-clang: diff --git a/experiments/kripke/cuda/execute_experiment.tpl b/experiments/kripke/cuda/execute_experiment.tpl new file mode 100755 index 00000000..89e73cf4 --- /dev/null +++ b/experiments/kripke/cuda/execute_experiment.tpl @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{command} diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml new file mode 100644 index 00000000..b4771bdb --- /dev/null +++ b/experiments/kripke/cuda/ramble.yaml @@ -0,0 +1,54 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + include: + - ./configs/spack.yaml + - ./configs/variables.yaml + - ./configs/modifier.yaml + + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + kripke: + workloads: + kripke: + env_vars: + variables: + n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: '2' + n_nodes: '2' + arch: 'CUDA' + experiments: + kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: + variables: + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '32' + npx: '2' + npy: '2' + npz: '1' + spack: + concretized: true + packages: + kripke: + spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02 + compiler: default-compiler + environments: + kripke: + packages: + - default-mpi + - kripke + - '{modifier_package_name}' diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl new file mode 100755 index 00000000..89e73cf4 --- /dev/null +++ b/experiments/kripke/openmp/execute_experiment.tpl @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{command} diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml new file mode 100644 index 00000000..afdbaa12 --- /dev/null +++ b/experiments/kripke/openmp/ramble.yaml @@ -0,0 +1,57 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + include: + - ./configs/spack.yaml + - ./configs/variables.yaml + - ./configs/modifier.yaml + + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + kripke: + workloads: + kripke: + env_vars: + set: + OMP_NUM_THREADS: '{omp_num_threads}' + variables: + n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: ['1', '16'] + n_nodes: '1' + omp_num_threads: ['32', '2'] + arch: 'OpenMP' + experiments: + kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: + variables: + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '32' + nzz: '32' + npx: ['1', '4'] + npy: ['1', '2'] + npz: ['1', '2'] + spack: + concretized: true + packages: + kripke: + spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02 + compiler: default-compiler + environments: + kripke: + packages: + - default-mpi + - kripke + - '{modifier_package_name}' diff --git a/experiments/kripke/rocm/execute_experiment.tpl b/experiments/kripke/rocm/execute_experiment.tpl new file mode 100755 index 00000000..89e73cf4 --- /dev/null +++ b/experiments/kripke/rocm/execute_experiment.tpl @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{command} diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml new file mode 100644 index 00000000..7439b57d --- /dev/null +++ b/experiments/kripke/rocm/ramble.yaml @@ -0,0 +1,54 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + include: + - ./configs/spack.yaml + - ./configs/variables.yaml + - ./configs/modifier.yaml + + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + kripke: + workloads: + kripke: + env_vars: + variables: + n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: ['8'] + n_nodes: '1' + arch: 'HIP' + experiments: + kripke_rocm_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: + variables: + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '64' + npx: '2' + npy: '2' + npz: '2' + spack: + concretized: true + packages: + kripke: + spack_spec: kripke@1.2.7.0 +mpi+rocm{modifier_spack_variant} ^chai@2024.02 + compiler: default-compiler + environments: + kripke: + packages: + - default-mpi + - kripke + - '{modifier_package_name}' diff --git a/repo/kripke/application.py b/repo/kripke/application.py new file mode 100644 index 00000000..61a79f7f --- /dev/null +++ b/repo/kripke/application.py @@ -0,0 +1,118 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from ramble.appkit import * + +import sys + +class Kripke(SpackApplication): + """Kripke benchmark uses RAJA Portability Layer""" + name = "Kripke" + + tags = ['asc','transport','deterministic','structured-grid', + 'large-scale','multi-node','single-node','c++','raja', + 'simd','vectorization','register-pressure','high-fp','atomics','high-branching', + 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', + 'mpi','network-latency-bound','network-collectives'] + + executable('kripke', 'kripke.exe' + + ' --groups {ngroups}' + + ' --legendre {lorder}' + + ' --quad {nquad}' + + ' --zones {nzx},{nzy},{nzz}' + + ' --sigt {sigt0},{sigt1},{sigt2}' + + ' --sigs {sigs0},{sigs1},{sigs2}' + + ' --arch {arch}' + + ' --layout {layout}' + +# ' --pdist {lout}' + + ' --procs {npx},{npy},{npz}' + + ' --dset {ds}' + + ' --gset {gs}' + + ' --zset {nzsetx},{nzsety},{nzsetz}' + + ' --niter {niter}' + + ' --pmethod {method}', use_mpi=True) + + workload('kripke', executables=['kripke']) + + workload_variable('ngroups', default='32', + description='Number of energy groups. (Default: --groups 32)', + workloads=['kripke']) + workload_variable('lorder', default='4', + description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', + workloads=['kripke']) + workload_variable('nquad', default='96', + description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in x. (Default: 16)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in y. (Default: 16)', + workloads=['kripke']) + workload_variable('nzz', default='16', + description='Number of zones in z. (Default: 16)', + workloads=['kripke']) + workload_variable('sigt0', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt1', default='0.0001', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt2', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs0', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs1', default='0.00005', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs2', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('arch', default='OpenMP', + description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', + workloads=['kripke']) + workload_variable('layout', default='DGZ', + description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', + workloads=['kripke']) +# workload_variable('lout', default='0', +# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', +# workloads=['kripke']) + workload_variable('npx', default='1', + description='Number of MPI ranks in x dimension', + workloads=['kripke']) + workload_variable('npy', default='1', + description='Number of MPI ranks in y dimension', + workloads=['kripke']) + workload_variable('npz', default='1', + description='Number of MPI ranks in z dimension', + workloads=['kripke']) + workload_variable('ds', default='8', + description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', + workloads=['kripke']) + workload_variable('gs', default='1', + description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', + workloads=['kripke']) + workload_variable('nzsetx', default='1', + description='Number of zone-sets in x', + workloads=['kripke']) + workload_variable('nzsety', default='1', + description='Number of zone-sets in y', + workloads=['kripke']) + workload_variable('nzsetz', default='1', + description='Number of zone-sets in z', + workloads=['kripke']) + workload_variable('niter', default='10', + description='Number of solver iterations to run. (Default: --niter 10)', + workloads=['kripke']) + workload_variable('method', default='sweep', + description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', + workloads=['kripke']) + + #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + + #TODO: Fix the FOM success_criteria(...) + #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/kripke/package.py b/repo/kripke/package.py new file mode 100644 index 00000000..7f126955 --- /dev/null +++ b/repo/kripke/package.py @@ -0,0 +1,146 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * + + +class Kripke(CMakePackage, CudaPackage, ROCmPackage): + """Kripke is a simple, scalable, 3D Sn deterministic particle + transport proxy/mini app. + """ + + homepage = "https://computing.llnl.gov/projects/co-design/kripke" + git = "https://github.com/LLNL/Kripke.git" + + tags = ["proxy-app"] + + maintainers("vsrana01") + + license("BSD-3-Clause") + + version("develop", branch="develop", submodules=False) + version( + "1.2.7.0", submodules=False, commit="db920c1f5e1dcbb9e949d120e7d86efcdb777635" + ) + version( + "1.2.4", submodules=False, tag="v1.2.4", commit="d85c6bc462f17a2382b11ba363059febc487f771" + ) + version( + "1.2.3", submodules=True, tag="v1.2.3", commit="66046d8cd51f5bcf8666fd8c810322e253c4ce0e" + ) + version( + "1.2.2", + submodules=True, + tag="v1.2.2-CORAL2", + commit="a12bce71e751f8f999009aa2fd0839b908b118a4", + ) + version( + "1.2.1", + submodules=True, + tag="v1.2.1-CORAL2", + commit="c36453301ddd684118bb0fb426cfa62764d42398", + ) + version( + "1.2.0", + submodules=True, + tag="v1.2.0-CORAL2", + commit="67e4b0a2f092009d61f44b5122111d388a3bec2a", + ) + + variant("mpi", default=True, description="Build with MPI.") + variant("openmp", default=False, description="Build with OpenMP enabled.") + variant("caliper", default=False, description="Build with Caliper support enabled.") + + depends_on("mpi", when="+mpi") + depends_on("caliper", when="+caliper") + depends_on("adiak@0.4:", when="+caliper") + conflicts("^blt@:0.3.6", when="+rocm") + + aligned_versions = ["2024.02"] + for v in aligned_versions: + depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") + depends_on(f"umpire@{v}~examples", when=f"^chai@{v}") + depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}") + depends_on(f"camp@{v}", when=f"^chai@{v}") + + depends_on("blt@0.6.2:", type="build", when=f"@1.2.7:") + + depends_on("chai+openmp", when="+openmp") + depends_on("chai~openmp", when="~openmp") + depends_on("chai+cuda", when="+cuda") + depends_on("chai~cuda", when="~cuda") + depends_on("chai+rocm", when="+rocm") + depends_on("chai~rocm", when="~rocm") + + depends_on("umpire+openmp", when="+openmp") + depends_on("umpire~openmp", when="~openmp") + depends_on("umpire+cuda", when="+cuda") + depends_on("umpire~cuda", when="~cuda") + depends_on("umpire+rocm", when="+rocm") + depends_on("umpire~rocm", when="~rocm") + + def setup_build_environment(self, env): + spec = self.spec + if "+cuda" in spec: + env.set("CUDAHOSTCXX", self.spec["mpi"].mpicxx) + + def cmake_args(self): + spec = self.spec + args = [] + + args.extend( + [ + "-DCAMP_DIR=%s" % self.spec["camp"].prefix, + "-DBLT_SOURCE_DIR=%s" % self.spec["blt"].prefix, + "-Dumpire_DIR=%s" % self.spec["umpire"].prefix, + "-DRAJA_DIR=%s" % self.spec["raja"].prefix, + "-Dchai_DIR=%s" % self.spec["chai"].prefix, + "-DENABLE_CHAI=ON", + ] + ) + + if "+openmp" in spec: + args.append("-DENABLE_OPENMP=ON") + + if "+caliper" in spec: + args.append("-DENABLE_CALIPER=ON") + + if "+mpi" in spec: + args.append("-DENABLE_MPI=ON") + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["mpi"].mpicxx)) + + if "+rocm" in spec: + # Set up the hip macros needed by the build + args.append("-DENABLE_HIP=ON") + args.append("-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)) + rocm_archs = spec.variants["amdgpu_target"].value + if "none" not in rocm_archs: + args.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(",".join(rocm_archs))) + args.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(rocm_archs)) + else: + # Ensure build with hip is disabled + args.append("-DENABLE_HIP=OFF") + + if "+cuda" in spec: + args.append("-DENABLE_CUDA=ON") + args.append(self.define("CMAKE_CUDA_HOST_COMPILER", self.spec["mpi"].mpicxx)) + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + args.append("-DCUDA_ARCH={0}".format(cuda_arch[0])) + args.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) + args.append( + "-DCMAKE_CUDA_FLAGS=--extended-lambda -I%s -I=%s" + % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include) + ) + else: + args.append("-DENABLE_CUDA=OFF") + + return args + + def install(self, spec, prefix): + # Kripke does not provide install target, so we have to copy + # things into place. + mkdirp(prefix.bin) + install(join_path(self.build_directory, "kripke.exe"), prefix.bin)