From e88c8830c6eedc9d65d313dd5643f55530df7e1a Mon Sep 17 00:00:00 2001 From: Arkadiusz Szczepkowicz Date: Mon, 14 Aug 2023 16:12:45 +0200 Subject: [PATCH] Panzer: Remove use of hardcoded MPI_COMM_WORLD --- .../Panzer_STK_Quad8ToQuad4MeshFactory.hpp | 3 +- ...anzer_STK_QuadraticToLinearMeshFactory.hpp | 3 +- .../panzer/core/src/Panzer_GlobalComm.hpp | 69 +++++++++++++++++++ .../panzer/core/src/Panzer_PauseToAttach.hpp | 3 +- .../disc-fe/src/Panzer_FaceToElement.hpp | 2 +- .../disc-fe/src/Panzer_FaceToElement_impl.hpp | 3 +- 6 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 packages/panzer/core/src/Panzer_GlobalComm.hpp diff --git a/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Quad8ToQuad4MeshFactory.hpp b/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Quad8ToQuad4MeshFactory.hpp index 816c3a49a7fa..5ddec493ea35 100644 --- a/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Quad8ToQuad4MeshFactory.hpp +++ b/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Quad8ToQuad4MeshFactory.hpp @@ -45,6 +45,7 @@ #include #include +#include #include #include @@ -61,7 +62,7 @@ class Quad8ToQuad4MeshFactory : public STK_MeshFactory { public: Quad8ToQuad4MeshFactory(const std::string& quad8MeshFileName, - stk::ParallelMachine mpi_comm = MPI_COMM_WORLD, + stk::ParallelMachine mpi_comm = panzer::get_global_comm(), const bool print_debug = false); Quad8ToQuad4MeshFactory(const Teuchos::RCP& quad8Mesh, diff --git a/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_QuadraticToLinearMeshFactory.hpp b/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_QuadraticToLinearMeshFactory.hpp index fe3fc975e5da..7c171ac23394 100644 --- a/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_QuadraticToLinearMeshFactory.hpp +++ b/packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_QuadraticToLinearMeshFactory.hpp @@ -45,6 +45,7 @@ #include #include +#include #include #include @@ -60,7 +61,7 @@ class QuadraticToLinearMeshFactory : public STK_MeshFactory { public: QuadraticToLinearMeshFactory(const std::string& quadMeshFileName, - stk::ParallelMachine mpi_comm = MPI_COMM_WORLD, + stk::ParallelMachine mpi_comm = panzer::get_global_comm(), const bool print_debug = false); QuadraticToLinearMeshFactory(const Teuchos::RCP& quadMesh, diff --git a/packages/panzer/core/src/Panzer_GlobalComm.hpp b/packages/panzer/core/src/Panzer_GlobalComm.hpp new file mode 100644 index 000000000000..2df0d7c64745 --- /dev/null +++ b/packages/panzer/core/src/Panzer_GlobalComm.hpp @@ -0,0 +1,69 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef PANZER_GLOBAL_COMM_HPP +#define PANZER_GLOBAL_COMM_HPP + +#ifdef HAVE_MPI + +#include +#include + +namespace panzer { + +static std::mutex mpi_mutex; +static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD; + +inline void initialize_global_comm(MPI_Comm comm) { + std::lock_guard guard(mpi_mutex); + Global_MPI_Comm = comm; +} + +inline MPI_Comm get_global_comm() { + std::lock_guard guard(mpi_mutex); + return Global_MPI_Comm; +} + +} + +#endif // HAVE_MPI +#endif /* PANZER_GLOBAL_COMM_HPP */ diff --git a/packages/panzer/core/src/Panzer_PauseToAttach.hpp b/packages/panzer/core/src/Panzer_PauseToAttach.hpp index cd35e638939b..652593d9f718 100644 --- a/packages/panzer/core/src/Panzer_PauseToAttach.hpp +++ b/packages/panzer/core/src/Panzer_PauseToAttach.hpp @@ -43,6 +43,7 @@ #ifndef PANZER_PAUSE_TO_ATTACH #define PANZER_PAUSE_TO_ATTACH +#include "Panzer_GlobalComm.hpp" #include "Teuchos_RCP.hpp" #include "Teuchos_DefaultMpiComm.hpp" #include "Teuchos_FancyOStream.hpp" @@ -74,7 +75,7 @@ namespace panzer { inline void pauseToAttach() { - pauseToAttach(MPI_COMM_WORLD); + pauseToAttach(panzer::get_global_comm()); } diff --git a/packages/panzer/disc-fe/src/Panzer_FaceToElement.hpp b/packages/panzer/disc-fe/src/Panzer_FaceToElement.hpp index 0a58faef3437..6b5b05f256d1 100644 --- a/packages/panzer/disc-fe/src/Panzer_FaceToElement.hpp +++ b/packages/panzer/disc-fe/src/Panzer_FaceToElement.hpp @@ -87,7 +87,7 @@ class FaceToElement { const Teuchos::RCP> comm); #ifndef PANZER_HIDE_DEPRECATED_CODE - /** Build the mapping from a mesh topology using MPI_COMM_WORLD. + /** Build the mapping from a mesh topology using panzer::get_global_comm(). * This method is deprecated in favor of initialize(conn, comm) which * explicitly specifies the communicator. This method is left here * for backward compatibility. diff --git a/packages/panzer/disc-fe/src/Panzer_FaceToElement_impl.hpp b/packages/panzer/disc-fe/src/Panzer_FaceToElement_impl.hpp index b58a67462276..7b6e9ec8043c 100644 --- a/packages/panzer/disc-fe/src/Panzer_FaceToElement_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_FaceToElement_impl.hpp @@ -55,6 +55,7 @@ #include "Panzer_EdgeFieldPattern.hpp" #include "Panzer_FaceFieldPattern.hpp" #include "Panzer_ElemFieldPattern.hpp" +#include "Panzer_GlobalComm.hpp" #include #include @@ -100,7 +101,7 @@ void FaceToElement:: initialize(panzer::ConnManager & conn) { - Teuchos::RCP> comm_world(new Teuchos::MpiComm< int>(MPI_COMM_WORLD)); + Teuchos::RCP> comm_world(new Teuchos::MpiComm< int>(panzer::get_global_comm())); initialize(conn, comm_world); } #endif