Panzer: Remove use of hardcoded MPI_COMM_WORLD

packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_Quad8ToQuad4MeshFactory.hpp

@@ -45,6 +45,7 @@
 
 #include <Panzer_STK_MeshFactory.hpp>
 #include <Panzer_STK_Interface.hpp>
+#include <Panzer_GlobalComm.hpp>
 #include <vector>
 #include <string>
 
@@ -61,7 +62,7 @@ class Quad8ToQuad4MeshFactory : public STK_MeshFactory {
 public:
 
   Quad8ToQuad4MeshFactory(const std::string& quad8MeshFileName,
-                          stk::ParallelMachine mpi_comm = MPI_COMM_WORLD,
+                          stk::ParallelMachine mpi_comm = panzer::get_global_comm(),
                           const bool print_debug = false);
 
   Quad8ToQuad4MeshFactory(const Teuchos::RCP<panzer_stk::STK_Interface>& quad8Mesh,

packages/panzer/adapters-stk/src/stk_interface/Panzer_STK_QuadraticToLinearMeshFactory.hpp

@@ -45,6 +45,7 @@
 
 #include <Panzer_STK_MeshFactory.hpp>
 #include <Panzer_STK_Interface.hpp>
+#include <Panzer_GlobalComm.hpp>
 #include <vector>
 #include <string>
 
@@ -60,7 +61,7 @@ class QuadraticToLinearMeshFactory : public STK_MeshFactory {
 public:
 
   QuadraticToLinearMeshFactory(const std::string& quadMeshFileName,
-                          stk::ParallelMachine mpi_comm = MPI_COMM_WORLD,
+                          stk::ParallelMachine mpi_comm = panzer::get_global_comm(),
                           const bool print_debug = false);
 
   QuadraticToLinearMeshFactory(const Teuchos::RCP<panzer_stk::STK_Interface>& quadMesh,

packages/panzer/core/src/Panzer_GlobalComm.hpp

@@ -0,0 +1,69 @@
+// @HEADER
+// ***********************************************************************
+//
+//           Panzer: A partial differential equation assembly
+//       engine for strongly coupled complex multiphysics systems
+//                 Copyright (2011) Sandia Corporation
+//
+// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and
+// Eric C. Cyr (eccyr@sandia.gov)
+// ***********************************************************************
+// @HEADER
+
+#ifndef PANZER_GLOBAL_COMM_HPP
+#define PANZER_GLOBAL_COMM_HPP
+
+#ifdef HAVE_MPI
+
+#include <mpi.h>
+#include <mutex>
+
+namespace panzer {
+
+static std::mutex mpi_mutex;
+static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD;
+
+inline void initialize_global_comm(MPI_Comm comm) {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    Global_MPI_Comm = comm;
+}
+
+inline MPI_Comm get_global_comm() {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    return Global_MPI_Comm;
+}
+
+}
+
+#endif // HAVE_MPI
+#endif /* PANZER_GLOBAL_COMM_HPP */

packages/panzer/core/src/Panzer_PauseToAttach.hpp

@@ -43,6 +43,7 @@
 #ifndef PANZER_PAUSE_TO_ATTACH
 #define PANZER_PAUSE_TO_ATTACH
 
+#include "Panzer_GlobalComm.hpp"
 #include "Teuchos_RCP.hpp"
 #include "Teuchos_DefaultMpiComm.hpp"
 #include "Teuchos_FancyOStream.hpp"
@@ -74,7 +75,7 @@ namespace panzer {
 
   inline void pauseToAttach()
   {
-    pauseToAttach(MPI_COMM_WORLD);
+    pauseToAttach(panzer::get_global_comm());
   }
 
 

packages/panzer/disc-fe/src/Panzer_FaceToElement.hpp

@@ -87,7 +87,7 @@ class FaceToElement {
                 const Teuchos::RCP<const Teuchos::Comm<int>> comm);
 
 #ifndef PANZER_HIDE_DEPRECATED_CODE
-  /** Build the mapping from a mesh topology using MPI_COMM_WORLD.
+  /** Build the mapping from a mesh topology using panzer::get_global_comm().
     * This method is deprecated in favor of initialize(conn, comm) which
     * explicitly specifies the communicator.  This method is left here
     * for backward compatibility.

packages/panzer/disc-fe/src/Panzer_FaceToElement_impl.hpp

@@ -55,6 +55,7 @@
 #include "Panzer_EdgeFieldPattern.hpp"
 #include "Panzer_FaceFieldPattern.hpp"
 #include "Panzer_ElemFieldPattern.hpp"
+#include "Panzer_GlobalComm.hpp"
 
 #include <vector>
 #include <set>
@@ -100,7 +101,7 @@ void
 FaceToElement<LocalOrdinal,GlobalOrdinal>::
 initialize(panzer::ConnManager & conn)
 {
-  Teuchos::RCP<const Teuchos::Comm<int>> comm_world(new Teuchos::MpiComm< int>(MPI_COMM_WORLD));
+  Teuchos::RCP<const Teuchos::Comm<int>> comm_world(new Teuchos::MpiComm< int>(panzer::get_global_comm()));
   initialize(conn, comm_world);
 }
 #endif