diff --git a/packages/aztecoo/src/AztecOO_GlobalComm.cpp b/packages/aztecoo/src/AztecOO_GlobalComm.cpp new file mode 100644 index 000000000000..29c4cd9f5491 --- /dev/null +++ b/packages/aztecoo/src/AztecOO_GlobalComm.cpp @@ -0,0 +1,62 @@ + +/* +//@HEADER +// *********************************************************************** +// +// AztecOO: An Object-Oriented Aztec Linear Solver Package +// Copyright (2002) Sandia Corporation +// +// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive +// license for use of this work by or on behalf of the U.S. Government. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// *********************************************************************** +//@HEADER +*/ + +#include "AztecOO_GlobalComm.h" + +pthread_mutex_t mpi_mutex = PTHREAD_MUTEX_INITIALIZER; +MPI_Comm Global_AztecOO_Comm = MPI_COMM_WORLD; + +void AZ_initialize_global_comm(MPI_Comm comm) { + pthread_mutex_lock(&mpi_mutex); + Global_AztecOO_Comm = comm; + pthread_mutex_unlock(&mpi_mutex); +} + +MPI_Comm AZ_get_global_comm() { + MPI_Comm tmp; + pthread_mutex_lock(&mpi_mutex); + tmp = Global_AztecOO_Comm; + pthread_mutex_unlock(&mpi_mutex); + return tmp; +} diff --git a/packages/aztecoo/src/AztecOO_GlobalComm.h b/packages/aztecoo/src/AztecOO_GlobalComm.h new file mode 100644 index 000000000000..c9599147c80e --- /dev/null +++ b/packages/aztecoo/src/AztecOO_GlobalComm.h @@ -0,0 +1,69 @@ + +/* +//@HEADER +// *********************************************************************** +// +// AztecOO: An Object-Oriented Aztec Linear Solver Package +// Copyright (2002) Sandia Corporation +// +// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive +// license for use of this work by or on behalf of the U.S. Government. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// *********************************************************************** +//@HEADER +*/ + +#ifndef AZTECOO_GLOBAL_COMM_HPP +#define AZTECOO_GLOBAL_COMM_HPP + +#include "AztecOO_ConfigDefs.h" + +#ifdef HAVE_MPI + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Function prototypes +void AZ_initialize_global_comm(MPI_Comm comm); +MPI_Comm AZ_get_global_comm(); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // HAVE_MPI + +#endif // AZTECOO_GLOBAL_COMM_HPP diff --git a/packages/aztecoo/src/CMakeLists.txt b/packages/aztecoo/src/CMakeLists.txt index b27c6f58a3b4..6f2e0ff14118 100644 --- a/packages/aztecoo/src/CMakeLists.txt +++ b/packages/aztecoo/src/CMakeLists.txt @@ -50,7 +50,7 @@ SET(HEADERS ${HEADERS} ) SET(SOURCES ${SOURCES} - az_comm.c + az_comm.c ) # @@ -73,6 +73,7 @@ SET(HEADERS ${HEADERS} AztecOO_StatusTestResNorm.h AztecOO_ConditionNumber.h AztecOO_ConfigDefs.h + AztecOO_GlobalComm.h ) SET(SOURCES ${SOURCES} @@ -88,10 +89,11 @@ SET(SOURCES ${SOURCES} AztecOO_StatusTestMaxIters.cpp AztecOO_StatusTestResNorm.cpp AztecOO_ConditionNumber.cpp + AztecOO_GlobalComm.cpp ) SET(HEADERS ${HEADERS} - AztecOO_Version.h + AztecOO_Version.h ) SET(SOURCES ${SOURCES} @@ -109,12 +111,12 @@ IF (TPL_ENABLE_MPI) SET(SOURCES ${SOURCES} md_timer_mpi.c - md_wrap_mpi_c.c + md_wrap_mpi_c.c ) ELSE() SET(HEADERS ${HEADERS} md_timer_generic.c - md_wrap_scalar_c.c + md_wrap_scalar_c.c ) ENDIF() @@ -151,7 +153,7 @@ SET(SOURCES ${SOURCES} az_precond.c az_scaling.c az_solve.c - az_sort.c + az_sort.c ) IF (${PROJECT_NAME}_ENABLE_Fortran) diff --git a/packages/aztecoo/src/az_comm.c b/packages/aztecoo/src/az_comm.c index 6c575b5198d6..a928ebcb8096 100644 --- a/packages/aztecoo/src/az_comm.c +++ b/packages/aztecoo/src/az_comm.c @@ -81,8 +81,6 @@ extern int az_iterate_id; -// Replaces the occurences of hardcoded MPI_COMM_WORLD -MPI_AZComm az_global_mpi_comm = AZ_MPI_COMM_WORLD; /******************************************************************************/ /******************************************************************************/ diff --git a/packages/aztecoo/src/az_old_matvec_mult.c b/packages/aztecoo/src/az_old_matvec_mult.c index f6f4344eb394..0f6431199ce9 100644 --- a/packages/aztecoo/src/az_old_matvec_mult.c +++ b/packages/aztecoo/src/az_old_matvec_mult.c @@ -44,8 +44,7 @@ #include #include #include "az_aztec.h" - -extern MPI_AZComm az_global_mpi_comm; +#include "AztecOO_GlobalComm.h" void AZ_matvec_mult(double *val, int *indx, int *bindx, int *rpntr, int *cpntr, int *bpntr, double *b, register double *c, @@ -122,9 +121,9 @@ void AZ_matvec_mult(double *val, int *indx, int *bindx, int *rpntr, int *cpntr, Amat.aux_matrix = NULL; Amat.matrix_type = data_org[AZ_matrix_type]; #ifdef AZTEC_MPI - AZ_set_comm(proc_config, az_global_mpi_comm); + AZ_set_comm(proc_config, AZ_get_global_comm()); if (first_time == 1) { - AZ_set_proc_config(proc_config, az_global_mpi_comm); + AZ_set_proc_config(proc_config, AZ_get_global_comm()); #else if (first_time == 1) { AZ_set_proc_config(proc_config, AZ_NOT_MPI); diff --git a/packages/aztecoo/src/md_wrap_mpi_c.c b/packages/aztecoo/src/md_wrap_mpi_c.c index 771fc98f1534..355e52ea572d 100644 --- a/packages/aztecoo/src/md_wrap_mpi_c.c +++ b/packages/aztecoo/src/md_wrap_mpi_c.c @@ -60,6 +60,8 @@ #include #include +#include "AztecOO_GlobalComm.h" + int gl_rbuf = 3; int gl_sbuf = 3; /******************************************************************************/ @@ -67,8 +69,6 @@ int gl_sbuf = 3; /******************************************************************************/ int the_proc_name = -1; -// declared in az_comm.c (we use MPI_Comm instead of MPI_AZComm as it's guaranteed AztecOO is built with MPI) -extern MPI_Comm az_global_mpi_comm; void get_parallel_info(int *proc, int *nprocs, int *dim) @@ -76,8 +76,8 @@ void get_parallel_info(int *proc, int *nprocs, int *dim) /* local variables */ - MPI_Comm_size(az_global_mpi_comm, nprocs); - MPI_Comm_rank(az_global_mpi_comm, proc); + MPI_Comm_size(AZ_get_global_comm(), nprocs); + MPI_Comm_rank(AZ_get_global_comm(), proc); *dim = 0; the_proc_name = *proc; @@ -102,11 +102,11 @@ int md_read(char *buf, int bytes, int *source, int *type, int *flag) if (*source == -1) *source = MPI_ANY_SOURCE; if (bytes == 0) { - err = MPI_Recv(&gl_rbuf, 1, MPI_BYTE, *source, *type, az_global_mpi_comm, + err = MPI_Recv(&gl_rbuf, 1, MPI_BYTE, *source, *type, AZ_get_global_comm(), &status); } else { - err = MPI_Recv(buf, bytes, MPI_BYTE, *source, *type, az_global_mpi_comm, + err = MPI_Recv(buf, bytes, MPI_BYTE, *source, *type, AZ_get_global_comm(), &status); } @@ -132,10 +132,10 @@ int md_write(char *buf, int bytes, int dest, int type, int *flag) int err; if (bytes == 0) { - err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm); + err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, AZ_get_global_comm()); } else { - err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm); + err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, AZ_get_global_comm()); } if (err != 0) (void) fprintf(stderr, "MPI_Send error = %d\n", err); @@ -185,11 +185,11 @@ int md_wrap_iread(void *buf, int bytes, int *source, int *type, if (*source == -1) *source = MPI_ANY_SOURCE; if (bytes == 0) { - err = MPI_Irecv(&gl_rbuf, 1, MPI_BYTE, *source, *type, az_global_mpi_comm, + err = MPI_Irecv(&gl_rbuf, 1, MPI_BYTE, *source, *type, AZ_get_global_comm(), request); } else { - err = MPI_Irecv(buf, bytes, MPI_BYTE, *source, *type, az_global_mpi_comm, + err = MPI_Irecv(buf, bytes, MPI_BYTE, *source, *type, AZ_get_global_comm(), request); } @@ -234,10 +234,10 @@ int md_wrap_write(void *buf, int bytes, int dest, int type, int *flag) int err = 0; if (bytes == 0) { - err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm); + err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, AZ_get_global_comm()); } else { - err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm); + err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, AZ_get_global_comm()); } return err; @@ -335,11 +335,11 @@ int md_wrap_iwrite(void *buf, int bytes, int dest, int type, int *flag, int err = 0; if (bytes == 0) { - err = MPI_Isend(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm, + err = MPI_Isend(&gl_sbuf, 1, MPI_BYTE, dest, type, AZ_get_global_comm(), request); } else { - err = MPI_Isend(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm, + err = MPI_Isend(buf, bytes, MPI_BYTE, dest, type, AZ_get_global_comm(), request); }