From a37e42d1a68a93cc696bfb5c979a18ea6f3da323 Mon Sep 17 00:00:00 2001 From: Jacob Domagala Date: Fri, 4 Aug 2023 13:04:31 +0200 Subject: [PATCH] AztecOO: Don't use hardcoded MPI_COMM_WORLD in AztecOO code (except for excluded files) --- packages/aztecoo/src/az_aztec.h | 23 ++++++++----- packages/aztecoo/src/az_comm.c | 3 ++ packages/aztecoo/src/az_old_matvec_mult.c | 19 ++++++----- packages/aztecoo/src/md_wrap_mpi_c.c | 41 ++++++++++++----------- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/packages/aztecoo/src/az_aztec.h b/packages/aztecoo/src/az_aztec.h index 85ab5cdcd2e7..c42be7400e78 100644 --- a/packages/aztecoo/src/az_aztec.h +++ b/packages/aztecoo/src/az_aztec.h @@ -1,13 +1,13 @@ /* //@HEADER // *********************************************************************** -// -// AztecOO: An Object-Oriented Aztec Linear Solver Package +// +// AztecOO: An Object-Oriented Aztec Linear Solver Package // Copyright (2002) Sandia Corporation -// +// // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive // license for use of this work by or on behalf of the U.S. Government. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -35,8 +35,8 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// Questions? Contact Michael A. Heroux (maherou@sandia.gov) -// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// // *********************************************************************** //@HEADER */ @@ -101,11 +101,16 @@ #include #define MPI_AZRequest MPI_Request #define MPI_AZComm MPI_Comm +#define AZ_MPI_COMM_WORLD MPI_COMM_WORLD #else #define MPI_AZRequest int #define MPI_AZComm int +#define AZ_MPI_COMM_WORLD 0 #endif +// declared in az_comm.c +extern MPI_AZComm az_global_mpi_comm; + #include @@ -314,7 +319,7 @@ struct AZ_CONVERGE_STRUCT { int * isnan, /* = 0 on return if not NaN, otherwise NaNs detected */ double * rnorm, /* = current norm on return */ int * r_avail); /* If set to AZ_TRUE on return, the residual vector is needed - by this convergence on subsequent calls and it should be + by this convergence on subsequent calls and it should be supplied by the calling routine */ }; @@ -598,7 +603,7 @@ extern void AZ_fortransolve(double x[], double b[], int options[], * There are different conventions for external names for fortran subroutines. * In addition, different compilers return differing caluse for a fortran * subroutine call. Finally, there is now also an option to disable Fortran, - * so we also supply C versions of Fortran subroutines defined in AztecOO. + * so we also supply C versions of Fortran subroutines defined in AztecOO. * In this section we take all of this into account. */ @@ -1444,7 +1449,7 @@ void PREFIX AZ_SLAIC1_F77(int * , int *, float *, float *, float *, float *, double params[], int proc_config[],double status[], AZ_MATRIX *Amat, AZ_PRECOND *precond, struct AZ_CONVERGE_STRUCT *convergence_info ); - + /*****************************************************************************/ /* IFPACK interface routine */ diff --git a/packages/aztecoo/src/az_comm.c b/packages/aztecoo/src/az_comm.c index 22e310480983..6c575b5198d6 100644 --- a/packages/aztecoo/src/az_comm.c +++ b/packages/aztecoo/src/az_comm.c @@ -81,6 +81,9 @@ extern int az_iterate_id; +// Replaces the occurences of hardcoded MPI_COMM_WORLD +MPI_AZComm az_global_mpi_comm = AZ_MPI_COMM_WORLD; + /******************************************************************************/ /******************************************************************************/ /******************************************************************************/ diff --git a/packages/aztecoo/src/az_old_matvec_mult.c b/packages/aztecoo/src/az_old_matvec_mult.c index 8aee5c736530..f6f4344eb394 100644 --- a/packages/aztecoo/src/az_old_matvec_mult.c +++ b/packages/aztecoo/src/az_old_matvec_mult.c @@ -1,13 +1,13 @@ /* //@HEADER // *********************************************************************** -// -// AztecOO: An Object-Oriented Aztec Linear Solver Package +// +// AztecOO: An Object-Oriented Aztec Linear Solver Package // Copyright (2002) Sandia Corporation -// +// // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive // license for use of this work by or on behalf of the U.S. Government. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -35,8 +35,8 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// Questions? Contact Michael A. Heroux (maherou@sandia.gov) -// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// // *********************************************************************** //@HEADER */ @@ -44,6 +44,9 @@ #include #include #include "az_aztec.h" + +extern MPI_AZComm az_global_mpi_comm; + void AZ_matvec_mult(double *val, int *indx, int *bindx, int *rpntr, int *cpntr, int *bpntr, double *b, register double *c, int exchange_flag, int *data_org) @@ -119,9 +122,9 @@ void AZ_matvec_mult(double *val, int *indx, int *bindx, int *rpntr, int *cpntr, Amat.aux_matrix = NULL; Amat.matrix_type = data_org[AZ_matrix_type]; #ifdef AZTEC_MPI - AZ_set_comm(proc_config, MPI_COMM_WORLD); + AZ_set_comm(proc_config, az_global_mpi_comm); if (first_time == 1) { - AZ_set_proc_config(proc_config, MPI_COMM_WORLD); + AZ_set_proc_config(proc_config, az_global_mpi_comm); #else if (first_time == 1) { AZ_set_proc_config(proc_config, AZ_NOT_MPI); diff --git a/packages/aztecoo/src/md_wrap_mpi_c.c b/packages/aztecoo/src/md_wrap_mpi_c.c index a0bd7bc30c08..771fc98f1534 100644 --- a/packages/aztecoo/src/md_wrap_mpi_c.c +++ b/packages/aztecoo/src/md_wrap_mpi_c.c @@ -1,13 +1,13 @@ /* //@HEADER // *********************************************************************** -// -// AztecOO: An Object-Oriented Aztec Linear Solver Package +// +// AztecOO: An Object-Oriented Aztec Linear Solver Package // Copyright (2002) Sandia Corporation -// +// // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive // license for use of this work by or on behalf of the U.S. Government. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -35,8 +35,8 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// Questions? Contact Michael A. Heroux (maherou@sandia.gov) -// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// // *********************************************************************** //@HEADER */ @@ -67,14 +67,17 @@ int gl_sbuf = 3; /******************************************************************************/ int the_proc_name = -1; +// declared in az_comm.c (we use MPI_Comm instead of MPI_AZComm as it's guaranteed AztecOO is built with MPI) +extern MPI_Comm az_global_mpi_comm; + void get_parallel_info(int *proc, int *nprocs, int *dim) { /* local variables */ - MPI_Comm_size(MPI_COMM_WORLD, nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, proc); + MPI_Comm_size(az_global_mpi_comm, nprocs); + MPI_Comm_rank(az_global_mpi_comm, proc); *dim = 0; the_proc_name = *proc; @@ -99,11 +102,11 @@ int md_read(char *buf, int bytes, int *source, int *type, int *flag) if (*source == -1) *source = MPI_ANY_SOURCE; if (bytes == 0) { - err = MPI_Recv(&gl_rbuf, 1, MPI_BYTE, *source, *type, MPI_COMM_WORLD, + err = MPI_Recv(&gl_rbuf, 1, MPI_BYTE, *source, *type, az_global_mpi_comm, &status); } else { - err = MPI_Recv(buf, bytes, MPI_BYTE, *source, *type, MPI_COMM_WORLD, + err = MPI_Recv(buf, bytes, MPI_BYTE, *source, *type, az_global_mpi_comm, &status); } @@ -129,10 +132,10 @@ int md_write(char *buf, int bytes, int dest, int type, int *flag) int err; if (bytes == 0) { - err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, MPI_COMM_WORLD); + err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm); } else { - err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, MPI_COMM_WORLD); + err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm); } if (err != 0) (void) fprintf(stderr, "MPI_Send error = %d\n", err); @@ -182,11 +185,11 @@ int md_wrap_iread(void *buf, int bytes, int *source, int *type, if (*source == -1) *source = MPI_ANY_SOURCE; if (bytes == 0) { - err = MPI_Irecv(&gl_rbuf, 1, MPI_BYTE, *source, *type, MPI_COMM_WORLD, + err = MPI_Irecv(&gl_rbuf, 1, MPI_BYTE, *source, *type, az_global_mpi_comm, request); } else { - err = MPI_Irecv(buf, bytes, MPI_BYTE, *source, *type, MPI_COMM_WORLD, + err = MPI_Irecv(buf, bytes, MPI_BYTE, *source, *type, az_global_mpi_comm, request); } @@ -231,10 +234,10 @@ int md_wrap_write(void *buf, int bytes, int dest, int type, int *flag) int err = 0; if (bytes == 0) { - err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, MPI_COMM_WORLD); + err = MPI_Send(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm); } else { - err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, MPI_COMM_WORLD); + err = MPI_Send(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm); } return err; @@ -303,7 +306,7 @@ int md_wrap_iwrite(void *buf, int bytes, int dest, int type, int *flag, /******************************************************************************* - Machine dependent wrapped message-sending (nonblocking) communication + Machine dependent wrapped message-sending (nonblocking) communication routine for MPI. Author: Scott A. Hutchinson, SNL, 9221 @@ -332,11 +335,11 @@ int md_wrap_iwrite(void *buf, int bytes, int dest, int type, int *flag, int err = 0; if (bytes == 0) { - err = MPI_Isend(&gl_sbuf, 1, MPI_BYTE, dest, type, MPI_COMM_WORLD, + err = MPI_Isend(&gl_sbuf, 1, MPI_BYTE, dest, type, az_global_mpi_comm, request); } else { - err = MPI_Isend(buf, bytes, MPI_BYTE, dest, type, MPI_COMM_WORLD, + err = MPI_Isend(buf, bytes, MPI_BYTE, dest, type, az_global_mpi_comm, request); }