From 411c069ba66fb10b0e1121d0499b7e888cac9c80 Mon Sep 17 00:00:00 2001 From: Tim Mattox Date: Fri, 17 Mar 2017 02:36:41 -0400 Subject: [PATCH 1/4] BUGFIX: Prevent possible deadlock in Irregular::create_atom and create_data --- src/irregular.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/irregular.cpp b/src/irregular.cpp index 3947001541..d0210244fb 100644 --- a/src/irregular.cpp +++ b/src/irregular.cpp @@ -395,7 +395,9 @@ int Irregular::create_atom(int n, int *sizes, int *proclist, int sortflag) sendmax_proc = 0; for (i = 0; i < nsend_proc; i++) { - MPI_Send(&length_send[i],1,MPI_INT,proc_send[i],0,world); + MPI_Request tmpReq; // Use non-blocking send to avoid possible deadlock + MPI_Isend(&length_send[i],1,MPI_INT,proc_send[i],0,world,&tmpReq); + MPI_Request_free(&tmpReq); // the MPI_Barrier below marks completion sendmax_proc = MAX(sendmax_proc,length_send[i]); } @@ -641,7 +643,9 @@ int Irregular::create_data(int n, int *proclist, int sortflag) sendmax_proc = 0; for (i = 0; i < nsend_proc; i++) { - MPI_Send(&num_send[i],1,MPI_INT,proc_send[i],0,world); + MPI_Request tmpReq; // Use non-blocking send to avoid possible deadlock + MPI_Isend(&num_send[i],1,MPI_INT,proc_send[i],0,world,&tmpReq); + MPI_Request_free(&tmpReq); // the MPI_Barrier below marks completion sendmax_proc = MAX(sendmax_proc,num_send[i]); } From 8e75616c144d7d75b7163e0f79668b7429de2126 Mon Sep 17 00:00:00 2001 From: Tim Mattox Date: Fri, 17 Mar 2017 02:55:41 -0400 Subject: [PATCH 2/4] In irregular.cpp use simpler and slightly faster MPI_Reduce_scatter_block() --- src/irregular.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/irregular.cpp b/src/irregular.cpp index d0210244fb..fe08e04be8 100644 --- a/src/irregular.cpp +++ b/src/irregular.cpp @@ -297,11 +297,9 @@ int Irregular::create_atom(int n, int *sizes, int *proclist, int sortflag) // setup for collective comm // work1 = 1 for procs I send a message to, not including self - // work2 = 1 for all procs, used for ReduceScatter for (i = 0; i < nprocs; i++) { work1[i] = 0; - work2[i] = 1; } for (i = 0; i < n; i++) work1[proclist[i]] = 1; work1[me] = 0; @@ -318,7 +316,7 @@ int Irregular::create_atom(int n, int *sizes, int *proclist, int sortflag) MPI_Allreduce(work1,work2,nprocs,MPI_INT,MPI_SUM,world); nrecv_proc = work2[me]; #else - MPI_Reduce_scatter(work1,&nrecv_proc,work2,MPI_INT,MPI_SUM,world); + MPI_Reduce_scatter_block(work1,&nrecv_proc,1,MPI_INT,MPI_SUM,world); #endif #endif @@ -545,11 +543,9 @@ int Irregular::create_data(int n, int *proclist, int sortflag) // setup for collective comm // work1 = 1 for procs I send a message to, not including self - // work2 = 1 for all procs, used for ReduceScatter for (i = 0; i < nprocs; i++) { work1[i] = 0; - work2[i] = 1; } for (i = 0; i < n; i++) work1[proclist[i]] = 1; work1[me] = 0; @@ -566,7 +562,7 @@ int Irregular::create_data(int n, int *proclist, int sortflag) MPI_Allreduce(work1,work2,nprocs,MPI_INT,MPI_SUM,world); nrecv_proc = work2[me]; #else - MPI_Reduce_scatter(work1,&nrecv_proc,work2,MPI_INT,MPI_SUM,world); + MPI_Reduce_scatter_block(work1,&nrecv_proc,1,MPI_INT,MPI_SUM,world); #endif #endif From 7fb741d53da8848e8f9466018af2e52d241b09aa Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 17 Mar 2017 11:35:59 -0400 Subject: [PATCH 3/4] Revert "In irregular.cpp use simpler and slightly faster MPI_Reduce_scatter_block()" This reverts commit 8e75616c144d7d75b7163e0f79668b7429de2126. --- src/irregular.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/irregular.cpp b/src/irregular.cpp index fe08e04be8..d0210244fb 100644 --- a/src/irregular.cpp +++ b/src/irregular.cpp @@ -297,9 +297,11 @@ int Irregular::create_atom(int n, int *sizes, int *proclist, int sortflag) // setup for collective comm // work1 = 1 for procs I send a message to, not including self + // work2 = 1 for all procs, used for ReduceScatter for (i = 0; i < nprocs; i++) { work1[i] = 0; + work2[i] = 1; } for (i = 0; i < n; i++) work1[proclist[i]] = 1; work1[me] = 0; @@ -316,7 +318,7 @@ int Irregular::create_atom(int n, int *sizes, int *proclist, int sortflag) MPI_Allreduce(work1,work2,nprocs,MPI_INT,MPI_SUM,world); nrecv_proc = work2[me]; #else - MPI_Reduce_scatter_block(work1,&nrecv_proc,1,MPI_INT,MPI_SUM,world); + MPI_Reduce_scatter(work1,&nrecv_proc,work2,MPI_INT,MPI_SUM,world); #endif #endif @@ -543,9 +545,11 @@ int Irregular::create_data(int n, int *proclist, int sortflag) // setup for collective comm // work1 = 1 for procs I send a message to, not including self + // work2 = 1 for all procs, used for ReduceScatter for (i = 0; i < nprocs; i++) { work1[i] = 0; + work2[i] = 1; } for (i = 0; i < n; i++) work1[proclist[i]] = 1; work1[me] = 0; @@ -562,7 +566,7 @@ int Irregular::create_data(int n, int *proclist, int sortflag) MPI_Allreduce(work1,work2,nprocs,MPI_INT,MPI_SUM,world); nrecv_proc = work2[me]; #else - MPI_Reduce_scatter_block(work1,&nrecv_proc,1,MPI_INT,MPI_SUM,world); + MPI_Reduce_scatter(work1,&nrecv_proc,work2,MPI_INT,MPI_SUM,world); #endif #endif From ae5ebf600189568de4fc8cc0af94db11149b5dcf Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 17 Mar 2017 11:40:09 -0400 Subject: [PATCH 4/4] add support for MPI_Request_free() to MPI STUBS library --- src/STUBS/mpi.c | 7 +++++++ src/STUBS/mpi.h | 1 + 2 files changed, 8 insertions(+) diff --git a/src/STUBS/mpi.c b/src/STUBS/mpi.c index ca0e921e13..281a12a4e7 100644 --- a/src/STUBS/mpi.c +++ b/src/STUBS/mpi.c @@ -190,6 +190,13 @@ int MPI_Type_size(MPI_Datatype datatype, int *size) /* ---------------------------------------------------------------------- */ +int MPI_Request_free(MPI_Request *request) +{ + return 0; +} + +/* ---------------------------------------------------------------------- */ + int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { diff --git a/src/STUBS/mpi.h b/src/STUBS/mpi.h index 82cbe302ba..1eca1ec527 100644 --- a/src/STUBS/mpi.h +++ b/src/STUBS/mpi.h @@ -89,6 +89,7 @@ int MPI_Finalize(); double MPI_Wtime(); int MPI_Type_size(int, int *); +int MPI_Request_free(MPI_Request *request); int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);