From f50d366df70b188b4e62ed4d824592ef53d3eb98 Mon Sep 17 00:00:00 2001 From: sjplimp Date: Wed, 6 Aug 2014 16:16:32 +0000 Subject: [PATCH] git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@12255 f3b2605a-c512-4ea7-a41b-209d697bcdaa --- src/comm_brick.cpp | 16 +++--- src/comm_tiled.cpp | 102 +++++++++++++++++++++-------------- src/compute.h | 4 +- src/compute_cluster_atom.cpp | 8 +-- src/compute_cluster_atom.h | 4 +- src/compute_contact_atom.cpp | 2 +- src/compute_pe_atom.cpp | 2 +- src/compute_stress_atom.cpp | 2 +- 8 files changed, 83 insertions(+), 57 deletions(-) diff --git a/src/comm_brick.cpp b/src/comm_brick.cpp index e35e9c150f..db3205d2b3 100644 --- a/src/comm_brick.cpp +++ b/src/comm_brick.cpp @@ -1153,11 +1153,13 @@ void CommBrick::forward_comm_compute(Compute *compute) MPI_Request request; MPI_Status status; + int nsize = compute->comm_forward; + for (iswap = 0; iswap < nswap; iswap++) { // pack buffer - n = compute->pack_comm(sendnum[iswap],sendlist[iswap], + n = compute->pack_forward_comm(sendnum[iswap],sendlist[iswap], buf_send,pbc_flag[iswap],pbc[iswap]); // exchange with another proc @@ -1165,17 +1167,17 @@ void CommBrick::forward_comm_compute(Compute *compute) if (sendproc[iswap] != me) { if (recvnum[iswap]) - MPI_Irecv(buf_recv,n*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0, + MPI_Irecv(buf_recv,nsize*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0, world,&request); if (sendnum[iswap]) - MPI_Send(buf_send,n*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0,world); + MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap],0,world); if (recvnum[iswap]) MPI_Wait(&request,&status); buf = buf_recv; } else buf = buf_send; // unpack buffer - compute->unpack_comm(recvnum[iswap],firstrecv[iswap],buf); + compute->unpack_forward_comm(recvnum[iswap],firstrecv[iswap],buf); } } @@ -1191,6 +1193,8 @@ void CommBrick::reverse_comm_compute(Compute *compute) MPI_Request request; MPI_Status status; + int nsize = compute->comm_reverse; + for (iswap = nswap-1; iswap >= 0; iswap--) { // pack buffer @@ -1202,10 +1206,10 @@ void CommBrick::reverse_comm_compute(Compute *compute) if (sendproc[iswap] != me) { if (sendnum[iswap]) - MPI_Irecv(buf_recv,n*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0, + MPI_Irecv(buf_recv,nsize*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0, world,&request); if (recvnum[iswap]) - MPI_Send(buf_send,n*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0,world); + MPI_Send(buf_send,n,MPI_DOUBLE,recvproc[iswap],0,world); if (sendnum[iswap]) MPI_Wait(&request,&status); buf = buf_recv; } else buf = buf_send; diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index 930d9177e2..4c71e4ded6 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -1482,31 +1482,44 @@ void CommTiled::reverse_comm_variable_fix(Fix *fix) void CommTiled::forward_comm_compute(Compute *compute) { - int i,irecv,n; + int i,irecv,n,nsend,nrecv; MPI_Status status; - for (int iswap = 0; iswap < nswap; iswap++) { - if (sendproc[iswap][0] != me) { - for (i = 0; i < nrecvproc[iswap]; i++) - MPI_Irecv(&buf_recv[forward_recv_offset[iswap][i]], - size_forward_recv[iswap][i], - MPI_DOUBLE,recvproc[iswap][i],0,world,&requests[i]); - for (i = 0; i < nsendproc[iswap]; i++) { - n = compute->pack_comm(sendnum[iswap][i],sendlist[iswap][i], - buf_send,pbc_flag[iswap][i],pbc[iswap][i]); - MPI_Send(buf_send,n*sendnum[iswap][i],MPI_DOUBLE, - sendproc[iswap][i],0,world); - } - for (i = 0; i < nrecvproc[iswap]; i++) { - MPI_Waitany(nrecvproc[iswap],requests,&irecv,&status); - compute->unpack_comm(recvnum[iswap][irecv],firstrecv[iswap][irecv], - &buf_recv[forward_recv_offset[iswap][irecv]]); - } + int nsize = compute->comm_forward; - } else { - n = compute->pack_comm(sendnum[iswap][0],sendlist[iswap][0], - buf_send,pbc_flag[iswap][0],pbc[iswap][0]); - compute->unpack_comm(recvnum[iswap][0],firstrecv[iswap][0],buf_send); + for (int iswap = 0; iswap < nswap; iswap++) { + nsend = nsendproc[iswap] - sendself[iswap]; + nrecv = nrecvproc[iswap] - sendself[iswap]; + + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) + MPI_Irecv(&buf_recv[nsize*forward_recv_offset[iswap][i]], + nsize*recvnum[iswap][i], + MPI_DOUBLE,recvproc[iswap][i],0,world,&requests[i]); + } + if (sendother[iswap]) { + for (i = 0; i < nsendproc[iswap]; i++) { + n = compute->pack_forward_comm(sendnum[iswap][i],sendlist[iswap][i], + buf_send,pbc_flag[iswap][i], + pbc[iswap][i]); + MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][i],0,world); + } + } + if (sendself[iswap]) { + compute->pack_forward_comm(sendnum[iswap][nsend],sendlist[iswap][nsend], + buf_send,pbc_flag[iswap][nsend], + pbc[iswap][nsend]); + compute->unpack_forward_comm(recvnum[iswap][nrecv], + firstrecv[iswap][nrecv],buf_send); + } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&irecv,&status); + compute-> + unpack_forward_comm(recvnum[iswap][irecv],firstrecv[iswap][irecv], + &buf_recv[nsize* + forward_recv_offset[iswap][irecv]]); + } } } } @@ -1518,33 +1531,42 @@ void CommTiled::forward_comm_compute(Compute *compute) void CommTiled::reverse_comm_compute(Compute *compute) { - int i,irecv,n; + int i,irecv,n,nsend,nrecv; MPI_Status status; + int nsize = compute->comm_reverse; + for (int iswap = nswap-1; iswap >= 0; iswap--) { - if (sendproc[iswap][0] != me) { - for (i = 0; i < nsendproc[iswap]; i++) - MPI_Irecv(&buf_recv[reverse_recv_offset[iswap][i]], - size_reverse_recv[iswap][i],MPI_DOUBLE, + nsend = nsendproc[iswap] - sendself[iswap]; + nrecv = nrecvproc[iswap] - sendself[iswap]; + + if (sendother[iswap]) { + for (i = 0; i < nsend; i++) + MPI_Irecv(&buf_recv[nsize*reverse_recv_offset[iswap][i]], + nsize*sendnum[iswap][i],MPI_DOUBLE, sendproc[iswap][i],0,world,&requests[i]); - for (i = 0; i < nrecvproc[iswap]; i++) { + } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { n = compute->pack_reverse_comm(recvnum[iswap][i],firstrecv[iswap][i], - buf_send); - MPI_Send(buf_send,n*recvnum[iswap][i],MPI_DOUBLE, - recvproc[iswap][i],0,world); + buf_send); + MPI_Send(buf_send,n,MPI_DOUBLE,recvproc[iswap][i],0,world); } - for (i = 0; i < nsendproc[iswap]; i++) { - MPI_Waitany(nsendproc[iswap],requests,&irecv,&status); + } + if (sendself[iswap]) { + compute->pack_reverse_comm(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], + buf_send); + compute->unpack_reverse_comm(sendnum[iswap][nsend],sendlist[iswap][nsend], + buf_send); + } + if (sendother[iswap]) { + for (i = 0; i < nsend; i++) { + MPI_Waitany(nsend,requests,&irecv,&status); compute-> unpack_reverse_comm(sendnum[iswap][irecv],sendlist[iswap][irecv], - &buf_recv[reverse_recv_offset[iswap][irecv]]); + &buf_recv[nsize* + reverse_recv_offset[iswap][irecv]]); } - - } else { - n = compute->pack_reverse_comm(recvnum[iswap][0],firstrecv[iswap][0], - buf_send); - compute->unpack_reverse_comm(sendnum[iswap][0],sendlist[iswap][0], - buf_send); } } } diff --git a/src/compute.h b/src/compute.h index 9b7c161b21..992acbef12 100644 --- a/src/compute.h +++ b/src/compute.h @@ -98,8 +98,8 @@ class Compute : protected Pointers { virtual void compute_peratom() {} virtual void compute_local() {} - virtual int pack_comm(int, int *, double *, int, int *) {return 0;} - virtual void unpack_comm(int, int, double *) {} + virtual int pack_forward_comm(int, int *, double *, int, int *) {return 0;} + virtual void unpack_forward_comm(int, int, double *) {} virtual int pack_reverse_comm(int, int, double *) {return 0;} virtual void unpack_reverse_comm(int, int *, double *) {} diff --git a/src/compute_cluster_atom.cpp b/src/compute_cluster_atom.cpp index ba986061bd..91c876ad9e 100644 --- a/src/compute_cluster_atom.cpp +++ b/src/compute_cluster_atom.cpp @@ -195,8 +195,8 @@ void ComputeClusterAtom::compute_peratom() /* ---------------------------------------------------------------------- */ -int ComputeClusterAtom::pack_comm(int n, int *list, double *buf, - int pbc_flag, int *pbc) +int ComputeClusterAtom::pack_forward_comm(int n, int *list, double *buf, + int pbc_flag, int *pbc) { int i,j,m; @@ -214,12 +214,12 @@ int ComputeClusterAtom::pack_comm(int n, int *list, double *buf, } } - return 1; + return m; } /* ---------------------------------------------------------------------- */ -void ComputeClusterAtom::unpack_comm(int n, int first, double *buf) +void ComputeClusterAtom::unpack_forward_comm(int n, int first, double *buf) { int i,m,last; diff --git a/src/compute_cluster_atom.h b/src/compute_cluster_atom.h index 5451f16d0f..48dfe7eb1a 100644 --- a/src/compute_cluster_atom.h +++ b/src/compute_cluster_atom.h @@ -31,8 +31,8 @@ class ComputeClusterAtom : public Compute { void init(); void init_list(int, class NeighList *); void compute_peratom(); - int pack_comm(int, int *, double *, int, int *); - void unpack_comm(int, int, double *); + int pack_forward_comm(int, int *, double *, int, int *); + void unpack_forward_comm(int, int, double *); double memory_usage(); private: diff --git a/src/compute_contact_atom.cpp b/src/compute_contact_atom.cpp index ef3d4ff581..492846b130 100644 --- a/src/compute_contact_atom.cpp +++ b/src/compute_contact_atom.cpp @@ -170,7 +170,7 @@ int ComputeContactAtom::pack_reverse_comm(int n, int first, double *buf) last = first + n; for (i = first; i < last; i++) buf[m++] = contact[i]; - return 1; + return m; } /* ---------------------------------------------------------------------- */ diff --git a/src/compute_pe_atom.cpp b/src/compute_pe_atom.cpp index 50af1267f0..b1892e42f4 100755 --- a/src/compute_pe_atom.cpp +++ b/src/compute_pe_atom.cpp @@ -168,7 +168,7 @@ int ComputePEAtom::pack_reverse_comm(int n, int first, double *buf) m = 0; last = first + n; for (i = first; i < last; i++) buf[m++] = energy[i]; - return 1; + return m; } /* ---------------------------------------------------------------------- */ diff --git a/src/compute_stress_atom.cpp b/src/compute_stress_atom.cpp index c95d370a87..0e5aa749ca 100644 --- a/src/compute_stress_atom.cpp +++ b/src/compute_stress_atom.cpp @@ -355,7 +355,7 @@ int ComputeStressAtom::pack_reverse_comm(int n, int first, double *buf) buf[m++] = stress[i][4]; buf[m++] = stress[i][5]; } - return 6; + return m; } /* ---------------------------------------------------------------------- */