From 304d6876f4e4079ff9d31a201fe18b7c82e62779 Mon Sep 17 00:00:00 2001 From: Steve Plimpton Date: Wed, 5 Aug 2020 15:44:02 -0600 Subject: [PATCH] a few bug fixes for support for multi and triclinic in CommTiled --- src/atom_vec.cpp | 6 ++-- src/comm_tiled.cpp | 68 +++++++++++++++++++++++++--------------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/atom_vec.cpp b/src/atom_vec.cpp index 7b75499b83..261b068997 100644 --- a/src/atom_vec.cpp +++ b/src/atom_vec.cpp @@ -1096,8 +1096,9 @@ void AtomVec::unpack_border(int n, int first, double *buf) m = 0; last = first + n; + while (last > nmax) grow(0); + for (i = first; i < last; i++) { - if (i == nmax) grow(0); x[i][0] = buf[m++]; x[i][1] = buf[m++]; x[i][2] = buf[m++]; @@ -1165,8 +1166,9 @@ void AtomVec::unpack_border_vel(int n, int first, double *buf) m = 0; last = first + n; + while (last > nmax) grow(0); + for (i = first; i < last; i++) { - if (i == nmax) grow(0); x[i][0] = buf[m++]; x[i][1] = buf[m++]; x[i][2] = buf[m++]; diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index 79a8f3ab7d..8719835c94 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -12,7 +12,8 @@ ------------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- - Contributing author (multi) : Adrian Diaz (University of Florida) + Contributing author (multi and triclinic support): + Adrian Diaz (University of Florida) ------------------------------------------------------------------------- */ #include "comm_tiled.h" @@ -99,7 +100,6 @@ void CommTiled::init_buffers() sendbox_multi = NULL; maxswap = 6; - nswap = 2*domain->dimension; allocate_swap(maxswap); } @@ -109,6 +109,11 @@ void CommTiled::init() { Comm::init(); + // cannot set nswap in init_buffers() b/c + // dimension command can be after comm_style command + + nswap = 2*domain->dimension; + memory->destroy(cutghostmulti); if (mode == Comm::MULTI) memory->create(cutghostmulti,atom->ntypes+1,3,"comm:cutghostmulti"); @@ -374,21 +379,23 @@ void CommTiled::setup() if (idir == 0) { sbox[idim] = sublo[idim]; - if (i < noverlap1) sbox[3+idim] = MIN(sbox[3+idim]+cut,subhi[idim]); - else sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cut,subhi[idim]); + if (i < noverlap1) + sbox[3+idim] = MIN(sbox[3+idim]+cutghost[idim],subhi[idim]); + else + sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cutghost[idim],subhi[idim]); } else { - if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cut,sublo[idim]); - else sbox[idim] = MAX(sbox[idim]+prd[idim]-cut,sublo[idim]); + if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cutghost[idim],sublo[idim]); + else sbox[idim] = MAX(sbox[idim]+prd[idim]-cutghost[idim],sublo[idim]); sbox[3+idim] = subhi[idim]; } if (idim >= 1) { - if (sbox[0] == oboxlo[0]) sbox[0] -= cut; - if (sbox[3] == oboxhi[0]) sbox[3] += cut; + if (sbox[0] == oboxlo[0]) sbox[0] -= cutghost[0]; + if (sbox[3] == oboxhi[0]) sbox[3] += cutghost[0]; } if (idim == 2) { - if (sbox[1] == oboxlo[1]) sbox[1] -= cut; - if (sbox[4] == oboxhi[1]) sbox[4] += cut; + if (sbox[1] == oboxlo[1]) sbox[1] -= cutghost[1]; + if (sbox[4] == oboxhi[1]) sbox[4] += cutghost[1]; } memcpy(sendbox[iswap][i],sbox,6*sizeof(double)); @@ -1074,9 +1081,8 @@ void CommTiled::borders() if (rmaxall*size_border > maxrecv) grow_recv(rmaxall*size_border); // swap atoms with other procs using pack_border(), unpack_border() - // use Waitall() instead of Waitany() because calls to unpack_border() - // must increment per-atom arrays in ascending order - // For the same reason, sendself unpacks must occur after recvother unpacks + // can use Waitany() because calls to unpack_border() + // increment per-atom arrays as much as needed if (ghost_velocity) { if (recvother[iswap]) { @@ -1092,13 +1098,6 @@ void CommTiled::borders() MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][m],0,world); } } - if (recvother[iswap]) { - MPI_Waitall(nrecv,requests,MPI_STATUS_IGNORE); - for (m = 0; m < nrecv; m++) - avec->unpack_border_vel(recvnum[iswap][m],firstrecv[iswap][m], - &buf_recv[size_border* - forward_recv_offset[iswap][m]]); - } if (sendself[iswap]) { avec->pack_border_vel(sendnum[iswap][nsend],sendlist[iswap][nsend], buf_send,pbc_flag[iswap][nsend], @@ -1106,6 +1105,14 @@ void CommTiled::borders() avec->unpack_border_vel(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], buf_send); } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&m,MPI_STATUS_IGNORE); + avec->unpack_border_vel(recvnum[iswap][m],firstrecv[iswap][m], + &buf_recv[size_border* + forward_recv_offset[iswap][m]]); + } + } } else { if (recvother[iswap]) { @@ -1121,19 +1128,20 @@ void CommTiled::borders() MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][m],0,world); } } - if (recvother[iswap]) { - MPI_Waitall(nrecv,requests,MPI_STATUS_IGNORE); - for (m = 0; m < nrecv; m++) - avec->unpack_border(recvnum[iswap][m],firstrecv[iswap][m], - &buf_recv[size_border* - forward_recv_offset[iswap][m]]); - } if (sendself[iswap]) { avec->pack_border(sendnum[iswap][nsend],sendlist[iswap][nsend], buf_send,pbc_flag[iswap][nsend],pbc[iswap][nsend]); avec->unpack_border(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], buf_send); } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&m,MPI_STATUS_IGNORE); + avec->unpack_border(recvnum[iswap][m],firstrecv[iswap][m], + &buf_recv[size_border* + forward_recv_offset[iswap][m]]); + } + } } // increment ghost atoms @@ -1636,11 +1644,9 @@ int CommTiled::exchange_variable(int n, double * /*inbuf*/, double *& /*outbuf*/ void CommTiled::box_drop_brick(int idim, double *lo, double *hi, int &indexme) { - // NOTE: this is not triclinic compatible - // NOTE: these error messages are internal sanity checks - // should not occur, can be removed at some point + int dir; + int index = -1; - int index=-1,dir; if (hi[idim] == sublo[idim]) { index = myloc[idim] - 1; dir = -1;