From 371df8ea72d6a5e00e44c78de2eb7fa3a2381316 Mon Sep 17 00:00:00 2001
From: Dan Ibanez <daibane@sandia.gov>
Date: Fri, 23 Dec 2016 13:08:34 -0700
Subject: [PATCH] repair sync bugs in fix_momentum_kokkos

Since the Group class is completely
unaware of Kokkos, the direct calls from
FixMomentumKokkos to Group methods
need to be preceded by atomKK->sync calls
for every atom variable that Group intends
to use.
fix_momentum_kokkos definitely does not
work on GPUs prior to this commit.
---
 src/KOKKOS/fix_momentum_kokkos.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/KOKKOS/fix_momentum_kokkos.cpp b/src/KOKKOS/fix_momentum_kokkos.cpp
index 333889ab98..8dc4a15551 100644
--- a/src/KOKKOS/fix_momentum_kokkos.cpp
+++ b/src/KOKKOS/fix_momentum_kokkos.cpp
@@ -108,6 +108,9 @@ void FixMomentumKokkos<DeviceType>::end_of_step()
 
   auto groupbit2 = groupbit;
   if (linear) {
+    /* this is needed because Group is not Kokkos-aware ! */
+    atomKK->sync(ExecutionSpaceFromDevice<LMPHostType>::space,
+        V_MASK | MASK_MASK | TYPE_MASK | RMASS_MASK);
     Few<double, 3> vcm;
     group->vcm(igroup,masstotal,&vcm[0]);
 
@@ -125,13 +128,21 @@ void FixMomentumKokkos<DeviceType>::end_of_step()
         if (zflag2) v(i,2) -= vcm[2];
       }
     });
+    atomKK->modified(execution_space, V_MASK);
   }
 
   if (angular) {
     Few<double, 3> xcm, angmom, omega;
     double inertia[3][3];
+    /* syncs for each Kokkos-unaware Group method */
+    atomKK->sync(ExecutionSpaceFromDevice<LMPHostType>::space,
+        X_MASK | MASK_MASK | TYPE_MASK | IMAGE_MASK | RMASS_MASK);
     group->xcm(igroup,masstotal,&xcm[0]);
+    atomKK->sync(ExecutionSpaceFromDevice<LMPHostType>::space,
+        X_MASK | V_MASK | MASK_MASK | TYPE_MASK | IMAGE_MASK | RMASS_MASK);
     group->angmom(igroup,&xcm[0],&angmom[0]);
+    atomKK->sync(ExecutionSpaceFromDevice<LMPHostType>::space,
+        X_MASK | MASK_MASK | TYPE_MASK | IMAGE_MASK | RMASS_MASK);
     group->inertia(igroup,&xcm[0],inertia);
     group->omega(&angmom[0],inertia,&omega[0]);
 
@@ -162,6 +173,7 @@ void FixMomentumKokkos<DeviceType>::end_of_step()
         v(i,2) -= omega[0]*dy - omega[1]*dx;
       }
     });
+    atomKK->modified(execution_space, V_MASK);
   }
 
   // compute kinetic energy after momentum removal, if needed
@@ -179,11 +191,8 @@ void FixMomentumKokkos<DeviceType>::end_of_step()
         v(i,2) *= factor;
       }
     });
+    atomKK->modified(execution_space, V_MASK);
   }
-
-  atomKK->modified(execution_space, V_MASK);
-  // the following sync should not be needed once everything supports Kokkos
-  atomKK->sync(ExecutionSpaceFromDevice<LMPHostType>::space, V_MASK);
 }
 
 namespace LAMMPS_NS {