From cd3a00c2c44086c7c2531e5f61c2985789e5658c Mon Sep 17 00:00:00 2001
From: Trung Nguyen <ndactrung@gmail.com>
Date: Wed, 14 Sep 2022 15:28:44 -0500
Subject: [PATCH] Added timing breakdown for fphi_uind

---
 lib/gpu/lal_hippo.cpp       | 27 ---------------------------
 lib/gpu/lal_hippo.h         |  1 -
 src/AMOEBA/pair_amoeba.cpp  |  7 ++++++-
 src/AMOEBA/pair_amoeba.h    |  1 +
 src/GPU/pair_amoeba_gpu.cpp |  7 +++++++
 5 files changed, 14 insertions(+), 29 deletions(-)

diff --git a/lib/gpu/lal_hippo.cpp b/lib/gpu/lal_hippo.cpp
index d980ae0ed6..79a8772c3e 100644
--- a/lib/gpu/lal_hippo.cpp
+++ b/lib/gpu/lal_hippo.cpp
@@ -592,33 +592,6 @@ int HippoT::umutual2b(const int eflag, const int vflag) {
   return GX;
 }
 
-// ---------------------------------------------------------------------------
-// Interpolate the potential from the PME grid
-// ---------------------------------------------------------------------------
-template <class numtyp, class acctyp>
-int HippoT::fphi_uind() {
-  int ainum=this->ans->inum();
-  if (ainum == 0)
-    return 0;
-
-  int _nall=this->atom->nall();
-  int nbor_pitch=this->nbor->nbor_pitch();
-
-  // Compute the block size and grid size to keep all cores busy
-  const int BX=this->block_size();
-  int GX=static_cast<int>(ceil(static_cast<double>(this->ans->inum())/
-                               (BX/this->_threads_per_atom)));
-/*
-  this->time_pair.start();
-
-  this->k_fphi_uind.set_size(GX,BX);
-  this->k_fphi_uind.run();
-  this->time_pair.stop();
-*/
-
-  return GX;
-}
-
 // ---------------------------------------------------------------------------
 // Reneighbor on GPU if necessary, and then compute polar real-space
 // ---------------------------------------------------------------------------
diff --git a/lib/gpu/lal_hippo.h b/lib/gpu/lal_hippo.h
index cece72caac..492712eb85 100644
--- a/lib/gpu/lal_hippo.h
+++ b/lib/gpu/lal_hippo.h
@@ -157,7 +157,6 @@ class Hippo : public BaseAmoeba<numtyp, acctyp> {
   int multipole_real(const int eflag, const int vflag);
   int udirect2b(const int eflag, const int vflag);
   int umutual2b(const int eflag, const int vflag);
-  int fphi_uind();
   int polar_real(const int eflag, const int vflag);
 
 };
diff --git a/src/AMOEBA/pair_amoeba.cpp b/src/AMOEBA/pair_amoeba.cpp
index 3b66ebc221..9890904e42 100644
--- a/src/AMOEBA/pair_amoeba.cpp
+++ b/src/AMOEBA/pair_amoeba.cpp
@@ -348,6 +348,7 @@ void PairAmoeba::compute(int eflag, int vflag)
     time_mutual_rspace = time_mutual_kspace = 0.0;
     time_polar_rspace = time_polar_kspace = 0.0;
 
+    time_fphi_uind = 0.0;
     if (ic_kspace) {
       ic_kspace->time_fft = 0.0;
     }
@@ -546,6 +547,9 @@ void PairAmoeba::finish()
   MPI_Allreduce(&time_polar_kspace,&ave,1,MPI_DOUBLE,MPI_SUM,world);
   time_polar_kspace = ave/comm->nprocs;
 
+  MPI_Allreduce(&time_fphi_uind,&ave,1,MPI_DOUBLE,MPI_SUM,world);
+  time_fphi_uind = ave/comm->nprocs;
+
   double time_mutual_fft = ic_kspace->time_fft;
   MPI_Allreduce(&time_mutual_fft,&ave,1,MPI_DOUBLE,MPI_SUM,world);
   time_mutual_fft = ave/comm->nprocs;
@@ -578,7 +582,8 @@ void PairAmoeba::finish()
     utils::logmesg(lmp,"      Mpole  time: {:.6g} {:.3g}%\n", time_mpole_kspace, time_mpole_kspace/time_total);
     utils::logmesg(lmp,"      Direct time: {:.6g} {:.3g}%\n", time_direct_kspace, time_direct_kspace/time_total);
     utils::logmesg(lmp,"      Mutual time: {:.6g} {:.3g}%\n", time_mutual_kspace, time_mutual_kspace/time_total);
-    utils::logmesg(lmp,"       - FFT time: {:.6g} {:.3g}%\n", time_mutual_fft, time_mutual_fft/time_total);
+    utils::logmesg(lmp,"       - FFT     : {:.6g} {:.3g}%\n", time_mutual_fft, time_mutual_fft/time_total);
+    utils::logmesg(lmp,"       - Interp  : {:.6g} {:.3g}%\n", time_fphi_uind, time_fphi_uind/time_total);
     utils::logmesg(lmp,"      Polar  time: {:.6g} {:.3g}%\n", time_polar_kspace, time_polar_kspace/time_total);
     
   }
diff --git a/src/AMOEBA/pair_amoeba.h b/src/AMOEBA/pair_amoeba.h
index 17b2d4a1e8..a95065d851 100644
--- a/src/AMOEBA/pair_amoeba.h
+++ b/src/AMOEBA/pair_amoeba.h
@@ -92,6 +92,7 @@ class PairAmoeba : public Pair {
   double time_direct_rspace,time_direct_kspace;
   double time_mutual_rspace,time_mutual_kspace;
   double time_polar_rspace,time_polar_kspace;
+  double time_fphi_uind;
 
   // energy/virial components
 
diff --git a/src/GPU/pair_amoeba_gpu.cpp b/src/GPU/pair_amoeba_gpu.cpp
index 3790ca4231..b85db8ea47 100644
--- a/src/GPU/pair_amoeba_gpu.cpp
+++ b/src/GPU/pair_amoeba_gpu.cpp
@@ -1058,9 +1058,16 @@ void PairAmoebaGPU::umutual1(double **field, double **fieldp)
   double ****gridpost = (double ****) ic_kspace->post_convolution();
 
   // get potential
+  double time0, time1;
+
+  MPI_Barrier(world);
+  time0 = MPI_Wtime();
 
   fphi_uind(gridpost,fdip_phi1,fdip_phi2,fdip_sum_phi);
 
+  time1 = MPI_Wtime();
+  time_fphi_uind += (time1 - time0);
+
   // store fractional reciprocal potentials for OPT method
 
   if (poltyp == OPT) {