From 3d36d38a5183587bd056cc51af3d0c3c73de6494 Mon Sep 17 00:00:00 2001 From: sjplimp Date: Tue, 13 Dec 2011 17:00:43 +0000 Subject: [PATCH] git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@7343 f3b2605a-c512-4ea7-a41b-209d697bcdaa --- src/comm.cpp | 67 +++++++++++++++++++++++++++++++++++-------------- src/comm.h | 6 +++-- src/procmap.cpp | 38 ++++++++++++++++++++++++++-- src/procmap.h | 5 +++- 4 files changed, 92 insertions(+), 24 deletions(-) diff --git a/src/comm.cpp b/src/comm.cpp index e8f4c0ca54..d0e1eea634 100644 --- a/src/comm.cpp +++ b/src/comm.cpp @@ -53,7 +53,7 @@ using namespace LAMMPS_NS; enum{SINGLE,MULTI}; enum{MULTIPLE}; // same as in ProcMap -enum{ONELEVEL,NUMA,CUSTOM}; +enum{ONELEVEL,TWOLEVEL,NUMA,CUSTOM}; enum{CART,CARTREORDER,XYZ}; /* ---------------------------------------------------------------------- @@ -173,6 +173,12 @@ void Comm::set_proc_grid() otherflag,other_style,other_procgrid); if (!flag) error->all(FLERR,"Could not create grid of processors"); + } else if (gridflag == TWOLEVEL) { + flag = pmap->twolevel_grid(nprocs,user_procgrid,procgrid, + ncores,user_coregrid,coregrid, + otherflag,other_style,other_procgrid); + if (!flag) error->all(FLERR,"Could not create grid of processors"); + } else if (gridflag == NUMA) { flag = pmap->numa_grid(nprocs,user_procgrid,procgrid,coregrid); if (!flag) { @@ -183,7 +189,7 @@ void Comm::set_proc_grid() if (!flag) error->all(FLERR,"Could not create grid of processors"); } else if (gridflag == CUSTOM) { - pmap->custom_grid(nprocs,user_procgrid,procgrid); + pmap->custom_grid(customfile,nprocs,user_procgrid,procgrid); } // error check on procgrid @@ -210,6 +216,14 @@ void Comm::set_proc_grid() else if (mapflag == XYZ) pmap->xyz_map(xyz,procgrid,myloc,procneigh,grid2proc); + } else if (gridflag == TWOLEVEL) { + if (mapflag == CART) + pmap->cart_map(0,procgrid,coregrid,myloc,procneigh,grid2proc); + else if (mapflag == CARTREORDER) + pmap->cart_map(1,procgrid,coregrid,myloc,procneigh,grid2proc); + else if (mapflag == XYZ) + pmap->xyz_map(xyz,procgrid,coregrid,myloc,procneigh,grid2proc); + } else if (gridflag == NUMA) { pmap->numa_map(coregrid,myloc,procneigh,grid2proc); @@ -223,14 +237,14 @@ void Comm::set_proc_grid() if (screen) { fprintf(screen," %d by %d by %d MPI processor grid\n", procgrid[0],procgrid[1],procgrid[2]); - if (gridflag == NUMA) + if (gridflag == NUMA || gridflag == TWOLEVEL) fprintf(screen," %d by %d by %d core grid within node\n", coregrid[0],coregrid[1],coregrid[2]); } if (logfile) { fprintf(logfile," %d by %d by %d MPI processor grid\n", procgrid[0],procgrid[1],procgrid[2]); - if (gridflag == NUMA) + if (gridflag == NUMA || gridflag == TWOLEVEL) fprintf(logfile," %d by %d by %d core grid within node\n", coregrid[0],coregrid[1],coregrid[2]); } @@ -1357,30 +1371,42 @@ void Comm::set_processors(int narg, char **arg) int iarg = 3; while (iarg < narg) { if (strcmp(arg[iarg],"grid") == 0) { - if (iarg+1 > narg) error->all(FLERR,"Illegal processors command"); + if (iarg+2 > narg) error->all(FLERR,"Illegal processors command"); - if (strcmp(arg[iarg+1],"level1") == 0) { - gridflag = LEVEL1; - iarg += 2; + if (strcmp(arg[iarg+1],"onelevel") == 0) { + gridflag = ONELEVEL; - if (strcmp(arg[iarg+1],"level2") == 0) { - gridflag = LEVEL2; - iarg += 2; + } else if (strcmp(arg[iarg+1],"twolevel") == 0) { + if (iarg+6 > narg) error->all(FLERR,"Illegal processors command"); + gridflag = TWOLEVEL; + + ncores = atoi(arg[2]); + if (strcmp(arg[3],"*") == 0) user_coregrid[0] = 0; + else user_coregrid[0] = atoi(arg[3]); + if (strcmp(arg[4],"*") == 0) user_coregrid[1] = 0; + else user_coregrid[1] = atoi(arg[4]); + if (strcmp(arg[5],"*") == 0) user_coregrid[2] = 0; + else user_coregrid[2] = atoi(arg[5]); + + if (ncores <= 0 || user_coregrid[0] < 0 || + user_coregrid[1] < 0 || user_coregrid[2] < 0) + error->all(FLERR,"Illegal processors command"); + iarg += 4; } else if (strcmp(arg[iarg+1],"numa") == 0) { gridflag = NUMA; - iarg += 2; } else if (strcmp(arg[iarg],"custom") == 0) { - if (iarg+2 > narg) error->all(FLERR,"Illegal processors command"); + if (iarg+3 > narg) error->all(FLERR,"Illegal processors command"); gridflag = CUSTOM; delete [] customfile; - int n = strlen(arg[iarg+1]) + 1; + int n = strlen(arg[iarg+2]) + 1; customfile = new char(n); - strcpy(customfile,arg[iarg+1]); - iarg += 2; + strcpy(customfile,arg[iarg+2]); + iarg += 1; } else error->all(FLERR,"Illegal processors command"); + iarg += 2; } else if (strcmp(arg[iarg],"map") == 0) { if (iarg+2 > narg) error->all(FLERR,"Illegal processors command"); @@ -1433,7 +1459,7 @@ void Comm::set_processors(int narg, char **arg) } else error->all(FLERR,"Illegal processors command"); iarg += 4; - } else if (strcmp(arg[iarg],"out") == 0) { + } else if (strcmp(arg[iarg],"file") == 0) { if (iarg+2 > narg) error->all(FLERR,"Illegal processors command"); delete [] outfile; int n = strlen(arg[iarg+1]) + 1; @@ -1444,10 +1470,13 @@ void Comm::set_processors(int narg, char **arg) } else error->all(FLERR,"Illegal processors command"); } - // error check + // error checks if (gridflag == NUMA && mapflag != CART) - error->all(FLERR,"Processors grid numa and map choice are incompatible"); + error->all(FLERR,"Processors grid numa and map style are incompatible"); + if (otherflag && (gridflag == NUMA || gridflag == CUSTOM)) + error->all(FLERR, + "Processors part option and grid style are incompatible"); } /* ---------------------------------------------------------------------- diff --git a/src/comm.h b/src/comm.h index 706b7766aa..4ab8d18422 100644 --- a/src/comm.h +++ b/src/comm.h @@ -21,7 +21,7 @@ namespace LAMMPS_NS { class Comm : protected Pointers { public: int me,nprocs; // proc info - int procgrid[3]; // assigned # of procs in each dim + int procgrid[3]; // procs assigned in each dim of 3d grid int user_procgrid[3]; // user request for procs in each dim int myloc[3]; // which proc I am in each dim int procneigh[3][2]; // my 6 neighboring procs @@ -92,7 +92,9 @@ class Comm : protected Pointers { int otherflag; // 1 if this partition dependent on another int other_style; // style of dependency int other_procgrid[3]; // proc layout of another partition - int coregrid[3]; // procs layout of cores within a node + int ncores; // # of cores per node + int coregrid[3]; // 3d grid of cores within a node + int user_coregrid[3]; // user request for cores in each dim int *firstrecv; // where to put 1st recv atom in each swap int **sendlist; // list of atoms to send in each swap diff --git a/src/procmap.cpp b/src/procmap.cpp index e412674224..f0a9a57359 100644 --- a/src/procmap.cpp +++ b/src/procmap.cpp @@ -33,7 +33,7 @@ enum{MULTIPLE}; // same as in Comm ProcMap::ProcMap(LAMMPS *lmp) : Pointers(lmp) {} /* ---------------------------------------------------------------------- - create a 1-level 3d grid of procs via procs2box() + create a one-level 3d grid of procs via procs2box() ------------------------------------------------------------------------- */ int ProcMap::onelevel_grid(int nprocs, int *user_procgrid, int *procgrid, @@ -49,6 +49,18 @@ int ProcMap::onelevel_grid(int nprocs, int *user_procgrid, int *procgrid, return flag; } +/* ---------------------------------------------------------------------- + create a two-level 3d grid of procs and cores via procs2box() +------------------------------------------------------------------------- */ + +int ProcMap::twolevel_grid(int nprocs, int *user_procgrid, int *procgrid, + int ncores, int *user_coregrid, int *coregrid, + int otherflag, int other_style_caller, + int *other_procgrid_caller) +{ + return 1; +} + /* ---------------------------------------------------------------------- create a 3d grid of procs that does a 2-level hierarchy within a node auto-detects NUMA sockets within a multi-core node @@ -151,7 +163,8 @@ int ProcMap::numa_grid(int nprocs, int *user_procgrid, int *procgrid, create a 1-level 3d grid of procs via procs2box() ------------------------------------------------------------------------- */ -void ProcMap::custom_grid(int nprocs, int *user_procgrid, int *procgrid) +void ProcMap::custom_grid(char *cfile, int nprocs, + int *user_procgrid, int *procgrid) { } @@ -315,6 +328,17 @@ void ProcMap::cart_map(int reorder, int *procgrid, MPI_Comm_free(&cartesian); } +/* ---------------------------------------------------------------------- + map processors to 3d grid via MPI_Cart routines + respect sub-grid of cores within each node + MPI may do layout in machine-optimized fashion +------------------------------------------------------------------------- */ + +void ProcMap::cart_map(int reorder, int *procgrid, int *coregrid, + int *myloc, int procneigh[3][2], int ***grid2proc) +{ +} + /* ---------------------------------------------------------------------- map processors to 3d grid in XYZ order ------------------------------------------------------------------------- */ @@ -362,6 +386,16 @@ void ProcMap::xyz_map(char *xyz, int *procgrid, procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus]; } +/* ---------------------------------------------------------------------- + map processors to 3d grid in XYZ order + respect sub-grid of cores within each node +------------------------------------------------------------------------- */ + +void ProcMap::xyz_map(char *xyz, int *procgrid, int *coregrid, + int *myloc, int procneigh[3][2], int ***grid2proc) +{ +} + /* ---------------------------------------------------------------------- map processors to 3d grid in 2-level NUMA ordering ------------------------------------------------------------------------- */ diff --git a/src/procmap.h b/src/procmap.h index 432d9d32fb..5d740acf5a 100644 --- a/src/procmap.h +++ b/src/procmap.h @@ -23,10 +23,13 @@ class ProcMap : protected Pointers { ProcMap(class LAMMPS *); ~ProcMap() {} int onelevel_grid(int, int *, int *, int, int, int *); + int twolevel_grid(int, int *, int *, int, int *, int *, int, int, int *); int numa_grid(int, int *, int *, int *); - void custom_grid(int, int *, int *); + void custom_grid(char *, int, int *, int *); void cart_map(int, int *, int *, int [3][2], int ***); + void cart_map(int, int *, int *, int *, int [3][2], int ***); void xyz_map(char *, int *, int *, int [3][2], int ***); + void xyz_map(char *, int *, int *, int *, int [3][2], int ***); void numa_map(int *, int *, int [3][2], int ***); void custom_map(int *, int [3][2], int ***); void output(int ***, char *);