jianmu
/
lammps
forked from lijiext/lammps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@7343 f3b2605a-c512-4ea7-a41b-209d697bcdaa

This commit is contained in:
sjplimp 2011-12-13 17:00:43 +00:00
parent 9b72a103ea
commit 3d36d38a51
4 changed files with 92 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/comm.cpp
View File

@ -53,7 +53,7 @@ using namespace LAMMPS_NS;
enum{SINGLE,MULTI};
enum{MULTIPLE}; // same as in ProcMap
enum{ONELEVEL,NUMA,CUSTOM};
enum{ONELEVEL,TWOLEVEL,NUMA,CUSTOM};
enum{CART,CARTREORDER,XYZ};
/* ----------------------------------------------------------------------
@ -173,6 +173,12 @@ void Comm::set_proc_grid()
otherflag,other_style,other_procgrid);
if (!flag) error->all(FLERR,"Could not create grid of processors");
} else if (gridflag == TWOLEVEL) {
flag = pmap->twolevel_grid(nprocs,user_procgrid,procgrid,
ncores,user_coregrid,coregrid,
otherflag,other_style,other_procgrid);
if (!flag) error->all(FLERR,"Could not create grid of processors");
} else if (gridflag == NUMA) {
flag = pmap->numa_grid(nprocs,user_procgrid,procgrid,coregrid);
if (!flag) {
@ -183,7 +189,7 @@ void Comm::set_proc_grid()
if (!flag) error->all(FLERR,"Could not create grid of processors");
} else if (gridflag == CUSTOM) {
pmap->custom_grid(nprocs,user_procgrid,procgrid);
pmap->custom_grid(customfile,nprocs,user_procgrid,procgrid);
}
// error check on procgrid
@ -210,6 +216,14 @@ void Comm::set_proc_grid()
else if (mapflag == XYZ)
pmap->xyz_map(xyz,procgrid,myloc,procneigh,grid2proc);
} else if (gridflag == TWOLEVEL) {
if (mapflag == CART)
pmap->cart_map(0,procgrid,coregrid,myloc,procneigh,grid2proc);
else if (mapflag == CARTREORDER)
pmap->cart_map(1,procgrid,coregrid,myloc,procneigh,grid2proc);
else if (mapflag == XYZ)
pmap->xyz_map(xyz,procgrid,coregrid,myloc,procneigh,grid2proc);
} else if (gridflag == NUMA) {
pmap->numa_map(coregrid,myloc,procneigh,grid2proc);
@ -223,14 +237,14 @@ void Comm::set_proc_grid()
if (screen) {
fprintf(screen," %d by %d by %d MPI processor grid\n",
procgrid[0],procgrid[1],procgrid[2]);
if (gridflag == NUMA)
if (gridflag == NUMA || gridflag == TWOLEVEL)
fprintf(screen," %d by %d by %d core grid within node\n",
coregrid[0],coregrid[1],coregrid[2]);
}
if (logfile) {
fprintf(logfile," %d by %d by %d MPI processor grid\n",
procgrid[0],procgrid[1],procgrid[2]);
if (gridflag == NUMA)
if (gridflag == NUMA || gridflag == TWOLEVEL)
fprintf(logfile," %d by %d by %d core grid within node\n",
coregrid[0],coregrid[1],coregrid[2]);
}
@ -1357,30 +1371,42 @@ void Comm::set_processors(int narg, char **arg)
int iarg = 3;
while (iarg < narg) {
if (strcmp(arg[iarg],"grid") == 0) {
if (iarg+1 > narg) error->all(FLERR,"Illegal processors command");
if (iarg+2 > narg) error->all(FLERR,"Illegal processors command");
if (strcmp(arg[iarg+1],"level1") == 0) {
gridflag = LEVEL1;
iarg += 2;
if (strcmp(arg[iarg+1],"onelevel") == 0) {
gridflag = ONELEVEL;
if (strcmp(arg[iarg+1],"level2") == 0) {
gridflag = LEVEL2;
iarg += 2;
} else if (strcmp(arg[iarg+1],"twolevel") == 0) {
if (iarg+6 > narg) error->all(FLERR,"Illegal processors command");
gridflag = TWOLEVEL;
ncores = atoi(arg[2]);
if (strcmp(arg[3],"*") == 0) user_coregrid[0] = 0;
else user_coregrid[0] = atoi(arg[3]);
if (strcmp(arg[4],"*") == 0) user_coregrid[1] = 0;
else user_coregrid[1] = atoi(arg[4]);
if (strcmp(arg[5],"*") == 0) user_coregrid[2] = 0;
else user_coregrid[2] = atoi(arg[5]);
if (ncores <= 0 || user_coregrid[0] < 0 ||
user_coregrid[1] < 0 || user_coregrid[2] < 0)
error->all(FLERR,"Illegal processors command");
iarg += 4;
} else if (strcmp(arg[iarg+1],"numa") == 0) {
gridflag = NUMA;
iarg += 2;
} else if (strcmp(arg[iarg],"custom") == 0) {
if (iarg+2 > narg) error->all(FLERR,"Illegal processors command");
if (iarg+3 > narg) error->all(FLERR,"Illegal processors command");
gridflag = CUSTOM;
delete [] customfile;
int n = strlen(arg[iarg+1]) + 1;
int n = strlen(arg[iarg+2]) + 1;
customfile = new char(n);
strcpy(customfile,arg[iarg+1]);
iarg += 2;
strcpy(customfile,arg[iarg+2]);
iarg += 1;
} else error->all(FLERR,"Illegal processors command");
iarg += 2;
} else if (strcmp(arg[iarg],"map") == 0) {
if (iarg+2 > narg) error->all(FLERR,"Illegal processors command");
@ -1433,7 +1459,7 @@ void Comm::set_processors(int narg, char **arg)
} else error->all(FLERR,"Illegal processors command");
iarg += 4;
} else if (strcmp(arg[iarg],"out") == 0) {
} else if (strcmp(arg[iarg],"file") == 0) {
if (iarg+2 > narg) error->all(FLERR,"Illegal processors command");
delete [] outfile;
int n = strlen(arg[iarg+1]) + 1;
@ -1444,10 +1470,13 @@ void Comm::set_processors(int narg, char **arg)
} else error->all(FLERR,"Illegal processors command");
}
// error check
// error checks
if (gridflag == NUMA && mapflag != CART)
error->all(FLERR,"Processors grid numa and map choice are incompatible");
error->all(FLERR,"Processors grid numa and map style are incompatible");
if (otherflag && (gridflag == NUMA || gridflag == CUSTOM))
error->all(FLERR,
"Processors part option and grid style are incompatible");
}
/* ----------------------------------------------------------------------

6
src/comm.h
View File

@ -21,7 +21,7 @@ namespace LAMMPS_NS {
class Comm : protected Pointers {
public:
int me,nprocs; // proc info
int procgrid[3]; // assigned # of procs in each dim
int procgrid[3]; // procs assigned in each dim of 3d grid
int user_procgrid[3]; // user request for procs in each dim
int myloc[3]; // which proc I am in each dim
int procneigh[3][2]; // my 6 neighboring procs
@ -92,7 +92,9 @@ class Comm : protected Pointers {
int otherflag; // 1 if this partition dependent on another
int other_style; // style of dependency
int other_procgrid[3]; // proc layout of another partition
int coregrid[3]; // procs layout of cores within a node
int ncores; // # of cores per node
int coregrid[3]; // 3d grid of cores within a node
int user_coregrid[3]; // user request for cores in each dim
int *firstrecv; // where to put 1st recv atom in each swap
int **sendlist; // list of atoms to send in each swap

38
src/procmap.cpp
View File

@ -33,7 +33,7 @@ enum{MULTIPLE}; // same as in Comm
ProcMap::ProcMap(LAMMPS *lmp) : Pointers(lmp) {}
/* ----------------------------------------------------------------------
create a 1-level 3d grid of procs via procs2box()
create a one-level 3d grid of procs via procs2box()
------------------------------------------------------------------------- */
int ProcMap::onelevel_grid(int nprocs, int *user_procgrid, int *procgrid,
@ -49,6 +49,18 @@ int ProcMap::onelevel_grid(int nprocs, int *user_procgrid, int *procgrid,
return flag;
}
/* ----------------------------------------------------------------------
create a two-level 3d grid of procs and cores via procs2box()
------------------------------------------------------------------------- */
int ProcMap::twolevel_grid(int nprocs, int *user_procgrid, int *procgrid,
int ncores, int *user_coregrid, int *coregrid,
int otherflag, int other_style_caller,
int *other_procgrid_caller)
{
return 1;
}
/* ----------------------------------------------------------------------
create a 3d grid of procs that does a 2-level hierarchy within a node
auto-detects NUMA sockets within a multi-core node
@ -151,7 +163,8 @@ int ProcMap::numa_grid(int nprocs, int *user_procgrid, int *procgrid,
create a 1-level 3d grid of procs via procs2box()
------------------------------------------------------------------------- */
void ProcMap::custom_grid(int nprocs, int *user_procgrid, int *procgrid)
void ProcMap::custom_grid(char *cfile, int nprocs,
int *user_procgrid, int *procgrid)
{
}
@ -315,6 +328,17 @@ void ProcMap::cart_map(int reorder, int *procgrid,
MPI_Comm_free(&cartesian);
}
/* ----------------------------------------------------------------------
map processors to 3d grid via MPI_Cart routines
respect sub-grid of cores within each node
MPI may do layout in machine-optimized fashion
------------------------------------------------------------------------- */
void ProcMap::cart_map(int reorder, int *procgrid, int *coregrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
}
/* ----------------------------------------------------------------------
map processors to 3d grid in XYZ order
------------------------------------------------------------------------- */
@ -362,6 +386,16 @@ void ProcMap::xyz_map(char *xyz, int *procgrid,
procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus];
}
/* ----------------------------------------------------------------------
map processors to 3d grid in XYZ order
respect sub-grid of cores within each node
------------------------------------------------------------------------- */
void ProcMap::xyz_map(char *xyz, int *procgrid, int *coregrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
}
/* ----------------------------------------------------------------------
map processors to 3d grid in 2-level NUMA ordering
------------------------------------------------------------------------- */

5
src/procmap.h
View File

@ -23,10 +23,13 @@ class ProcMap : protected Pointers {
ProcMap(class LAMMPS *);
~ProcMap() {}
int onelevel_grid(int, int *, int *, int, int, int *);
int twolevel_grid(int, int *, int *, int, int *, int *, int, int, int *);
int numa_grid(int, int *, int *, int *);
void custom_grid(int, int *, int *);
void custom_grid(char *, int, int *, int *);
void cart_map(int, int *, int *, int [3][2], int ***);
void cart_map(int, int *, int *, int *, int [3][2], int ***);
void xyz_map(char *, int *, int *, int [3][2], int ***);
void xyz_map(char *, int *, int *, int *, int [3][2], int ***);
void numa_map(int *, int *, int [3][2], int ***);
void custom_map(int *, int [3][2], int ***);
void output(int ***, char *);