jianmu
/
lammps
forked from lijiext/lammps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
lammps/src/procmap.cpp

901 lines
30 KiB
C++
Raw Blame History

/* ----------------------------------------------------------------------
LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
http://lammps.sandia.gov, Sandia National Laboratories
Steve Plimpton, sjplimp@sandia.gov
Copyright (2003) Sandia Corporation. Under the terms of Contract
DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
certain rights in this software. This software is distributed under
the GNU General Public License.
See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */
/* ----------------------------------------------------------------------
Contributing author (NUMA option) : Mike Brown (ORNL)
------------------------------------------------------------------------- */
#include "procmap.h"
#include <mpi.h>
#include <cmath>
#include <cstring>
#include <map>
#include <string>
#include <utility>
#include "universe.h"
#include "comm.h"
#include "domain.h"
#include "math_extra.h"
#include "memory.h"
#include "error.h"
using namespace LAMMPS_NS;
#define MAXLINE 128
/* ---------------------------------------------------------------------- */
ProcMap::ProcMap(LAMMPS *lmp) : Pointers(lmp) {}
/* ----------------------------------------------------------------------
create a one-level 3d grid of procs
------------------------------------------------------------------------- */
void ProcMap::onelevel_grid(int nprocs, int *user_procgrid, int *procgrid,
int otherflag, int other_style,
int *other_procgrid, int *other_coregrid)
{
int **factors;
// factors = list of all possible 3 factors of processor count
int npossible = factor(nprocs,NULL);
memory->create(factors,npossible,3,"procmap:factors");
npossible = factor(nprocs,factors);
// constrain by 2d, user request, other partition
if (domain->dimension == 2) npossible = cull_2d(npossible,factors,3);
npossible = cull_user(npossible,factors,3,user_procgrid);
if (otherflag) npossible = cull_other(npossible,factors,3,
other_style,other_procgrid,
other_coregrid);
// user/other constraints make failure possible
if (npossible == 0)
error->all(FLERR,"Could not create 3d grid of processors");
// select best set of 3 factors based on surface area of proc sub-domains
best_factors(npossible,factors,procgrid,1,1,1);
// clean-up
memory->destroy(factors);
}
/* ----------------------------------------------------------------------
create a two-level 3d grid of procs
------------------------------------------------------------------------- */
void ProcMap::twolevel_grid(int nprocs, int *user_procgrid, int *procgrid,
int ncores, int *user_coregrid, int *coregrid,
int otherflag, int other_style,
int *other_procgrid, int *other_coregrid)
{
int **nfactors,**cfactors,**factors;
if (nprocs % ncores)
error->all(FLERR,"Processors twogrid requires proc count "
"be a multiple of core count");
// nfactors = list of all possible 3 factors of node count
// constrain by 2d
int nnpossible = factor(nprocs/ncores,NULL);
memory->create(nfactors,nnpossible,3,"procmap:nfactors");
nnpossible = factor(nprocs/ncores,nfactors);
if (domain->dimension == 2) nnpossible = cull_2d(nnpossible,nfactors,3);
// cfactors = list of all possible 3 factors of core count
// constrain by 2d
int ncpossible = factor(ncores,NULL);
memory->create(cfactors,ncpossible,3,"procmap:cfactors");
ncpossible = factor(ncores,cfactors);
if (domain->dimension == 2) ncpossible = cull_2d(ncpossible,cfactors,3);
ncpossible = cull_user(ncpossible,cfactors,3,user_coregrid);
// factors = all combinations of nfactors and cfactors
// factors stores additional index pointing to corresponding cfactors
// constrain by user request, other partition
int npossible = nnpossible * ncpossible;
memory->create(factors,npossible,4,"procmap:factors");
npossible = combine_factors(nnpossible,nfactors,ncpossible,cfactors,factors);
npossible = cull_user(npossible,factors,4,user_procgrid);
if (otherflag) npossible = cull_other(npossible,factors,4,
other_style,other_procgrid,
other_coregrid);
// user/other constraints make failure possible
if (npossible == 0)
error->all(FLERR,"Could not create twolevel 3d grid of processors");
// select best set of 3 factors based on surface area of proc sub-domains
// index points to corresponding core factorization
int index = best_factors(npossible,factors,procgrid,1,1,1);
coregrid[0] = cfactors[factors[index][3]][0];
coregrid[1] = cfactors[factors[index][3]][1];
coregrid[2] = cfactors[factors[index][3]][2];
// clean-up
memory->destroy(nfactors);
memory->destroy(cfactors);
memory->destroy(factors);
}
/* ----------------------------------------------------------------------
create a 3d grid of procs that does a 2-level hierarchy within a node
auto-detects NUMA sockets within a multi-core node
------------------------------------------------------------------------- */
void ProcMap::numa_grid(int nprocs, int *user_procgrid, int *procgrid,
int *numagrid)
{
// hardwire this for now
int numa_nodes = 1;
// get names of all nodes
int name_length;
char node_name[MPI_MAX_PROCESSOR_NAME];
MPI_Get_processor_name(node_name,&name_length);
node_name[name_length] = '\0';
char *node_names = new char[MPI_MAX_PROCESSOR_NAME*nprocs];
MPI_Allgather(node_name,MPI_MAX_PROCESSOR_NAME,MPI_CHAR,node_names,
MPI_MAX_PROCESSOR_NAME,MPI_CHAR,world);
std::string node_string = std::string(node_name);
// get number of procs per node
// NOTE: could do this without STL map
std::map<std::string,int> name_map;
std::map<std::string,int>::iterator np;
for (int i = 0; i < nprocs; i++) {
std::string i_string = std::string(&node_names[i*MPI_MAX_PROCESSOR_NAME]);
np = name_map.find(i_string);
if (np == name_map.end()) name_map[i_string] = 1;
else np->second++;
}
procs_per_node = name_map.begin()->second;
procs_per_numa = procs_per_node / numa_nodes;
delete [] node_names;
// error if any of these conditions met
if (nprocs % procs_per_numa || // total procs not a multiple of node
user_procgrid[0] > 1 || // user specified grid > 1 in any dim
user_procgrid[1] > 1 ||
user_procgrid[2] > 1)
error->all(FLERR,"Could not create numa grid of processors");
// user settings for the factorization per numa node
// currently not user settable
// if user specifies 1 for a proc grid dimension,
// also use 1 for the numa grid dimension
int user_numagrid[3];
user_numagrid[0] = user_numagrid[1] = user_numagrid[2] = 0;
if (user_procgrid[0] == 1) user_numagrid[0] = 1;
if (user_procgrid[1] == 1) user_numagrid[1] = 1;
if (user_procgrid[2] == 1) user_numagrid[2] = 1;
// initial factorization within NUMA node
int **numafactors;
int numapossible = factor(procs_per_numa,NULL);
memory->create(numafactors,numapossible,3,"procmap:numafactors");
numapossible = factor(procs_per_numa,numafactors);
if (domain->dimension == 2)
numapossible = cull_2d(numapossible,numafactors,3);
numapossible = cull_user(numapossible,numafactors,3,user_numagrid);
if (numapossible == 0)
error->all(FLERR,"Could not create numa grid of processors");
best_factors(numapossible,numafactors,numagrid,1,1,1);
// user_nodegrid = implied user constraints on nodes
int user_nodegrid[3];
user_nodegrid[0] = user_procgrid[0] / numagrid[0];
user_nodegrid[1] = user_procgrid[1] / numagrid[1];
user_nodegrid[2] = user_procgrid[2] / numagrid[2];
// factorization for the grid of NUMA nodes
int node_count = nprocs / procs_per_numa;
int **nodefactors;
int nodepossible = factor(node_count,NULL);
memory->create(nodefactors,nodepossible,3,"procmap:nodefactors");
nodepossible = factor(node_count,nodefactors);
if (domain->dimension == 2)
nodepossible = cull_2d(nodepossible,nodefactors,3);
nodepossible = cull_user(nodepossible,nodefactors,3,user_nodegrid);
if (nodepossible == 0)
error->all(FLERR,"Could not create numa grid of processors");
best_factors(nodepossible,nodefactors,nodegrid,
numagrid[0],numagrid[1],numagrid[2]);
// repeat NUMA node factorization using subdomain sizes
// refines the factorization if the user specified the node layout
// NOTE: this will not re-enforce user-procgrid constraint will it?
best_factors(numapossible,numafactors,numagrid,
nodegrid[0],nodegrid[1],nodegrid[2]);
memory->destroy(numafactors);
memory->destroy(nodefactors);
// assign a unique id to each node
node_id = 0;
int node_num = 0;
for (np = name_map.begin(); np != name_map.end(); ++np) {
if (np->first == node_string) node_id = node_num;
node_num++;
}
// return the proc-level factorization
procgrid[0] = nodegrid[0] * numagrid[0];
procgrid[1] = nodegrid[1] * numagrid[1];
procgrid[2] = nodegrid[2] * numagrid[2];
}
/* ----------------------------------------------------------------------
define a 3d grid from a custom file
------------------------------------------------------------------------- */
void ProcMap::custom_grid(char *cfile, int nprocs,
int *user_procgrid, int *procgrid)
{
int me;
MPI_Comm_rank(world,&me);
char line[MAXLINE];
FILE *fp = NULL;
if (me == 0) {
fp = fopen(cfile,"r");
if (fp == NULL) error->one(FLERR,"Cannot open custom file");
// skip header = blank and comment lines
char *ptr;
if (!fgets(line,MAXLINE,fp))
error->one(FLERR,"Unexpected end of custom file");
while (1) {
if ((ptr = strchr(line,'#'))) *ptr = '\0';
if (strspn(line," \t\n\r") != strlen(line)) break;
if (!fgets(line,MAXLINE,fp))
error->one(FLERR,"Unexpected end of custom file");
}
}
int n = strlen(line) + 1;
MPI_Bcast(&n,1,MPI_INT,0,world);
MPI_Bcast(line,n,MPI_CHAR,0,world);
int rv = sscanf(line,"%d %d %d",&procgrid[0],&procgrid[1],&procgrid[2]);
if (rv != 3) error->all(FLERR,"Processors custom grid file is inconsistent");
int flag = 0;
if (procgrid[0]*procgrid[1]*procgrid[2] != nprocs) flag = 1;
if (user_procgrid[0] && procgrid[0] != user_procgrid[0]) flag = 1;
if (user_procgrid[1] && procgrid[1] != user_procgrid[1]) flag = 1;
if (user_procgrid[2] && procgrid[2] != user_procgrid[2]) flag = 1;
if (flag) error->all(FLERR,"Processors custom grid file is inconsistent");
// cmap = map of procs to grid
// store for use in custom_map()
memory->create(cmap,nprocs,4,"procmap:cmap");
for (int i = 0; i < nprocs; i++) cmap[i][0] = -1;
if (me == 0) {
for (int i = 0; i < nprocs; i++) {
if (!fgets(line,MAXLINE,fp))
error->one(FLERR,"Unexpected end of custom file");
rv = sscanf(line,"%d %d %d %d",
&cmap[i][0],&cmap[i][1],&cmap[i][2],&cmap[i][3]);
if (rv != 4)
error->one(FLERR,"Processors custom grid file is inconsistent");
}
fclose(fp);
}
MPI_Bcast(&cmap[0][0],nprocs*4,MPI_INT,0,world);
// error check on cmap values
flag = 0;
for (int i = 0; i < nprocs; i++) {
if (cmap[i][0] == -1) flag = 1;
else {
if (cmap[i][1] <= 0 || cmap[i][1] > procgrid[0]) flag = 1;
if (cmap[i][2] <= 0 || cmap[i][2] > procgrid[1]) flag = 1;
if (cmap[i][3] <= 0 || cmap[i][3] > procgrid[2]) flag = 1;
}
}
if (flag) error->all(FLERR,"Processors custom grid file is inconsistent");
}
/* ----------------------------------------------------------------------
map processors to 3d grid via MPI_Cart routines
MPI may do layout in machine-optimized fashion
------------------------------------------------------------------------- */
void ProcMap::cart_map(int reorder, int *procgrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
int periods[3];
periods[0] = periods[1] = periods[2] = 1;
MPI_Comm cartesian;
MPI_Cart_create(world,3,procgrid,periods,reorder,&cartesian);
MPI_Cart_get(cartesian,3,procgrid,periods,myloc);
MPI_Cart_shift(cartesian,0,1,&procneigh[0][0],&procneigh[0][1]);
MPI_Cart_shift(cartesian,1,1,&procneigh[1][0],&procneigh[1][1]);
MPI_Cart_shift(cartesian,2,1,&procneigh[2][0],&procneigh[2][1]);
int coords[3];
int i,j,k;
for (i = 0; i < procgrid[0]; i++)
for (j = 0; j < procgrid[1]; j++)
for (k = 0; k < procgrid[2]; k++) {
coords[0] = i; coords[1] = j; coords[2] = k;
MPI_Cart_rank(cartesian,coords,&grid2proc[i][j][k]);
}
MPI_Comm_free(&cartesian);
}
/* ----------------------------------------------------------------------
map processors to 3d grid via MPI_Cart routines
respect sub-grid of cores within each node
MPI may do layout in machine-optimized fashion
------------------------------------------------------------------------- */
void ProcMap::cart_map(int reorder, int *procgrid, int ncores, int *coregrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
// setup NUMA params that numa_grid() sets up
int me;
MPI_Comm_rank(world,&me);
procs_per_node = ncores;
procs_per_numa = ncores;
node_id = me/ncores;
nodegrid[0] = procgrid[0] / coregrid[0];
nodegrid[1] = procgrid[1] / coregrid[1];
nodegrid[2] = procgrid[2] / coregrid[2];
// now can use numa_map() to perform mapping
numa_map(reorder,coregrid,myloc,procneigh,grid2proc);
}
/* ----------------------------------------------------------------------
map processors to 3d grid in XYZ order
called by onelevel
------------------------------------------------------------------------- */
void ProcMap::xyz_map(char *xyz, int *procgrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
int me;
MPI_Comm_rank(world,&me);
int i,j,k;
for (i = 0; i < procgrid[0]; i++)
for (j = 0; j < procgrid[1]; j++)
for (k = 0; k < procgrid[2]; k++) {
if (xyz[0] == 'x' && xyz[1] == 'y' && xyz[2] == 'z')
grid2proc[i][j][k] = k*procgrid[1]*procgrid[0] + j*procgrid[0] + i;
else if (xyz[0] == 'x' && xyz[1] == 'z' && xyz[2] == 'y')
grid2proc[i][j][k] = j*procgrid[2]*procgrid[0] + k*procgrid[0] + i;
else if (xyz[0] == 'y' && xyz[1] == 'x' && xyz[2] == 'z')
grid2proc[i][j][k] = k*procgrid[0]*procgrid[1] + i*procgrid[1] + j;
else if (xyz[0] == 'y' && xyz[1] == 'z' && xyz[2] == 'x')
grid2proc[i][j][k] = i*procgrid[2]*procgrid[1] + k*procgrid[1] + j;
else if (xyz[0] == 'z' && xyz[1] == 'x' && xyz[2] == 'y')
grid2proc[i][j][k] = j*procgrid[0]*procgrid[2] + i*procgrid[2] + k;
else if (xyz[0] == 'z' && xyz[1] == 'y' && xyz[2] == 'x')
grid2proc[i][j][k] = i*procgrid[1]*procgrid[2] + j*procgrid[2] + k;
if (grid2proc[i][j][k] == me) {
myloc[0] = i; myloc[1] = j, myloc[2] = k;
}
}
// proc IDs of neighbors
int minus,plus;
grid_shift(myloc[0],procgrid[0],minus,plus);
procneigh[0][0] = grid2proc[minus][myloc[1]][myloc[2]];
procneigh[0][1] = grid2proc[plus][myloc[1]][myloc[2]];
grid_shift(myloc[1],procgrid[1],minus,plus);
procneigh[1][0] = grid2proc[myloc[0]][minus][myloc[2]];
procneigh[1][1] = grid2proc[myloc[0]][plus][myloc[2]];
grid_shift(myloc[2],procgrid[2],minus,plus);
procneigh[2][0] = grid2proc[myloc[0]][myloc[1]][minus];
procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus];
}
/* ----------------------------------------------------------------------
map processors to 3d grid in XYZ order
respect sub-grid of cores within each node
called by twolevel
------------------------------------------------------------------------- */
void ProcMap::xyz_map(char *xyz, int *procgrid, int ncores, int *coregrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
int me;
MPI_Comm_rank(world,&me);
nodegrid[0] = procgrid[0] / coregrid[0];
nodegrid[1] = procgrid[1] / coregrid[1];
nodegrid[2] = procgrid[2] / coregrid[2];
int i,j,k,inode,jnode,knode,icore,jcore,kcore;
for (i = 0; i < procgrid[0]; i++)
for (j = 0; j < procgrid[1]; j++)
for (k = 0; k < procgrid[2]; k++) {
inode = i/coregrid[0];
jnode = j/coregrid[1];
knode = k/coregrid[2];
icore = i % coregrid[0];
jcore = j % coregrid[1];
kcore = k % coregrid[2];
if (xyz[0] == 'x' && xyz[1] == 'y' && xyz[2] == 'z') {
grid2proc[i][j][k] = ncores *
(knode*nodegrid[1]*nodegrid[0] + jnode*nodegrid[0] + inode) +
(kcore*coregrid[1]*coregrid[0] + jcore*coregrid[0] + icore);
} else if (xyz[0] == 'x' && xyz[1] == 'z' && xyz[2] == 'y')
grid2proc[i][j][k] = ncores *
(jnode*nodegrid[2]*nodegrid[0] + knode*nodegrid[0] + inode) +
(jcore*coregrid[2]*coregrid[0] + kcore*coregrid[0] + icore);
else if (xyz[0] == 'y' && xyz[1] == 'x' && xyz[2] == 'z')
grid2proc[i][j][k] = ncores *
(knode*nodegrid[0]*nodegrid[1] + inode*nodegrid[1] + jnode) +
(kcore*coregrid[0]*coregrid[1] + icore*coregrid[1] + jcore);
else if (xyz[0] == 'y' && xyz[1] == 'z' && xyz[2] == 'x')
grid2proc[i][j][k] = ncores *
(inode*nodegrid[2]*nodegrid[1] + knode*nodegrid[1] + jnode) +
(icore*coregrid[2]*coregrid[1] + kcore*coregrid[1] + jcore);
else if (xyz[0] == 'z' && xyz[1] == 'x' && xyz[2] == 'y')
grid2proc[i][j][k] = ncores *
(jnode*nodegrid[0]*nodegrid[2] + inode*nodegrid[2] + knode) +
(jcore*coregrid[0]*coregrid[2] + icore*coregrid[2] + kcore);
else if (xyz[0] == 'z' && xyz[1] == 'y' && xyz[2] == 'x')
grid2proc[i][j][k] = ncores *
(inode*nodegrid[1]*nodegrid[2] + jnode*nodegrid[2] + knode) +
(icore*coregrid[1]*coregrid[2] + jcore*coregrid[2] + kcore);
if (grid2proc[i][j][k] == me) {
myloc[0] = i; myloc[1] = j, myloc[2] = k;
}
}
// proc IDs of neighbors
int minus,plus;
grid_shift(myloc[0],procgrid[0],minus,plus);
procneigh[0][0] = grid2proc[minus][myloc[1]][myloc[2]];
procneigh[0][1] = grid2proc[plus][myloc[1]][myloc[2]];
grid_shift(myloc[1],procgrid[1],minus,plus);
procneigh[1][0] = grid2proc[myloc[0]][minus][myloc[2]];
procneigh[1][1] = grid2proc[myloc[0]][plus][myloc[2]];
grid_shift(myloc[2],procgrid[2],minus,plus);
procneigh[2][0] = grid2proc[myloc[0]][myloc[1]][minus];
procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus];
}
/* ----------------------------------------------------------------------
map processors to 3d grid in 2-level NUMA ordering
------------------------------------------------------------------------- */
void ProcMap::numa_map(int reorder, int *numagrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
// setup a per node communicator and find rank within
MPI_Comm node_comm;
MPI_Comm_split(world,node_id,0,&node_comm);
int node_rank;
MPI_Comm_rank(node_comm,&node_rank);
// setup a per numa communicator and find rank within
MPI_Comm numa_comm;
int local_numa = node_rank / procs_per_numa;
MPI_Comm_split(node_comm,local_numa,0,&numa_comm);
int numa_rank;
MPI_Comm_rank(numa_comm,&numa_rank);
// setup a communicator with the rank 0 procs from each numa node
MPI_Comm numa_leaders;
MPI_Comm_split(world,numa_rank,0,&numa_leaders);
// use the MPI Cartesian routines to map the nodes to the grid
int periods[3];
periods[0] = periods[1] = periods[2] = 1;
MPI_Comm cartesian;
if (numa_rank == 0) {
MPI_Cart_create(numa_leaders,3,nodegrid,periods,reorder,&cartesian);
MPI_Cart_get(cartesian,3,nodegrid,periods,myloc);
}
// broadcast numa node location in grid to other procs in numa node
MPI_Bcast(myloc,3,MPI_INT,0,numa_comm);
// compute my location within the node grid
int z_offset = numa_rank / (numagrid[0] * numagrid[1]);
int y_offset = (numa_rank % (numagrid[0] * numagrid[1]))/numagrid[0];
int x_offset = numa_rank % numagrid[0];
myloc[0] = myloc[0] * numagrid[0] + x_offset;
myloc[1] = myloc[1] * numagrid[1] + y_offset;
myloc[2] = myloc[2] * numagrid[2] + z_offset;
// allgather of myloc into gridi to fill grid2proc
int nprocs;
MPI_Comm_size(world,&nprocs);
int **gridi;
memory->create(gridi,nprocs,3,"comm:gridi");
MPI_Allgather(myloc,3,MPI_INT,gridi[0],3,MPI_INT,world);
for (int i = 0; i < nprocs; i++)
grid2proc[gridi[i][0]][gridi[i][1]][gridi[i][2]] = i;
memory->destroy(gridi);
// proc IDs of neighbors
int minus,plus;
grid_shift(myloc[0],nodegrid[0]*numagrid[0],minus,plus);
procneigh[0][0] = grid2proc[minus][myloc[1]][myloc[2]];
procneigh[0][1] = grid2proc[plus][myloc[1]][myloc[2]];
grid_shift(myloc[1],nodegrid[1]*numagrid[1],minus,plus);
procneigh[1][0] = grid2proc[myloc[0]][minus][myloc[2]];
procneigh[1][1] = grid2proc[myloc[0]][plus][myloc[2]];
grid_shift(myloc[2],nodegrid[2]*numagrid[2],minus,plus);
procneigh[2][0] = grid2proc[myloc[0]][myloc[1]][minus];
procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus];
// clean-up
if (numa_rank == 0) MPI_Comm_free(&cartesian);
MPI_Comm_free(&numa_leaders);
MPI_Comm_free(&numa_comm);
MPI_Comm_free(&node_comm);
}
/* ----------------------------------------------------------------------
map processors to 3d grid in custom ordering
------------------------------------------------------------------------- */
void ProcMap::custom_map(int *procgrid,
int *myloc, int procneigh[3][2], int ***grid2proc)
{
int me,nprocs;
MPI_Comm_rank(world,&me);
MPI_Comm_size(world,&nprocs);
for (int i = 0; i < nprocs; i++) {
grid2proc[cmap[i][1]-1][cmap[i][2]-1][cmap[i][3]-1] = cmap[i][0];
if (cmap[i][0] == me) {
myloc[0] = cmap[i][1] - 1;
myloc[1] = cmap[i][2] - 1;
myloc[2] = cmap[i][3] - 1;
}
}
// proc IDs of neighbors
int minus,plus;
grid_shift(myloc[0],procgrid[0],minus,plus);
procneigh[0][0] = grid2proc[minus][myloc[1]][myloc[2]];
procneigh[0][1] = grid2proc[plus][myloc[1]][myloc[2]];
grid_shift(myloc[1],procgrid[1],minus,plus);
procneigh[1][0] = grid2proc[myloc[0]][minus][myloc[2]];
procneigh[1][1] = grid2proc[myloc[0]][plus][myloc[2]];
grid_shift(myloc[2],procgrid[2],minus,plus);
procneigh[2][0] = grid2proc[myloc[0]][myloc[1]][minus];
procneigh[2][1] = grid2proc[myloc[0]][myloc[1]][plus];
memory->destroy(cmap);
}
/* ----------------------------------------------------------------------
output mapping of processors to 3d grid to file
------------------------------------------------------------------------- */
void ProcMap::output(char *file, int *procgrid, int ***grid2proc)
{
int me,nprocs;
MPI_Comm_rank(world,&me);
MPI_Comm_size(world,&nprocs);
FILE *fp;
if (me == 0) {
fp = fopen(file,"w");
if (fp == NULL) error->one(FLERR,"Cannot open processors output file");
fprintf(fp,"LAMMPS mapping of processors to 3d grid\n");
fprintf(fp,"partition = %d\n",universe->iworld+1);
fprintf(fp,"Px Py Pz = %d %d %d\n",procgrid[0],procgrid[1],procgrid[2]);
fprintf(fp,"world-ID universe-ID original-ID: I J K: name\n\n");
}
// find me in the grid
int ime,jme,kme;
for (int i = 0; i < procgrid[0]; i++)
for (int j = 0; j < procgrid[1]; j++)
for (int k = 0; k < procgrid[2]; k++)
if (grid2proc[i][j][k] == me) {
ime = i; jme = j; kme = k;
}
// polled comm of grid mapping info from each proc to proc 0
int tmp;
int vec[6];
char procname[MPI_MAX_PROCESSOR_NAME+1];
vec[0] = me;
vec[1] = universe->me;
MPI_Comm_rank(universe->uorig,&vec[2]);
vec[3] = ime + 1;
vec[4] = jme + 1;
vec[5] = kme + 1;
int len;
MPI_Get_processor_name(procname,&len);
procname[len] = '\0';
if (me == 0) {
for (int iproc = 0; iproc < nprocs; iproc++) {
if (iproc) {
MPI_Send(&tmp,0,MPI_INT,iproc,0,world);
MPI_Recv(vec,6,MPI_INT,iproc,0,world,MPI_STATUS_IGNORE);
MPI_Recv(procname,MPI_MAX_PROCESSOR_NAME+1,MPI_CHAR,
iproc,0,world,MPI_STATUS_IGNORE);
}
fprintf(fp,"%d %d %d: %d %d %d: %s\n",
vec[0],vec[1],vec[2],vec[3],vec[4],vec[5],procname);
}
} else {
MPI_Recv(&tmp,0,MPI_INT,0,0,world,MPI_STATUS_IGNORE);
MPI_Send(vec,6,MPI_INT,0,0,world);
MPI_Send(procname,strlen(procname)+1,MPI_CHAR,0,0,world);
}
// close output file
if (me == 0) fclose(fp);
}
/* ----------------------------------------------------------------------
generate all possible 3-integer factorizations of N
store them in factors if non-NULL
return # of factorizations
------------------------------------------------------------------------- */
int ProcMap::factor(int n, int **factors)
{
int i,j,nyz;
int m = 0;
for (i = 1; i <= n; i++) {
if (n % i) continue;
nyz = n/i;
for (j = 1; j <= nyz; j++) {
if (nyz % j) continue;
if (factors) {
factors[m][0] = i;
factors[m][1] = j;
factors[m][2] = nyz/j;
}
m++;
}
}
return m;
}
/* ----------------------------------------------------------------------
create N1*N2 new factors (procs) from factors1 (nodes) and factors2 (cores)
store index of corresponding core factors in factors[][3]
------------------------------------------------------------------------- */
int ProcMap::combine_factors(int n1, int **factors1, int n2, int **factors2,
int **factors)
{
int m = 0;
for (int i = 0; i < n1; i++)
for (int j = 0; j < n2; j++) {
factors[m][0] = factors1[i][0]*factors2[j][0];
factors[m][1] = factors1[i][1]*factors2[j][1];
factors[m][2] = factors1[i][2]*factors2[j][2];
factors[m][3] = j;
m++;
}
return n1*n2;
}
/* ----------------------------------------------------------------------
remove any factors where Pz != 1 for 2d
------------------------------------------------------------------------- */
int ProcMap::cull_2d(int n, int **factors, int m)
{
int i = 0;
while (i < n) {
if (factors[i][2] != 1) {
for (int j = 0; j < m; j++) factors[i][j] = factors[n-1][j];
n--;
} else i++;
}
return n;
}
/* ----------------------------------------------------------------------
remove any factors that do not match non-zero user_factors Px,Py,Pz
------------------------------------------------------------------------- */
int ProcMap::cull_user(int n, int **factors, int m, int *user_factors)
{
int i = 0;
while (i < n) {
int flag = 0;
if (user_factors[0] && factors[i][0] != user_factors[0]) flag = 1;
if (user_factors[1] && factors[i][1] != user_factors[1]) flag = 1;
if (user_factors[2] && factors[i][2] != user_factors[2]) flag = 1;
if (flag) {
for (int j = 0; j < m; j++) factors[i][j] = factors[n-1][j];
n--;
} else i++;
}
return n;
}
/* ----------------------------------------------------------------------
remove any factors that do not match settings from other partition
MULTIPLE = other Nx,Ny,Nz must be multiple of my Px,Py,Pz
where Nx,Ny,Nz = node grid = procgrid/coregrid
------------------------------------------------------------------------- */
int ProcMap::cull_other(int n, int **factors, int m,
int other_style, int *other_procgrid,
int *other_coregrid)
{
int i = 0;
while (i < n) {
if (other_style == Comm::MULTIPLE) {
int flag = 0;
if ((other_procgrid[0]/other_coregrid[0]) % factors[i][0]) flag = 1;
if ((other_procgrid[1]/other_coregrid[1]) % factors[i][1]) flag = 1;
if ((other_procgrid[2]/other_coregrid[2]) % factors[i][2]) flag = 1;
if (flag) {
for (int j = 0; j < m; j++) factors[i][j] = factors[n-1][j];
n--;
} else i++;
}
}
return n;
}
/* ----------------------------------------------------------------------
choose best factors from list of Npossible factors
best = minimal surface area of sub-domain
return best = 3 factors
return index of best factors in factors
------------------------------------------------------------------------- */
int ProcMap::best_factors(int npossible, int **factors, int *best,
const int sx, const int sy, const int sz)
{
// determine cross-sectional areas for orthogonal and triclinic boxes
// for triclinic, area = cross product of 2 edge vectors stored in h matrix
// area[3] = surface area 3 box faces divided by sx,sy,sz
// area[0] = xy, area[1] = xz, area[2] = yz
double area[3];
if (domain->triclinic == 0) {
area[0] = domain->xprd * domain->yprd / (sx*sy);
area[1] = domain->xprd * domain->zprd / (sx*sz);
area[2] = domain->yprd * domain->zprd / (sy*sz);
} else {
double *h = domain->h;
double a[3],b[3],c[3];
a[0] = h[0]; a[1] = 0.0; a[2] = 0.0;
b[0] = h[5]; b[1] = h[1]; b[2] = 0.0;
MathExtra::cross3(a,b,c);
area[0] = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]) / (sx*sy);
a[0] = h[0]; a[1] = 0.0; a[2] = 0.0;
b[0] = h[4]; b[1] = h[3]; b[2] = h[2];
MathExtra::cross3(a,b,c);
area[1] = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]) / (sx*sz);
a[0] = h[5]; a[1] = h[1]; a[2] = 0.0;
b[0] = h[4]; b[1] = h[3]; b[2] = h[2];
MathExtra::cross3(a,b,c);
area[2] = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]) / (sy*sz);
}
int index;
double surf;
double bestsurf = 2.0 * (area[0]+area[1]+area[2]);
for (int m = 0; m < npossible; m++) {
surf = area[0]/factors[m][0]/factors[m][1] +
area[1]/factors[m][0]/factors[m][2] +
area[2]/factors[m][1]/factors[m][2];
if (surf < bestsurf) {
bestsurf = surf;
best[0] = factors[m][0];
best[1] = factors[m][1];
best[2] = factors[m][2];
index = m;
}
}
return index;
}
/* ----------------------------------------------------------------------
minus,plus = indices of neighboring processors in a dimension
------------------------------------------------------------------------- */
void ProcMap::grid_shift(int myloc, int nprocs, int &minus, int &plus)
{
minus = myloc - 1;
if (minus < 0) minus = nprocs - 1;
plus = myloc + 1;
if (plus == nprocs) plus = 0;
}
Reference in New Issue View Git Blame Copy Permalink