void
setup_layout(void)
{
int c[4];
int i,n_mach;
int d[4];
#ifdef FIX_NODE_GEOM
int *geom = node_geometry;
#else
int *geom = NULL;
#endif
if(mynode()==0){
printf("LAYOUT = Hypercubes, options = ");
printf("QDP");
printf("\n");
}
/* Is there already a grid?
This could be a grid architecture with a preset dimension, or
a geometry could have been set by the -qmp-geom command line arg.
In either case we have a nonzero allocated number of dimensions.
*/
if(QMP_get_allocated_number_of_dimensions() == 0)
/* Set the geometry if requested */
set_qmp_layout_grid(geom, 4);
c[0] = nx;
c[1] = ny;
c[2] = nz;
c[3] = nt;
QDP_set_latsize(4, c);
QDP_create_layout();
sites_on_node = QDP_sites_on_node;
even_sites_on_node = QDP_subset_len(QDP_even);
odd_sites_on_node = QDP_subset_len(QDP_odd);
n_mach = QMP_get_logical_number_of_dimensions();
dim_mach = QMP_get_logical_dimensions();
/* Initialize I/O node function */
#ifdef FIX_IONODE_GEOM
init_io_node();
#endif
/* Report sublattice dimensions */
for(i = 0; i < 4; i++){
/* Any extra machine dimensions are assumed to be 1 */
if(i < n_mach)d[i] = c[i]/dim_mach[i];
else d[i] = c[i];
}
if( mynode()==0)
printf("ON EACH NODE %d x %d x %d x %d\n",d[0],d[1],d[2],d[3]);
#if 0
mpi_whoami(); /* Debug */
#endif
}
/*--------------------------------------------------------------------*/
static void setup_qmp_grid(){
int ndim = 4;
int len[4];
int ndim2, i;
const int *nsquares2;
len[0] = nx; len[1] = ny; len[2] = nz; len[3] = nt;
if(mynode()==0){
printf("qmp_grid,");
printf("\n");
}
ndim2 = QMP_get_allocated_number_of_dimensions();
nsquares2 = QMP_get_allocated_dimensions();
/* If the dimensions are not already allocated, use the
node_geometry request. Otherwise a hardware or command line
specification trumps the parameter input. */
#ifdef FIX_NODE_GEOM
if(ndim2 == 0){
ndim2 = 4;
nsquares2 = node_geometry;
}
else{
node0_printf("setup_qmp_grid: Preallocated machine geometry overrides request\n");
}
#endif
if(mynode()==0){
printf("Using machine geometry: ");
for(i=0; i<ndim; i++){
printf("%d ",nsquares2[i]);
if(i < ndim-1)printf("X ");
}
printf("\n");
}
/* In principle, we could now rotate coordinate axes */
/* Save this for a future upgrade */
set_qmp_layout_grid(nsquares2, ndim2);
ndim2 = QMP_get_logical_number_of_dimensions();
nsquares2 = QMP_get_logical_dimensions();
for(i=0; i<ndim; i++) {
if(i<ndim2) nsquares[i] = nsquares2[i];
else nsquares[i] = 1;
}
for(i=0; i<ndim; i++) {
if(len[i]%nsquares[i] != 0) {
node0_printf("LATTICE SIZE DOESN'T FIT GRID\n");
QMP_abort(0);
}
squaresize[i] = len[i]/nsquares[i];
}
}
static void
eo_setup(QDP_Lattice *lat, void *args)
{
mLattice *S = args;
QDP_allocate_lattice_params(lat, sizeof (params));
params *p = QDP_get_lattice_params(lat);
p->S = S;
if (QMP_get_msg_passing_type() != QMP_SWITCH) {
int nd2 = QMP_get_allocated_number_of_dimensions();
const int *nsquares2 = QMP_get_allocated_dimensions();
int i;
for (i = 0; i < S->rank; i++) {
S->net[i] = (i < nd2) ? nsquares2[i] : 1;
}
} else { /* not QMP_GRID */
int squaresize[QLUA_MAX_LATTICE_RANK];
int extrafactors[QLUA_MAX_LATTICE_RANK];
int i;
for (i = 0; i < S->rank; i++) {
squaresize[i] = S->dim[i];
extrafactors[i] = 1;
S->net[i] = 1;
}
/* Figure out dimensions of rectangle */
int n = QMP_get_number_of_nodes(); /* nodes to factor */
int k = MAXPRIMES-1;
while (n > 1) {
/* figure out which prime to divide by starting with largest */
/* if no factor found, assume n is prime */
while ((k >= 0) && (n % prime[k] != 0)) --k;
int pfac = (k>=0) ? prime[k] : n;
/* figure out which direction to divide */
/* find largest divisible dimension of h-cubes */
/* if one direction with largest dimension has already been
divided, divide it again. Otherwise divide first direction
with largest dimension. */
int j = -1;
int i;
for (i = 0; i < S->rank; i++) {
if (squaresize[i] % pfac == 0) {
if ((j<0) ||
(extrafactors[j] * squaresize[i] >
extrafactors[i] * squaresize[j])) {
j = i;
} else if (extrafactors[j] * squaresize[i] ==
extrafactors[i] * squaresize[j]) {
if ((S->net[j] == 1) || (S->net[i] != 1))
j = i;
}
}
}
/* This can fail if we run out of prime factors in the dimensions */
/* then just choose largest dimension */
if (j < 0) {
int i;
for (i = 0; i < S->rank; i++) {
if ((j<0) ||
(extrafactors[j] * squaresize[i] >
extrafactors[i] * squaresize[j]) ) {
j = i;
} else if (extrafactors[j] * squaresize[i] ==
extrafactors[i] * squaresize[j]) {
if((S->net[j] == 1) || (S->net[i] != 1))
j = i;
}
}
n /= pfac;
extrafactors[j] *= pfac;
S->net[j] *= pfac;
} else {
n /= pfac;
squaresize[j] /= pfac;
S->net[j] *= pfac;
}
}
} /* not QMP_GRID */
int mc[QLUA_MAX_LATTICE_RANK];
int i;
S->node = QDP_this_node;
node2coord(mc, QDP_this_node, S);
for (i = 0; i < S->rank; i++) {
int x = mc[i];
mc[i] = x + 1;
if (mc[i] == S->net[i])
mc[i] = 0;
S->neighbor_up[i] = coord2node(mc, S);
mc[i] = x - 1;
if (mc[i] < 0)
mc[i] = S->net[i] - 1;
S->neighbor_down[i] = coord2node(mc, S);
mc[i] = x;
}
}
void init_qmp(int * argc, char ***argv) {
#if 0
printf("init_qmp(%d %p)\n",*argc,*argv);
for(int i = 0; i<*argc;i++){
printf("argv[%d](before)=%s\n",i,(*argv)[i]);
}
#endif
#if 0
spi_init();
#endif
QMP_thread_level_t prv;
#ifndef UNIFORM_SEED_NO_COMMS
QMP_status_t init_status = QMP_init_msg_passing(argc, argv, QMP_THREAD_SINGLE, &prv);
if (init_status) printf("QMP_init_msg_passing returned %d\n",init_status);
peRank = QMP_get_node_number();
peNum = QMP_get_number_of_nodes();
if(!peRank)printf("QMP_init_msg_passing returned %d\n",init_status);
if (init_status != QMP_SUCCESS) {
QMP_error("%s\n",QMP_error_string(init_status));
}
// check QMP thread level
// Added by Hantao
if(peRank == 0) {
switch(prv) {
case QMP_THREAD_SINGLE:
printf("QMP thread level = QMP_THREAD_SINGLE\n");
break;
case QMP_THREAD_FUNNELED:
printf("QMP thread level = QMP_THREAD_FUNNELED\n");
break;
case QMP_THREAD_SERIALIZED:
printf("QMP thread level = QMP_THREAD_SERIALIZED\n");
break;
case QMP_THREAD_MULTIPLE:
printf("QMP thread level = QMP_THREAD_MULTIPLE\n");
break;
default:
printf("QMP thread level = no idea what this is, boom!\n");
}
}
//Check to make sure that this machine is a GRID machine
//Exit if not GRID machine
QMP_ictype qmp_type = QMP_get_msg_passing_type();
//Get information about the allocated machine
peNum = QMP_get_number_of_nodes();
NDIM = QMP_get_allocated_number_of_dimensions();
peGrid = QMP_get_allocated_dimensions();
pePos = QMP_get_allocated_coordinates();
if(peRank==0){
for(int i = 0; i<*argc;i++){
printf("argv[%d])(after)=%s\n",i,(*argv)[i]);
}
}
#else
QMP_status_t init_status = QMP_SUCCESS;
peRank=0;
peNum=1;
NDIM=4;
#endif
//#if (TARGET == BGL) || (TARGET == BGP)
if (NDIM>5){
peNum = 1;
for(int i = 0;i<5;i++)
peNum *= peGrid[i];
peRank = peRank % peNum;
}
int if_print=1;
for(int i = 0;i<NDIM;i++)
if (pePos[i]>=2) if_print=0;
if (if_print){
printf("Rank=%d Num=%d NDIM=%d\n",peRank,peNum,NDIM);
printf("dim:");
for(int i = 0;i<NDIM;i++)
printf(" %d",peGrid[i]);
printf("\n");
printf("pos:");
for(int i = 0;i<NDIM;i++)
printf(" %d",pePos[i]);
printf("\n");
#if 0
int rc;
BGLPersonality pers;
rts_get_personality(&pers, sizeof(pers));
printf("from personality: %d %d %d %d\n",pers.xCoord,pers.yCoord,pers.zCoord,rts_get_processor_id());
#endif
}
// printf("from personality:\n");
#if 0
if ( (qmp_type!= QMP_GRID) && (qmp_type !=QMP_MESH) ) {
QMP_error("CPS on QMP only implemented for GRID or MESH, not (%d) machines\n",qmp_type);
}
#endif
// printf("QMP_declare_logical_topology(peGrid, NDIM)\n");
#ifndef UNIFORM_SEED_NO_COMMS
//Declare the logical topology (Redundant for GRID machines)
if (QMP_declare_logical_topology(peGrid, NDIM) != QMP_SUCCESS) {
QMP_error("Node %d: Failed to declare logical topology\n",peRank);
exit(-4);
}
#endif
initialized = true;
printf("Rank=%d init_qmp() done\n",peRank);
}
void setup_layout(){
int k = mynode();
#ifdef FIX_NODE_GEOM
int *geom = node_geometry;
#else
int *geom = NULL;
#endif
if(k == 0)
printf("LAYOUT = Hypercubes, options = ");
#ifdef HAVE_QMP
/* QMP treatment */
/* Is there already a grid?
This could be a grid architecture with a preset dimension, or
a geometry could have been set by the -qmp-geom command line arg.
In either case we have a nonzero allocated number of dimensions.
*/
if(QMP_get_allocated_number_of_dimensions() == 0)
/* Set the geometry if requested */
set_qmp_layout_grid(geom, 4);
/* Has a grid been set up now? */
if(QMP_get_msg_passing_type() == QMP_GRID)
setup_qmp_grid();
else if(geom != NULL)
setup_fixed_geom(geom, 4);
else
setup_hyper_prime();
#else
/* Non QMP treatment */
if(geom != NULL)
setup_fixed_geom(geom, 4);
else
setup_hyper_prime();
#endif
#ifdef FIX_IONODE_GEOM
/* Initialize I/O node function */
init_io_node();
#endif
/* Compute machine coordinates for this node */
lex_coords(machine_coordinates, 4, nsquares, k);
/* Number of sites on node */
sites_on_node =
squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*squaresize[TUP];
/* Need number of sites per hypercube divisible by 32 */
if( mynode()==0)if( sites_on_node%32 != 0){
printf("SORRY, CAN'T LAY OUT THIS LATTICE\n");
terminate(0);
}
subl_sites_on_node = sites_on_node/32;
if( mynode()==0)
printf("ON EACH NODE %d x %d x %d x %d\n",squaresize[XUP],squaresize[YUP],
squaresize[ZUP],squaresize[TUP]);
/* Actually we have already required sites_on_node to be a multiple of 32 */
if( mynode()==0 && sites_on_node%2 != 0)
printf("WATCH OUT FOR EVEN/ODD SITES ON NODE BUG!!!\n");
even_sites_on_node = odd_sites_on_node = sites_on_node/2;
}
