int DML_clear_to_send(char *scratch_buf, size_t size,
int my_io_node, int new_node)
{
if (QMP_get_msg_passing_type() != QMP_GRID)
{
int this_node = QMP_get_node_number();
if(this_node == my_io_node && new_node != my_io_node)
DML_send_bytes(scratch_buf,size,new_node); /* junk message */
if(this_node == new_node && new_node != my_io_node)
DML_get_bytes(scratch_buf,size,my_io_node);
}
return 0;
}
int DML_route_bytes(char *buf, size_t size, int fromnode, int tonode)
{
if (QMP_get_msg_passing_type() == QMP_GRID)
{
DML_grid_route(buf, size, fromnode, tonode);
}
else
{
int this_node = QMP_get_node_number();
if (this_node == tonode)
DML_get_bytes(buf,size,fromnode);
if (this_node == fromnode)
DML_send_bytes(buf,size,tonode);
}
return 0;
}
static void
eo_setup(QDP_Lattice *lat, void *args)
{
mLattice *S = args;
QDP_allocate_lattice_params(lat, sizeof (params));
params *p = QDP_get_lattice_params(lat);
p->S = S;
if (QMP_get_msg_passing_type() != QMP_SWITCH) {
int nd2 = QMP_get_allocated_number_of_dimensions();
const int *nsquares2 = QMP_get_allocated_dimensions();
int i;
for (i = 0; i < S->rank; i++) {
S->net[i] = (i < nd2) ? nsquares2[i] : 1;
}
} else { /* not QMP_GRID */
int squaresize[QLUA_MAX_LATTICE_RANK];
int extrafactors[QLUA_MAX_LATTICE_RANK];
int i;
for (i = 0; i < S->rank; i++) {
squaresize[i] = S->dim[i];
extrafactors[i] = 1;
S->net[i] = 1;
}
/* Figure out dimensions of rectangle */
int n = QMP_get_number_of_nodes(); /* nodes to factor */
int k = MAXPRIMES-1;
while (n > 1) {
/* figure out which prime to divide by starting with largest */
/* if no factor found, assume n is prime */
while ((k >= 0) && (n % prime[k] != 0)) --k;
int pfac = (k>=0) ? prime[k] : n;
/* figure out which direction to divide */
/* find largest divisible dimension of h-cubes */
/* if one direction with largest dimension has already been
divided, divide it again. Otherwise divide first direction
with largest dimension. */
int j = -1;
int i;
for (i = 0; i < S->rank; i++) {
if (squaresize[i] % pfac == 0) {
if ((j<0) ||
(extrafactors[j] * squaresize[i] >
extrafactors[i] * squaresize[j])) {
j = i;
} else if (extrafactors[j] * squaresize[i] ==
extrafactors[i] * squaresize[j]) {
if ((S->net[j] == 1) || (S->net[i] != 1))
j = i;
}
}
}
/* This can fail if we run out of prime factors in the dimensions */
/* then just choose largest dimension */
if (j < 0) {
int i;
for (i = 0; i < S->rank; i++) {
if ((j<0) ||
(extrafactors[j] * squaresize[i] >
extrafactors[i] * squaresize[j]) ) {
j = i;
} else if (extrafactors[j] * squaresize[i] ==
extrafactors[i] * squaresize[j]) {
if((S->net[j] == 1) || (S->net[i] != 1))
j = i;
}
}
n /= pfac;
extrafactors[j] *= pfac;
S->net[j] *= pfac;
} else {
n /= pfac;
squaresize[j] /= pfac;
S->net[j] *= pfac;
}
}
} /* not QMP_GRID */
int mc[QLUA_MAX_LATTICE_RANK];
int i;
S->node = QDP_this_node;
node2coord(mc, QDP_this_node, S);
for (i = 0; i < S->rank; i++) {
int x = mc[i];
mc[i] = x + 1;
if (mc[i] == S->net[i])
mc[i] = 0;
S->neighbor_up[i] = coord2node(mc, S);
mc[i] = x - 1;
if (mc[i] < 0)
mc[i] = S->net[i] - 1;
S->neighbor_down[i] = coord2node(mc, S);
mc[i] = x;
}
}
void init_qmp(int * argc, char ***argv) {
#if 0
printf("init_qmp(%d %p)\n",*argc,*argv);
for(int i = 0; i<*argc;i++){
printf("argv[%d](before)=%s\n",i,(*argv)[i]);
}
#endif
#if 0
spi_init();
#endif
QMP_thread_level_t prv;
#ifndef UNIFORM_SEED_NO_COMMS
QMP_status_t init_status = QMP_init_msg_passing(argc, argv, QMP_THREAD_SINGLE, &prv);
if (init_status) printf("QMP_init_msg_passing returned %d\n",init_status);
peRank = QMP_get_node_number();
peNum = QMP_get_number_of_nodes();
if(!peRank)printf("QMP_init_msg_passing returned %d\n",init_status);
if (init_status != QMP_SUCCESS) {
QMP_error("%s\n",QMP_error_string(init_status));
}
// check QMP thread level
// Added by Hantao
if(peRank == 0) {
switch(prv) {
case QMP_THREAD_SINGLE:
printf("QMP thread level = QMP_THREAD_SINGLE\n");
break;
case QMP_THREAD_FUNNELED:
printf("QMP thread level = QMP_THREAD_FUNNELED\n");
break;
case QMP_THREAD_SERIALIZED:
printf("QMP thread level = QMP_THREAD_SERIALIZED\n");
break;
case QMP_THREAD_MULTIPLE:
printf("QMP thread level = QMP_THREAD_MULTIPLE\n");
break;
default:
printf("QMP thread level = no idea what this is, boom!\n");
}
}
//Check to make sure that this machine is a GRID machine
//Exit if not GRID machine
QMP_ictype qmp_type = QMP_get_msg_passing_type();
//Get information about the allocated machine
peNum = QMP_get_number_of_nodes();
NDIM = QMP_get_allocated_number_of_dimensions();
peGrid = QMP_get_allocated_dimensions();
pePos = QMP_get_allocated_coordinates();
if(peRank==0){
for(int i = 0; i<*argc;i++){
printf("argv[%d])(after)=%s\n",i,(*argv)[i]);
}
}
#else
QMP_status_t init_status = QMP_SUCCESS;
peRank=0;
peNum=1;
NDIM=4;
#endif
//#if (TARGET == BGL) || (TARGET == BGP)
if (NDIM>5){
peNum = 1;
for(int i = 0;i<5;i++)
peNum *= peGrid[i];
peRank = peRank % peNum;
}
int if_print=1;
for(int i = 0;i<NDIM;i++)
if (pePos[i]>=2) if_print=0;
if (if_print){
printf("Rank=%d Num=%d NDIM=%d\n",peRank,peNum,NDIM);
printf("dim:");
for(int i = 0;i<NDIM;i++)
printf(" %d",peGrid[i]);
printf("\n");
printf("pos:");
for(int i = 0;i<NDIM;i++)
printf(" %d",pePos[i]);
printf("\n");
#if 0
int rc;
BGLPersonality pers;
rts_get_personality(&pers, sizeof(pers));
printf("from personality: %d %d %d %d\n",pers.xCoord,pers.yCoord,pers.zCoord,rts_get_processor_id());
#endif
}
// printf("from personality:\n");
#if 0
if ( (qmp_type!= QMP_GRID) && (qmp_type !=QMP_MESH) ) {
QMP_error("CPS on QMP only implemented for GRID or MESH, not (%d) machines\n",qmp_type);
}
#endif
// printf("QMP_declare_logical_topology(peGrid, NDIM)\n");
#ifndef UNIFORM_SEED_NO_COMMS
//Declare the logical topology (Redundant for GRID machines)
if (QMP_declare_logical_topology(peGrid, NDIM) != QMP_SUCCESS) {
QMP_error("Node %d: Failed to declare logical topology\n",peRank);
exit(-4);
}
#endif
initialized = true;
printf("Rank=%d init_qmp() done\n",peRank);
}
void setup_layout(){
int k = mynode();
#ifdef FIX_NODE_GEOM
int *geom = node_geometry;
#else
int *geom = NULL;
#endif
if(k == 0)
printf("LAYOUT = Hypercubes, options = ");
#ifdef HAVE_QMP
/* QMP treatment */
/* Is there already a grid?
This could be a grid architecture with a preset dimension, or
a geometry could have been set by the -qmp-geom command line arg.
In either case we have a nonzero allocated number of dimensions.
*/
if(QMP_get_allocated_number_of_dimensions() == 0)
/* Set the geometry if requested */
set_qmp_layout_grid(geom, 4);
/* Has a grid been set up now? */
if(QMP_get_msg_passing_type() == QMP_GRID)
setup_qmp_grid();
else if(geom != NULL)
setup_fixed_geom(geom, 4);
else
setup_hyper_prime();
#else
/* Non QMP treatment */
if(geom != NULL)
setup_fixed_geom(geom, 4);
else
setup_hyper_prime();
#endif
#ifdef FIX_IONODE_GEOM
/* Initialize I/O node function */
init_io_node();
#endif
/* Compute machine coordinates for this node */
lex_coords(machine_coordinates, 4, nsquares, k);
/* Number of sites on node */
sites_on_node =
squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*squaresize[TUP];
/* Need number of sites per hypercube divisible by 32 */
if( mynode()==0)if( sites_on_node%32 != 0){
printf("SORRY, CAN'T LAY OUT THIS LATTICE\n");
terminate(0);
}
subl_sites_on_node = sites_on_node/32;
if( mynode()==0)
printf("ON EACH NODE %d x %d x %d x %d\n",squaresize[XUP],squaresize[YUP],
squaresize[ZUP],squaresize[TUP]);
/* Actually we have already required sites_on_node to be a multiple of 32 */
if( mynode()==0 && sites_on_node%2 != 0)
printf("WATCH OUT FOR EVEN/ODD SITES ON NODE BUG!!!\n");
even_sites_on_node = odd_sites_on_node = sites_on_node/2;
}
void setup_layout(){
register int i,j,k,dir;
#ifdef HAVE_QMP
if(QMP_get_msg_passing_type()==QMP_GRID){
printf("This layout should not be used on a grid architecture\n");
terminate(1);
}
#endif
if(mynode()==0){
printf("LAYOUT = Hypercubes, options = ");
printf("tstretch,");
printf("\n");
}
/* Figure out dimensions of rectangle */
if( (nx*ny*nz*nt)%numnodes() == 0){
squaresize[XUP] = nx; squaresize[YUP] = ny;
squaresize[ZUP] = nz; squaresize[TUP] = nt;
}
else if ( (nx*ny*nz*(nt+1))%numnodes() == 0){
squaresize[XUP] = nx; squaresize[YUP] = ny;
squaresize[ZUP] = nz; squaresize[TUP] = nt+1;
}
nsquares[XUP] = nsquares[YUP] = nsquares[ZUP] = nsquares[TUP] = 1;
i = 1; /* current number of hypercubes */
while(i<numnodes()){
/* figure out which prime to divide by starting with largest */
k = MAXPRIMES-1;
while( (numnodes()/i)%prime[k] != 0 && k>0 ) --k;
/* figure out which direction to divide */
/* find largest even dimension of h-cubes */
for(j=1,dir=XUP;dir<=TUP;dir++)
if( squaresize[dir]>j && squaresize[dir]%prime[k]==0 )
j=squaresize[dir];
/* if one direction with largest dimension has already been
divided, divide it again. Otherwise divide first direction
with largest dimension. */
for(dir=XUP;dir<=TUP;dir++)
if( squaresize[dir]==j && nsquares[dir]>1 )break;
if( dir > TUP)for(dir=XUP;dir<=TUP;dir++)
if( squaresize[dir]==j )break;
/* This can fail if I run out of prime factors in the dimensions */
if(dir > TUP){
if(mynode()==0)printf(
"LAYOUT: Can't lay out this lattice, not enough factors of %d\n"
,prime[k]);
terminate(1);
}
/* do the surgery */
i*=prime[k]; squaresize[dir] /= prime[k]; nsquares[dir] *= prime[k];
}
sites_on_node =
squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*squaresize[TUP];
sites_on_lastnodes = sites_on_node;
if( mynode()==0)
printf("ON MOST NODES %d x %d x %d x %d\n",squaresize[XUP],squaresize[YUP],
squaresize[ZUP],squaresize[TUP]);
if( !((nx*ny*nz*nt)%numnodes() == 0) && (nx*ny*nz*(nt+1))%numnodes() == 0 ){
/* stretched t direction by one */
if( mynode()==0)printf("SOME NODES HAVE FEWER SITES\n");
if( mynode() >= nsquares[XUP]*nsquares[YUP]*nsquares[ZUP]*(nsquares[TUP]-1) ){
sites_on_node = squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*(squaresize[TUP]-1);
sites_on_lastnodes = sites_on_node;
}
}
/* Need even number of sites per hypercube */
if( mynode()==0)if( sites_on_node%2 != 0){
printf("SORRY, CAN'T LAY OUT THIS LATTICE\n");
terminate(0);
}
if( mynode()==0 && sites_on_node%2 != 0)
printf("WATCH OUT FOR EVEN/ODD SITES ON NODE BUG!!!\n");
even_sites_on_node = odd_sites_on_node = sites_on_node/2;
}
