int DML_clear_to_send(char *scratch_buf, size_t size, int my_io_node, int new_node) { if (QMP_get_msg_passing_type() != QMP_GRID) { int this_node = QMP_get_node_number(); if(this_node == my_io_node && new_node != my_io_node) DML_send_bytes(scratch_buf,size,new_node); /* junk message */ if(this_node == new_node && new_node != my_io_node) DML_get_bytes(scratch_buf,size,my_io_node); } return 0; }
int DML_route_bytes(char *buf, size_t size, int fromnode, int tonode) { if (QMP_get_msg_passing_type() == QMP_GRID) { DML_grid_route(buf, size, fromnode, tonode); } else { int this_node = QMP_get_node_number(); if (this_node == tonode) DML_get_bytes(buf,size,fromnode); if (this_node == fromnode) DML_send_bytes(buf,size,tonode); } return 0; }
static void eo_setup(QDP_Lattice *lat, void *args) { mLattice *S = args; QDP_allocate_lattice_params(lat, sizeof (params)); params *p = QDP_get_lattice_params(lat); p->S = S; if (QMP_get_msg_passing_type() != QMP_SWITCH) { int nd2 = QMP_get_allocated_number_of_dimensions(); const int *nsquares2 = QMP_get_allocated_dimensions(); int i; for (i = 0; i < S->rank; i++) { S->net[i] = (i < nd2) ? nsquares2[i] : 1; } } else { /* not QMP_GRID */ int squaresize[QLUA_MAX_LATTICE_RANK]; int extrafactors[QLUA_MAX_LATTICE_RANK]; int i; for (i = 0; i < S->rank; i++) { squaresize[i] = S->dim[i]; extrafactors[i] = 1; S->net[i] = 1; } /* Figure out dimensions of rectangle */ int n = QMP_get_number_of_nodes(); /* nodes to factor */ int k = MAXPRIMES-1; while (n > 1) { /* figure out which prime to divide by starting with largest */ /* if no factor found, assume n is prime */ while ((k >= 0) && (n % prime[k] != 0)) --k; int pfac = (k>=0) ? prime[k] : n; /* figure out which direction to divide */ /* find largest divisible dimension of h-cubes */ /* if one direction with largest dimension has already been divided, divide it again. Otherwise divide first direction with largest dimension. */ int j = -1; int i; for (i = 0; i < S->rank; i++) { if (squaresize[i] % pfac == 0) { if ((j<0) || (extrafactors[j] * squaresize[i] > extrafactors[i] * squaresize[j])) { j = i; } else if (extrafactors[j] * squaresize[i] == extrafactors[i] * squaresize[j]) { if ((S->net[j] == 1) || (S->net[i] != 1)) j = i; } } } /* This can fail if we run out of prime factors in the dimensions */ /* then just choose largest dimension */ if (j < 0) { int i; for (i = 0; i < S->rank; i++) { if ((j<0) || (extrafactors[j] * squaresize[i] > extrafactors[i] * squaresize[j]) ) { j = i; } else if (extrafactors[j] * squaresize[i] == extrafactors[i] * squaresize[j]) { if((S->net[j] == 1) || (S->net[i] != 1)) j = i; } } n /= pfac; extrafactors[j] *= pfac; S->net[j] *= pfac; } else { n /= pfac; squaresize[j] /= pfac; S->net[j] *= pfac; } } } /* not QMP_GRID */ int mc[QLUA_MAX_LATTICE_RANK]; int i; S->node = QDP_this_node; node2coord(mc, QDP_this_node, S); for (i = 0; i < S->rank; i++) { int x = mc[i]; mc[i] = x + 1; if (mc[i] == S->net[i]) mc[i] = 0; S->neighbor_up[i] = coord2node(mc, S); mc[i] = x - 1; if (mc[i] < 0) mc[i] = S->net[i] - 1; S->neighbor_down[i] = coord2node(mc, S); mc[i] = x; } }
void init_qmp(int * argc, char ***argv) { #if 0 printf("init_qmp(%d %p)\n",*argc,*argv); for(int i = 0; i<*argc;i++){ printf("argv[%d](before)=%s\n",i,(*argv)[i]); } #endif #if 0 spi_init(); #endif QMP_thread_level_t prv; #ifndef UNIFORM_SEED_NO_COMMS QMP_status_t init_status = QMP_init_msg_passing(argc, argv, QMP_THREAD_SINGLE, &prv); if (init_status) printf("QMP_init_msg_passing returned %d\n",init_status); peRank = QMP_get_node_number(); peNum = QMP_get_number_of_nodes(); if(!peRank)printf("QMP_init_msg_passing returned %d\n",init_status); if (init_status != QMP_SUCCESS) { QMP_error("%s\n",QMP_error_string(init_status)); } // check QMP thread level // Added by Hantao if(peRank == 0) { switch(prv) { case QMP_THREAD_SINGLE: printf("QMP thread level = QMP_THREAD_SINGLE\n"); break; case QMP_THREAD_FUNNELED: printf("QMP thread level = QMP_THREAD_FUNNELED\n"); break; case QMP_THREAD_SERIALIZED: printf("QMP thread level = QMP_THREAD_SERIALIZED\n"); break; case QMP_THREAD_MULTIPLE: printf("QMP thread level = QMP_THREAD_MULTIPLE\n"); break; default: printf("QMP thread level = no idea what this is, boom!\n"); } } //Check to make sure that this machine is a GRID machine //Exit if not GRID machine QMP_ictype qmp_type = QMP_get_msg_passing_type(); //Get information about the allocated machine peNum = QMP_get_number_of_nodes(); NDIM = QMP_get_allocated_number_of_dimensions(); peGrid = QMP_get_allocated_dimensions(); pePos = QMP_get_allocated_coordinates(); if(peRank==0){ for(int i = 0; i<*argc;i++){ printf("argv[%d])(after)=%s\n",i,(*argv)[i]); } } #else QMP_status_t init_status = QMP_SUCCESS; peRank=0; peNum=1; NDIM=4; #endif //#if (TARGET == BGL) || (TARGET == BGP) if (NDIM>5){ peNum = 1; for(int i = 0;i<5;i++) peNum *= peGrid[i]; peRank = peRank % peNum; } int if_print=1; for(int i = 0;i<NDIM;i++) if (pePos[i]>=2) if_print=0; if (if_print){ printf("Rank=%d Num=%d NDIM=%d\n",peRank,peNum,NDIM); printf("dim:"); for(int i = 0;i<NDIM;i++) printf(" %d",peGrid[i]); printf("\n"); printf("pos:"); for(int i = 0;i<NDIM;i++) printf(" %d",pePos[i]); printf("\n"); #if 0 int rc; BGLPersonality pers; rts_get_personality(&pers, sizeof(pers)); printf("from personality: %d %d %d %d\n",pers.xCoord,pers.yCoord,pers.zCoord,rts_get_processor_id()); #endif } // printf("from personality:\n"); #if 0 if ( (qmp_type!= QMP_GRID) && (qmp_type !=QMP_MESH) ) { QMP_error("CPS on QMP only implemented for GRID or MESH, not (%d) machines\n",qmp_type); } #endif // printf("QMP_declare_logical_topology(peGrid, NDIM)\n"); #ifndef UNIFORM_SEED_NO_COMMS //Declare the logical topology (Redundant for GRID machines) if (QMP_declare_logical_topology(peGrid, NDIM) != QMP_SUCCESS) { QMP_error("Node %d: Failed to declare logical topology\n",peRank); exit(-4); } #endif initialized = true; printf("Rank=%d init_qmp() done\n",peRank); }
void setup_layout(){ int k = mynode(); #ifdef FIX_NODE_GEOM int *geom = node_geometry; #else int *geom = NULL; #endif if(k == 0) printf("LAYOUT = Hypercubes, options = "); #ifdef HAVE_QMP /* QMP treatment */ /* Is there already a grid? This could be a grid architecture with a preset dimension, or a geometry could have been set by the -qmp-geom command line arg. In either case we have a nonzero allocated number of dimensions. */ if(QMP_get_allocated_number_of_dimensions() == 0) /* Set the geometry if requested */ set_qmp_layout_grid(geom, 4); /* Has a grid been set up now? */ if(QMP_get_msg_passing_type() == QMP_GRID) setup_qmp_grid(); else if(geom != NULL) setup_fixed_geom(geom, 4); else setup_hyper_prime(); #else /* Non QMP treatment */ if(geom != NULL) setup_fixed_geom(geom, 4); else setup_hyper_prime(); #endif #ifdef FIX_IONODE_GEOM /* Initialize I/O node function */ init_io_node(); #endif /* Compute machine coordinates for this node */ lex_coords(machine_coordinates, 4, nsquares, k); /* Number of sites on node */ sites_on_node = squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*squaresize[TUP]; /* Need number of sites per hypercube divisible by 32 */ if( mynode()==0)if( sites_on_node%32 != 0){ printf("SORRY, CAN'T LAY OUT THIS LATTICE\n"); terminate(0); } subl_sites_on_node = sites_on_node/32; if( mynode()==0) printf("ON EACH NODE %d x %d x %d x %d\n",squaresize[XUP],squaresize[YUP], squaresize[ZUP],squaresize[TUP]); /* Actually we have already required sites_on_node to be a multiple of 32 */ if( mynode()==0 && sites_on_node%2 != 0) printf("WATCH OUT FOR EVEN/ODD SITES ON NODE BUG!!!\n"); even_sites_on_node = odd_sites_on_node = sites_on_node/2; }
void setup_layout(){ register int i,j,k,dir; #ifdef HAVE_QMP if(QMP_get_msg_passing_type()==QMP_GRID){ printf("This layout should not be used on a grid architecture\n"); terminate(1); } #endif if(mynode()==0){ printf("LAYOUT = Hypercubes, options = "); printf("tstretch,"); printf("\n"); } /* Figure out dimensions of rectangle */ if( (nx*ny*nz*nt)%numnodes() == 0){ squaresize[XUP] = nx; squaresize[YUP] = ny; squaresize[ZUP] = nz; squaresize[TUP] = nt; } else if ( (nx*ny*nz*(nt+1))%numnodes() == 0){ squaresize[XUP] = nx; squaresize[YUP] = ny; squaresize[ZUP] = nz; squaresize[TUP] = nt+1; } nsquares[XUP] = nsquares[YUP] = nsquares[ZUP] = nsquares[TUP] = 1; i = 1; /* current number of hypercubes */ while(i<numnodes()){ /* figure out which prime to divide by starting with largest */ k = MAXPRIMES-1; while( (numnodes()/i)%prime[k] != 0 && k>0 ) --k; /* figure out which direction to divide */ /* find largest even dimension of h-cubes */ for(j=1,dir=XUP;dir<=TUP;dir++) if( squaresize[dir]>j && squaresize[dir]%prime[k]==0 ) j=squaresize[dir]; /* if one direction with largest dimension has already been divided, divide it again. Otherwise divide first direction with largest dimension. */ for(dir=XUP;dir<=TUP;dir++) if( squaresize[dir]==j && nsquares[dir]>1 )break; if( dir > TUP)for(dir=XUP;dir<=TUP;dir++) if( squaresize[dir]==j )break; /* This can fail if I run out of prime factors in the dimensions */ if(dir > TUP){ if(mynode()==0)printf( "LAYOUT: Can't lay out this lattice, not enough factors of %d\n" ,prime[k]); terminate(1); } /* do the surgery */ i*=prime[k]; squaresize[dir] /= prime[k]; nsquares[dir] *= prime[k]; } sites_on_node = squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*squaresize[TUP]; sites_on_lastnodes = sites_on_node; if( mynode()==0) printf("ON MOST NODES %d x %d x %d x %d\n",squaresize[XUP],squaresize[YUP], squaresize[ZUP],squaresize[TUP]); if( !((nx*ny*nz*nt)%numnodes() == 0) && (nx*ny*nz*(nt+1))%numnodes() == 0 ){ /* stretched t direction by one */ if( mynode()==0)printf("SOME NODES HAVE FEWER SITES\n"); if( mynode() >= nsquares[XUP]*nsquares[YUP]*nsquares[ZUP]*(nsquares[TUP]-1) ){ sites_on_node = squaresize[XUP]*squaresize[YUP]*squaresize[ZUP]*(squaresize[TUP]-1); sites_on_lastnodes = sites_on_node; } } /* Need even number of sites per hypercube */ if( mynode()==0)if( sites_on_node%2 != 0){ printf("SORRY, CAN'T LAY OUT THIS LATTICE\n"); terminate(0); } if( mynode()==0 && sites_on_node%2 != 0) printf("WATCH OUT FOR EVEN/ODD SITES ON NODE BUG!!!\n"); even_sites_on_node = odd_sites_on_node = sites_on_node/2; }