void initCommsQuda(int argc, char **argv, const int *X, const int nDim) {
if (nDim != 4) errorQuda("Comms dimensions %d != 4", nDim);
#ifdef MULTI_GPU
#ifdef QMP_COMMS
QMP_thread_level_t tl;
QMP_init_msg_passing(&argc, &argv, QMP_THREAD_SINGLE, &tl);
QMP_declare_logical_topology(X, nDim);
#elif defined(MPI_COMMS)
MPI_Init (&argc, &argv);
int volume = 1;
for (int d=0; d<nDim; d++) volume *= X[d];
int size = -1;
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (volume != size)
errorQuda("Number of processes %d must match requested MPI volume %d",
size, volume);
comm_set_gridsize(X[0], X[1], X[2], X[3]);
comm_init();
#endif
#endif
}
/* Sets the QMP logical topology if we need one */
static void set_qmp_layout_grid(const int *geom, int n){
if(geom == NULL)return;
if(QMP_declare_logical_topology(geom, n) != QMP_SUCCESS){
node0_printf("setup_layout: QMP_declare_logical_topology failed on %d %d %d %d \n",
geom[0], geom[1], geom[2], geom[3] );
terminate(1);
}
}
int main(int argc, char **argv) {
#ifdef QMP_COMMS
int ndim=4, dims[4];
QMP_thread_level_t tl;
QMP_init_msg_passing(&argc, &argv, QMP_THREAD_SINGLE, &tl);
dims[0] = dims[1] = dims[2] = 1;
dims[3] = QMP_get_number_of_nodes();
QMP_declare_logical_topology(dims, ndim);
#endif
SU3Test();
#ifdef QMP_COMMS
QMP_finalize_msg_passing();
#endif
return 0;
}
int
main(int argc, char *argv[])
{
struct QOP_MDWF_State *mdwf_state = NULL;
struct QOP_MDWF_Parameters *mdwf_params = NULL;
QMP_thread_level_t qt = QMP_THREAD_SINGLE;
int status = 1;
int i;
if (QMP_init_msg_passing(&argc, &argv, qt, &qt) != QMP_SUCCESS) {
fprintf(stderr, "QMP_init() failed\n");
return 1;
}
for (i = 0; i < NELEM(b5); i++) {
b5[i] = 0.1 * i * (NELEM(b5) - i);
c5[i] = 0.1 * i * i * (NELEM(b5) - i);
}
self = QMP_get_node_number();
primary = QMP_is_primary_node();
if (argc != 7) {
zprint("7 arguments expected, found %d", argc);
zprint("usage: localheat Lx Ly Lz Lt Ls time");
QMP_finalize_msg_passing();
return 1;
}
for (i = 0; i < 4; i++) {
mynetwork[i] = 1;
mylocal[i] = atoi(argv[i+1]);
mylattice[i] = mylocal[i] * mynetwork[i];
}
mylocal[4] = mylattice[4] = atoi(argv[5]);
total_sec = atoi(argv[6]);
zshowv4("network", mynetwork);
zshowv5("local lattice", mylocal);
zshowv5("lattice", mylattice);
zprint("total requested runtime %.0f sec", total_sec);
#if 0
if (QMP_declare_logical_topology(mynetwork, 4) != QMP_SUCCESS) {
zprint("declare_logical_top failed");
goto end;
}
getv(mynode, 0, QMP_get_logical_number_of_dimensions(),
QMP_get_logical_coordinates());
#else
{ int i;
for (i = 0; i < 4; i++)
mynode[i] = 0;
}
#endif
if (QOP_MDWF_init(&mdwf_state,
mylattice, mynetwork, mynode, primary,
getsub, NULL)) {
zprint("MDWF_init() failed");
goto end;
}
zprint("MDWF_init() done");
if (QOP_MDWF_set_generic(&mdwf_params, mdwf_state, b5, c5, 0.123, 0.05)) {
zprint("MDW_set_generic() failed");
goto end;
}
zprint("MDWF_set_generic() done");
if (do_run(mdwf_state, mdwf_params)) {
zprint("float test failed");
goto end;
}
QOP_MDWF_fini(&mdwf_state);
zprint("Heater test finished");
status = 0;
end:
QMP_finalize_msg_passing();
return status;
}
void init_qmp(int * argc, char ***argv) {
#if 0
printf("init_qmp(%d %p)\n",*argc,*argv);
for(int i = 0; i<*argc;i++){
printf("argv[%d](before)=%s\n",i,(*argv)[i]);
}
#endif
#if 0
spi_init();
#endif
QMP_thread_level_t prv;
#ifndef UNIFORM_SEED_NO_COMMS
QMP_status_t init_status = QMP_init_msg_passing(argc, argv, QMP_THREAD_SINGLE, &prv);
if (init_status) printf("QMP_init_msg_passing returned %d\n",init_status);
peRank = QMP_get_node_number();
peNum = QMP_get_number_of_nodes();
if(!peRank)printf("QMP_init_msg_passing returned %d\n",init_status);
if (init_status != QMP_SUCCESS) {
QMP_error("%s\n",QMP_error_string(init_status));
}
// check QMP thread level
// Added by Hantao
if(peRank == 0) {
switch(prv) {
case QMP_THREAD_SINGLE:
printf("QMP thread level = QMP_THREAD_SINGLE\n");
break;
case QMP_THREAD_FUNNELED:
printf("QMP thread level = QMP_THREAD_FUNNELED\n");
break;
case QMP_THREAD_SERIALIZED:
printf("QMP thread level = QMP_THREAD_SERIALIZED\n");
break;
case QMP_THREAD_MULTIPLE:
printf("QMP thread level = QMP_THREAD_MULTIPLE\n");
break;
default:
printf("QMP thread level = no idea what this is, boom!\n");
}
}
//Check to make sure that this machine is a GRID machine
//Exit if not GRID machine
QMP_ictype qmp_type = QMP_get_msg_passing_type();
//Get information about the allocated machine
peNum = QMP_get_number_of_nodes();
NDIM = QMP_get_allocated_number_of_dimensions();
peGrid = QMP_get_allocated_dimensions();
pePos = QMP_get_allocated_coordinates();
if(peRank==0){
for(int i = 0; i<*argc;i++){
printf("argv[%d])(after)=%s\n",i,(*argv)[i]);
}
}
#else
QMP_status_t init_status = QMP_SUCCESS;
peRank=0;
peNum=1;
NDIM=4;
#endif
//#if (TARGET == BGL) || (TARGET == BGP)
if (NDIM>5){
peNum = 1;
for(int i = 0;i<5;i++)
peNum *= peGrid[i];
peRank = peRank % peNum;
}
int if_print=1;
for(int i = 0;i<NDIM;i++)
if (pePos[i]>=2) if_print=0;
if (if_print){
printf("Rank=%d Num=%d NDIM=%d\n",peRank,peNum,NDIM);
printf("dim:");
for(int i = 0;i<NDIM;i++)
printf(" %d",peGrid[i]);
printf("\n");
printf("pos:");
for(int i = 0;i<NDIM;i++)
printf(" %d",pePos[i]);
printf("\n");
#if 0
int rc;
BGLPersonality pers;
rts_get_personality(&pers, sizeof(pers));
printf("from personality: %d %d %d %d\n",pers.xCoord,pers.yCoord,pers.zCoord,rts_get_processor_id());
#endif
}
// printf("from personality:\n");
#if 0
if ( (qmp_type!= QMP_GRID) && (qmp_type !=QMP_MESH) ) {
QMP_error("CPS on QMP only implemented for GRID or MESH, not (%d) machines\n",qmp_type);
}
#endif
// printf("QMP_declare_logical_topology(peGrid, NDIM)\n");
#ifndef UNIFORM_SEED_NO_COMMS
//Declare the logical topology (Redundant for GRID machines)
if (QMP_declare_logical_topology(peGrid, NDIM) != QMP_SUCCESS) {
QMP_error("Node %d: Failed to declare logical topology\n",peRank);
exit(-4);
}
#endif
initialized = true;
printf("Rank=%d init_qmp() done\n",peRank);
}
int
main (int argc, char** argv)
{
int i, nc;
QMP_status_t status;
int **smem, **rmem;
QMP_msgmem_t *recvmem;
QMP_msghandle_t *recvh;
QMP_msgmem_t *sendmem;
QMP_msghandle_t *sendh;
struct perf_argv pargv;
QMP_thread_level_t req, prv;
/**
* Simple point to point topology
*/
int dims[4] = {2,2,2,2};
int ndims = 1;
//if(QMP_get_node_number()==0)
//printf("starting init\n"); fflush(stdout);
req = QMP_THREAD_SINGLE;
status = QMP_init_msg_passing (&argc, &argv, req, &prv);
if (status != QMP_SUCCESS) {
fprintf (stderr, "QMP_init failed\n");
return -1;
}
if(QMP_get_node_number()==0)
printf("finished init\n"); fflush(stdout);
if (parse_options (argc, argv, &pargv) == -1) {
if(QMP_get_node_number()==0)
usage (argv[0]);
exit (1);
}
{
int maxdims = 4;
int k=0;
int nodes = QMP_get_number_of_nodes();
ndims = 0;
while( (nodes&1) == 0 ) {
if(ndims<maxdims) ndims++;
else {
dims[k] *= 2;
k++;
if(k>=maxdims) k = 0;
}
nodes /= 2;
}
if(nodes != 1) {
QMP_error("invalid number of nodes %i", QMP_get_number_of_nodes());
QMP_error(" must power of 2");
QMP_abort(1);
}
pargv.ndims = ndims;
}
status = QMP_declare_logical_topology (dims, ndims);
if (status != QMP_SUCCESS) {
fprintf (stderr, "Cannot declare logical grid\n");
return -1;
}
/* do a broadcast of parameter */
if (QMP_broadcast (&pargv, sizeof (pargv)) != QMP_SUCCESS) {
QMP_printf ("Broadcast parameter failed\n");
exit (1);
}
{
int k=1;
const int *lc = QMP_get_logical_coordinates();
for(i=0; i<ndims; i++) k += lc[i];
pargv.sender = k&1;
}
QMP_printf("%s options: num_channels[%d] verify[%d] option[%d] datasize[%d] numloops[%d] sender[%d] strided_send[%i] strided_recv[%i] strided_array_send[%i] ",
argv[0], pargv.num_channels, pargv.verify,
pargv.option, pargv.size, pargv.loops, pargv.sender,
strided_send, strided_recv, strided_array_send);
fflush(stdout);
/**
* Create memory
*/
nc = pargv.num_channels;
smem = (int **)malloc(nc*sizeof (int *));
rmem = (int **)malloc(nc*sizeof (int *));
sendmem = (QMP_msgmem_t *)malloc(ndims*nc*sizeof (QMP_msgmem_t));
recvmem = (QMP_msgmem_t *)malloc(ndims*nc*sizeof (QMP_msgmem_t));
sendh = (QMP_msghandle_t *)malloc(nc*sizeof (QMP_msghandle_t));
recvh = (QMP_msghandle_t *)malloc(nc*sizeof (QMP_msghandle_t));
QMP_barrier();
if(QMP_get_node_number()==0) printf("\n"); fflush(stdout);
if(pargv.option & TEST_SIMUL) {
int opts = pargv.option;
pargv.option = TEST_SIMUL;
if(QMP_get_node_number()==0)
QMP_printf("starting simultaneous sends"); fflush(stdout);
for(i=pargv.minsize; i<=pargv.maxsize; i*=pargv.facsize) {
pargv.size = i;
create_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc, i, &pargv);
test_simultaneous_send (smem, rmem, sendh, recvh, &pargv);
check_mem(rmem, ndims, nc, i);
free_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc);
}
if(QMP_get_node_number()==0)
QMP_printf("finished simultaneous sends\n"); fflush(stdout);
pargv.option = opts;
}
if(pargv.option & TEST_PINGPONG) {
int opts = pargv.option;
pargv.option = TEST_PINGPONG;
if(QMP_get_node_number()==0)
QMP_printf("starting ping pong sends"); fflush(stdout);
for(i=pargv.minsize; i<=pargv.maxsize; i*=pargv.facsize) {
pargv.size = i;
create_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc, i, &pargv);
if(pargv.verify)
test_pingpong_verify(smem, rmem, sendh, recvh, &pargv);
else
test_pingpong(smem, rmem, sendh, recvh, &pargv);
check_mem(rmem, ndims, nc, i);
free_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc);
}
if(QMP_get_node_number()==0)
QMP_printf("finished ping pong sends\n"); fflush(stdout);
pargv.option = opts;
}
if(pargv.option & TEST_ONEWAY) {
int opts = pargv.option;
pargv.option = TEST_ONEWAY;
if(QMP_get_node_number()==0)
QMP_printf("starting one way sends"); fflush(stdout);
for(i=pargv.minsize; i<=pargv.maxsize; i*=pargv.facsize) {
pargv.size = i;
create_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc, i, &pargv);
test_oneway (smem, rmem, sendh, recvh, &pargv);
if(!pargv.sender) check_mem(rmem, ndims, nc, i);
free_msgs(smem, rmem, sendmem, recvmem, sendh, recvh, ndims, nc);
}
if(QMP_get_node_number()==0)
QMP_printf("finished one way sends"); fflush(stdout);
pargv.option = opts;
}
/**
* Free memory
*/
free (smem);
free (rmem);
free (sendh);
free (recvh);
free (sendmem);
free (recvmem);
QMP_finalize_msg_passing ();
return 0;
}
