int MPIDO_Reduce_global_tree(void * sendbuf, void * recvbuf, int count, DCMF_Dt dcmf_dt, DCMF_Op dcmf_op, MPI_Datatype data_type, int root, MPID_Comm * comm) { int rc, hw_root = comm->vcr[root]; DCMF_CollectiveRequest_t request; volatile unsigned active = 1; DCMF_Callback_t callback = { reduce_cb_done, (void *) &active }; rc = DCMF_GlobalAllreduce(&MPIDI_Protocols.globalallreduce, (DCMF_Request_t *)&request, callback, DCMF_MATCH_CONSISTENCY, hw_root, sendbuf, recvbuf, count, dcmf_dt, dcmf_op); MPID_PROGRESS_WAIT_WHILE(active); return rc; }
int main() { int i, rank, nranks, msgsize, status, expected; long bufsize; int *buffer; DCMF_Protocol_t ga_protocol; DCMF_GlobalAllreduce_Configuration_t ga_conf; DCMF_Request_t request; DCMF_Callback_t done_callback; volatile unsigned ga_active = 0; DCMF_Messager_initialize(); rank = DCMF_Messager_rank(); nranks = DCMF_Messager_size(); bufsize = MAX_MSG_SIZE; buffer = (int *) malloc(bufsize); ga_conf.protocol = DCMF_DEFAULT_GLOBALALLREDUCE_PROTOCOL; status = DCMF_GlobalAllreduce_register(&ga_protocol, &ga_conf); if(status != DCMF_SUCCESS) { printf("DCMF_GlobalAllreduce_register returned with error %d \n", status); exit(-1); } done_callback.function = done; done_callback.clientdata = (void *) &ga_active; if (rank == 0) { printf("DCMF_Allreduce Test\n"); fflush(stdout); } for (msgsize = sizeof(int); msgsize < MAX_MSG_SIZE; msgsize *= 2) { /*initializing buffer*/ for (i = 0; i < bufsize/sizeof(int); i++) { buffer[i] = rank; } ga_active += 1; /*sum reduce operation*/ status = DCMF_GlobalAllreduce(&ga_protocol, &request, done_callback, DCMF_SEQUENTIAL_CONSISTENCY, -1, (char *) buffer, (char *) buffer, msgsize/sizeof(int), DCMF_SIGNED_INT, DCMF_SUM); while(ga_active > 0) DCMF_Messager_advance(); expected = (nranks-1)*(nranks)/2; for (i = 0; i < msgsize/sizeof(int); i++) { if(buffer[i] - expected != 0) { printf("[%d] Validation has failed Expected: %d, Actual: %d, i: %d \n", rank, expected, buffer[i], i); fflush(stdout); exit(-1); } } printf("[%d] %d message sum reduce successful \n", rank, msgsize); fflush(stdout); for (i = 0; i < bufsize/sizeof(int); i++) { buffer[i] = 1; } ga_active += 1; status = DCMF_GlobalAllreduce(&ga_protocol, &request, done_callback, DCMF_SEQUENTIAL_CONSISTENCY, -1, (char *) buffer, (char *) buffer, msgsize/sizeof(int), DCMF_SIGNED_INT, DCMF_PROD); while(ga_active > 0) DCMF_Messager_advance(); expected = 1; for (i = 0; i < msgsize/sizeof(int); i++) { if(buffer[i] - expected != 0) { printf("[%d] Validation has failed Expected: %d, Actual: %d, i: %d \n", rank, expected, buffer[i], i); fflush(stdout); exit(-1); } } printf("[%d] %d message product reduce successful\n", rank, msgsize); fflush(stdout); } free(buffer); DCMF_Messager_finalize(); return 0; }