void F77_FUNC(mpix_comm_rank2global,MPIX_COMM_RANK2GLOBAL)(MPI_Fint * comm, MPI_Fint * crank, MPI_Fint * grank, MPI_Fint * ierr) { MPI_Comm mycomm = MPI_Comm_f2c(*comm); int mycrank = -1, mygrank = -1; safecast(*crank, &mycrank); *ierr = (MPI_Fint)MPIX_Comm_rank2global(mycomm, mycrank, &mygrank); *grank = (MPI_Fint)mygrank; return; }
static void QMP_declare_multiple_bgspi(QMP_msghandle_t mh) { if(mh->paired) { int npaired = 0; int src0 = -1; while(1) { int src = -1; for(QMP_msghandle_t m=mh->next; m; m=m->next) { if(src<=src0 || (m->srce_node>src0 && m->srce_node<src)) src = m->srce_node; } if(src<=src0) break; for(QMP_msghandle_t m=mh->next; m; m=m->next) { if(m->srce_node==src) { int nmsg = 0, nmsgo; switch(m->mm->type) { case MM_user_buf: nmsg = 1; break; case MM_strided_buf: if(m->type==MH_send) nmsg = m->mm->mm.st.nblocks; break; case MM_strided_array_buf: if(m->type==MH_send) { for(int i=0; i<m->mm->mm.sa.narray; i++) { nmsg += m->mm->mm.sa.nblocks[i]; } } break; case MM_indexed_buf: if(m->type==MH_send) nmsg = m->mm->mm.in.count; break; } //printf("%i pairing %i -> %i\n", QMP_get_node_number(), m->srce_node, m->dest_node); if(m->type==MH_send) { MPI_Status st; MPI_Send(&nmsg, sizeof(nmsg), MPI_BYTE, m->dest_node, 0, m->comm->mpicomm); MPI_Recv(&nmsgo, sizeof(nmsgo), MPI_BYTE, m->dest_node, 0, m->comm->mpicomm, &st); } else if(m->type==MH_recv) { MPI_Status st; MPI_Recv(&nmsgo, sizeof(nmsgo), MPI_BYTE, m->srce_node, 0, m->comm->mpicomm, &st); MPI_Send(&nmsg, sizeof(nmsg), MPI_BYTE, m->srce_node, 0, m->comm->mpicomm); } else { QMP_FATAL("error: bad msg type"); } m->paired = 0; if(nmsg>0 && nmsgo>0) { void *base[nmsg]; size_t size[nmsg]; switch(m->mm->type) { case MM_user_buf: base[0]=m->base; size[0]=m->mm->nbytes; break; case MM_strided_buf: { int offset = 0; for(int i=0; i<nmsg; i++) { base[i] = (char*)m->base + offset; size[i] = m->mm->mm.st.blksize; offset += m->mm->mm.st.stride; } } break; case MM_strided_array_buf: { int k = 0; for(int i=0; i<m->mm->mm.sa.narray; i++) { int offset = 0; char *basei = (char*)m->base + m->mm->mm.sa.disp[i]; for(int j=0; j<m->mm->mm.sa.nblocks[i]; j++) { base[k] = basei + offset; size[k] = m->mm->mm.sa.blksize[i]; k++; offset += m->mm->mm.sa.stride[i]; } } } break; case MM_indexed_buf: { double avgbl = 0; for(int i=0; i<nmsg; i++) { base[i] = (char*)m->base + m->mm->mm.in.elemsize*m->mm->mm.in.index[i]; size[i] = m->mm->mm.in.elemsize*m->mm->mm.in.blocklen[i]; avgbl += m->mm->mm.in.blocklen[i]; } avgbl = (m->mm->mm.in.elemsize*avgbl)/nmsg; //if(QMP_get_node_number()==0) printf("indexed: n: %6i e: %6i a: %9.2f\n", nmsg, m->mm->mm.in.elemsize, avgbl); } break; } if(m->type==MH_send) { int grank; MPIX_Comm_rank2global(m->comm->mpicomm, m->dest_node, &grank); m->qspimsg = qspi_create_msg(); qspi_set_send_multi(grank, base, size, nmsg, m->qspimsg); qspi_prepare(&m->qspimsg, 1); //printf("%i testing msg send to %i\n", m->srce_node, m->dest_node); //qspi_start(m->qspimsg); //qspi_wait(m->qspimsg); //printf("%i testing msg send to %i done\n", m->srce_node, m->dest_node); m->qspicts = qspi_create_msg(); qspi_set_recv(grank, &m->clear_to_send, sizeof(m->clear_to_send), m->qspicts); qspi_prepare(&m->qspicts, 1); //printf("%i testing cts send to %i\n", m->srce_node, m->dest_node); //qspi_start(m->qspicts); //qspi_wait(m->qspicts); //printf("%i testing cts send to %i done\n", m->srce_node, m->dest_node); } else { // recv int grank; MPIX_Comm_rank2global(m->comm->mpicomm, m->srce_node, &grank); m->qspimsg = qspi_create_msg(); //qspi_set_recv_multi(grank, base, size, nmsg, m->qspimsg); qspi_set_recv(grank, base[0], size[0], m->qspimsg); qspi_prepare(&m->qspimsg, 1); //printf("%i testing msg recv from %i\n", m->dest_node, m->srce_node); //qspi_start(m->qspimsg); //qspi_wait(m->qspimsg); //printf("%i testing msg recv from %i done\n", m->dest_node, m->srce_node); m->qspicts = qspi_create_msg(); qspi_set_send(grank, &m->clear_to_send, sizeof(m->clear_to_send), m->qspicts); qspi_prepare(&m->qspicts, 1); //printf("%i testing cts recv from %i\n", m->dest_node, m->srce_node); //qspi_start(m->qspicts); //qspi_wait(m->qspicts); //printf("%i testing cts recv from %i done\n", m->dest_node, m->srce_node); } npaired++; m->paired = 1; m->useSPI = 1; //printf("%i paired %i -> %i\n", QMP_get_node_number(), m->srce_node, m->dest_node); } else { //printf("%i unpaired %i -> %i type: %i\n", QMP_get_node_number(), m->srce_node, m->dest_node, m->mm->type); } } } src0 = src; } mh->useSPI = npaired; } //QMP_declare_multiple_mpi(mh); }