inline int DDI_ARMCI_Acc_domain_SMP(DDI_Patch *patch, void *scale, void *buf, int id) {
int handle = patch->handle;
int i,nsubp,nops;
int ranks[MAX_SMP_PROCS];
DDI_Patch subp[MAX_SMP_PROCS];
char *src = (char*)buf;
DDI_Subpatch_node(handle,patch,&nsubp,ranks,subp,id);
for (i = 0; i < nsubp; ++i) {
DDI_ARMCI_Acc_proc(&subp[i],scale,src,ranks[i]);
src += subp[i].size;
}
nops = nsubp;
return nops;
}
void DDI_ARR_select_(DDI_Patch *dAPatch, double *alpha, int *index) {
int i;
int op = dAPatch->oper;
DDI_Patch dASubPatch[MAX_PROCESSORS];
int dASubPatchRank[MAX_PROCESSORS], dANSubPatch;
DDI_ARR_Element recvbuf, *element = NULL;
const DDI_Comm *comm = (const DDI_Comm *) Comm_find(DDI_WORKING_COMM);
int rank = comm->me;
int node = comm->my;
int smpRank = comm->me_local;
int smpRoot;
DDI_Subpatch_node(dAPatch->handle, dAPatch, &dANSubPatch, dASubPatchRank, dASubPatch, node);
for (i = 0; i < dANSubPatch; ++i) {
if (dASubPatchRank[i] == rank) {
if (element == NULL) element = (DDI_ARR_Element*)malloc(sizeof(DDI_ARR_Element));
#ifdef DDI_ARR_DMA
DDI_ARR_select_local(&dASubPatch[i], element);
#else
DDI_ARR_select_remote(&dASubPatch[i], element, rank);
#endif
}
}
/* Now selected elements need to be gathered from other nodes.
Point-to-point is used because patch may not be located on all processors,
but perhaps consider change to collective operation.
IRecv or Recvany are not used because they seem to give problems.
*/
/* select intra-node element */
smpRoot = comm->node_master[node];
if (smpRank == 0) {
for (i = 0; i < dANSubPatch; ++i) {
if (dASubPatchRank[i] == rank) continue;
Comm_recv(&recvbuf, sizeof(DDI_ARR_Element), dASubPatchRank[i], comm);
DDI_ARR_Element_select(element, &recvbuf, op);
}
}
else if (element != NULL) {
Comm_send(element, sizeof(DDI_ARR_Element), smpRoot, comm);
}
/* select inter-node element */
DDI_Subpatch(dAPatch->handle, dAPatch, &dANSubPatch, dASubPatchRank, dASubPatch);
if (rank == 0) {
for (i = 0; i < dANSubPatch; ++i) {
if (dASubPatchRank[i] == rank) continue;
smpRoot = comm->node_master[dASubPatchRank[i]];
Comm_recv(&recvbuf, sizeof(DDI_ARR_Element), smpRoot, comm);
DDI_ARR_Element_select(element, &recvbuf, op);
}
}
else if (element != NULL && rank == smpRoot) {
Comm_send(element, sizeof(DDI_ARR_Element), 0, comm);
}
/* broadcast selected element */
if (element == NULL) element = (DDI_ARR_Element*)malloc(sizeof(DDI_ARR_Element));
Comm_bcast(element, sizeof(DDI_ARR_Element), 0, comm);
*alpha = element->alpha;
index[0] = element->index[0];
index[1] = element->index[1];
free(element);
}