void armci_completion_handler(lapi_handle_t *t_hndl, void *save)
{
lapi_handle_t hndl = *t_hndl;
int need_data;
void *message;
int whofrom, msglen;
request_header_t *msginfo = (request_header_t *)save;
char *descr= (char*)(msginfo+1), *buf=MessageRcvBuffer;
int buflen=MSG_BUFLEN;
#if ARMCI_ENABLE_GPC_CALLS
extern pthread_t data_server;
data_server = pthread_self();
#endif
if(DEBUG_)
fprintf(stderr,"%d:CH:op=%d from=%d datalen=%d dscrlen=%d\n", armci_me,
msginfo->operation, msginfo->from,msginfo->datalen,msginfo->dscrlen);
/*** assure that descriptor and data are in the right format and place ***/
if( msginfo->dscrlen < 0 || msginfo->datalen <0 ){
/* for large put/acc/scatter need to get the data */
int rc;
lapi_cntr_t req_cntr;
int bytes=0;
char *origin_ptr = msginfo->tag.buf;
if (msginfo->dscrlen<0) {
descr =MessageRcvBuffer;
msginfo->dscrlen = -msginfo->dscrlen;
buf = descr + msginfo->dscrlen;
buflen += msginfo->dscrlen;
bytes += msginfo->dscrlen;
}
if (msginfo->datalen <0){
msginfo->datalen = -msginfo->datalen;
bytes += msginfo->datalen;
}
if(rc=LAPI_Setcntr(hndl, &req_cntr, 0)) ERROR("CH:setcntr failed",rc);
if(rc=LAPI_Get(hndl, (uint)msginfo->from, bytes,
origin_ptr, MessageRcvBuffer,
msginfo->tag.cntr,&req_cntr))ERROR("CH:LAPI_Get failed",rc);
if(rc=LAPI_Waitcntr(hndl, &req_cntr,1,NULL))ERROR("CH:Waitcntr failed",rc);
} else{
/* desc is in save, data could be but not for GET */
if(msginfo->operation !=GET)buf = descr + msginfo->dscrlen;
buflen = MSG_BUFLEN;
}
/* fprintf(stderr,"CH: val=%lf\n",*(double*)(buf+msginfo->datalen -8));*/
/*** dispatch request to the appropriate handler function ***/
switch(msginfo->operation){
case LOCK: armci_server_lock(msginfo);
break;
case UNLOCK: armci_server_unlock(msginfo, descr);
break;
default:
if(msginfo->format == STRIDED)
armci_server(msginfo, descr, buf, buflen);
else
armci_server_vector(msginfo, descr, buf, buflen);
}
free(msginfo);
#ifdef LINUX
(void)fetch_and_add(&num_malloc, (long)-1);
#else
(void)fetch_and_addlp(&num_malloc, (long)-1);
#endif
}
void DDI_GetAcc_lapi_server(DDI_Patch *patch, void *buffer) {
/* --------------- *\
Local Variables
\* --------------- */
lapi_handle_t hndl;
uint tgt;
ulong len;
void *tgt_addr,*org_addr;
lapi_cntr_t *tgt_cntr,*org_cntr,*cmpl_cntr;
lapi_cntr_t local_cntr;
STD_DEBUG((stdout,"%s: Entering DDI_GetAcc_lapi_server.\n",DDI_Id()))
/* ----------------------------------------- *\
Set up the calling arguments for LAPI_Get
\* ----------------------------------------- */
hndl = gv(lapi_hnd); /* LAPI Handle */
tgt = patch->cp_lapi_id; /* Target for the LAPI_Get */
len = (ulong) patch->size; /* Amount of data to get */
tgt_addr = patch->cp_buffer_addr; /* Addr at target to get */
org_addr = buffer; /* Local Addr for data that is got */
tgt_cntr = (lapi_cntr_t *) patch->cp_lapi_cntr; /* Target counter */
org_cntr = &local_cntr; /* Local counter -> incremented once
the LAPI_Get is completed */
cmpl_cntr = NULL;
ULTRA_DEBUG((stdout,"%s: DDI_Patch -> ilo=%i ihi=%i jlo=%i jhi=%i size=%lu.\n",
DDI_Id(),patch->ilo,patch->ihi,patch->jlo,patch->jhi,patch->size))
ULTRA_DEBUG((stdout,"%s: org buffer_addr=%x cntr_addr=%x.\n",DDI_Id(),org_addr,org_cntr))
ULTRA_DEBUG((stdout,"%s: tgt id=%i buffer_addr=%x cntr_addr=%x.\n",DDI_Id(),tgt,tgt_addr,tgt_cntr))
/* -------------------------------------------------------------------------- *\
We are interested in when the LAPI_Get is finished, zero the local counter
\* -------------------------------------------------------------------------- */
if(LAPI_Setcntr(hndl,org_cntr,0) != LAPI_SUCCESS) {
fprintf(stdout,"%s: LAPI_Setcntr failed in DDI_GetAcc_lapi_server.\n",DDI_Id());
Fatal_error(911);
}
ULTRA_DEBUG((stdout,"%s: Initializing local LAPI counter.\n",DDI_Id()))
/* --------------------------------------------------- *\
Execute the LAPI_Get. This is a non-blocking call.
\* --------------------------------------------------- */
if(LAPI_Get(hndl,tgt,len,tgt_addr,org_addr,tgt_cntr,org_cntr) != LAPI_SUCCESS) {
fprintf(stdout,"%s: LAPI_Get failed in DDI_GetAcc_lapi_server.\n",DDI_Id());
Fatal_error(911);
}
MAX_DEBUG((stdout,"%s: Executing LAPI_Get from %i.\n",DDI_Id(),tgt))
/* ------------------------------------------------------------------------ *\
Wait here until the local counter is incremented ==> LAPI_Get completed.
\* ------------------------------------------------------------------------ */
if(LAPI_Waitcntr(hndl,org_cntr,1,NULL) != LAPI_SUCCESS) {
fprintf(stdout,"%s: LAPI_Waitcntr failed in DDI_GetAcc_lapi_server.\n",DDI_Id());
Fatal_error(911);
}
MAX_DEBUG((stdout,"%s: LAPI_Get from %i completed.\n",DDI_Id(),tgt))
/* -------------------------------------------------------------- *\
Place the data (now local) into the shared-memory of the node.
\* -------------------------------------------------------------- */
MAX_DEBUG((stdout,"%s: LAPI handler calling DDI_GetAcc_local.\n",DDI_Id()))
DDI_GetAcc_local(patch,buffer);
MAX_DEBUG((stdout,"%s: LAPI handler completed DDI_GetAcc_local.\n",DDI_Id()))
STD_DEBUG((stdout,"%s: Exiting DDI_GetAcc_lapi_server.\n",DDI_Id()))
/* --------------------------------------------------- *\
Execute the LAPI_Put. This is a non-blocking call.
\* --------------------------------------------------- */
if(LAPI_Put(hndl,tgt,len,tgt_addr,org_addr,tgt_cntr,org_cntr,cmpl_cntr) != LAPI_SUCCESS) {
fprintf(stdout,"%s: LAPI_Get failed in DDI_GetAcc_lapi_server.\n",DDI_Id());
Fatal_error(911);
}
MAX_DEBUG((stdout,"%s: Executing LAPI_Put from %i.\n",DDI_Id(),tgt))
/* ------------------------------------------------------------------------- *\
Wait here until the local counter is incremented ==> LAPI_Put has copied.
\* ------------------------------------------------------------------------- */
if(LAPI_Waitcntr(hndl,org_cntr,1,NULL) != LAPI_SUCCESS) {
fprintf(stdout,"%s: LAPI_Waitcntr failed in DDI_GetAcc_lapi_server.\n",DDI_Id());
Fatal_error(911);
}
MAX_DEBUG((stdout,"%s: LAPI_Put copied data enroute to %i.\n",DDI_Id(),tgt))
}
