int MPIC_Send(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag,
MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPID_Request *request_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SEND);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
default:
MPIR_TAG_SET_ERROR_BIT(tag);
}
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
mpi_errno = MPID_Send(buf, count, datatype, dest, tag, comm_ptr,
context_id, &request_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPID_Request_release(request_ptr);
}
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SEND);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) MPID_Request_release(request_ptr);
if (mpi_errno && !*errflag) {
if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
*errflag = MPIR_ERR_PROC_FAILED;
} else {
*errflag = MPIR_ERR_OTHER;
}
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_lmt_vmsplice_done_send(MPIDI_VC_t *vc, MPID_Request *sreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_VMSPLICE_DONE_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_VMSPLICE_DONE_SEND);
/* nop */
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_VMSPLICE_DONE_SEND);
return MPI_SUCCESS;
}
void *MPIDI_Alloc_mem( size_t size, MPID_Info *info_ptr )
{
void *ap;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_ALLOC_MEM);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_ALLOC_MEM);
ap = MPIU_Malloc(size);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_ALLOC_MEM);
return ap;
}
int MPID_nem_lmt_vmsplice_handle_cookie(MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV cookie)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_VMSPLICE_HANDLE_COOKIE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_VMSPLICE_HANDLE_COOKIE);
/* nop */
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_VMSPLICE_HANDLE_COOKIE);
return MPI_SUCCESS;
}
int MPIDI_Free_mem( void *ptr )
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_FREE_MEM);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_FREE_MEM);
MPIU_Free(ptr);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_FREE_MEM);
return mpi_errno;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
MPIDI_Message_match * match)
{
MPID_Request * rreq;
MPID_Request * prev_rreq;
MPID_Request * cur_rreq;
MPID_Request * matching_prev_rreq;
MPID_Request * matching_cur_rreq;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
matching_prev_rreq = NULL;
matching_cur_rreq = NULL;
prev_rreq = NULL;
/* Note that since this routine is used only in the case of send_cancel,
there can be only one match if at all. */
/* FIXME: Why doesn't this exit after it finds the first match? */
cur_rreq = recvq_unexpected_head;
while (cur_rreq != NULL) {
if (cur_rreq->dev.sender_req_id == sreq_id &&
(MATCH_WITH_NO_MASK(cur_rreq->dev.match, *match))) {
matching_prev_rreq = prev_rreq;
matching_cur_rreq = cur_rreq;
}
prev_rreq = cur_rreq;
cur_rreq = cur_rreq->dev.next;
}
if (matching_cur_rreq != NULL) {
if (matching_prev_rreq != NULL) {
matching_prev_rreq->dev.next = matching_cur_rreq->dev.next;
}
else {
recvq_unexpected_head = matching_cur_rreq->dev.next;
}
if (matching_cur_rreq->dev.next == NULL) {
recvq_unexpected_tail = matching_prev_rreq;
}
rreq = matching_cur_rreq;
}
else {
rreq = NULL;
}
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
return rreq;
}
int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPI_Status mystatus;
MPID_Request *request_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_RECV);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
if (status == MPI_STATUS_IGNORE)
status = &mystatus;
mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
context_id, status, &request_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr, errflag);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
*status = request_ptr->status;
mpi_errno = status->MPI_ERROR;
MPID_Request_release(request_ptr);
} else {
MPIR_Process_status(status, errflag);
MPIR_TAG_CLEAR_ERROR_BITS(status->MPI_TAG);
}
if (MPI_SUCCESS == MPIR_ERR_GET_CLASS(status->MPI_ERROR)) {
MPIU_Assert(status->MPI_TAG == tag);
}
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_RECV);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) MPID_Request_release(request_ptr);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_tcp_init (MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_INIT);
MPID_nem_net_module_vc_dbg_print_sendq = MPID_nem_tcp_vc_dbg_print_sendq;
/* first make sure that our private fields in the vc fit into the area provided */
MPIU_Assert(sizeof(MPID_nem_tcp_vc_area) <= MPIDI_NEM_VC_NETMOD_AREA_LEN);
/* set up listener socket */
mpi_errno = set_up_listener();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* create business card */
mpi_errno = MPID_nem_tcp_get_business_card(pg_rank, bc_val_p, val_max_sz_p);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_tcp_sm_init();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_tcp_send_init();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
#ifdef HAVE_SIGNAL
{
/* In order to be able to handle socket errors on our own, we need
to ignore SIGPIPE. This may cause problems for programs that
intend to handle SIGPIPE or count on being killed, but I expect
such programs are very rare, and I'm not sure what the best
solution would be anyway. */
void *ret;
ret = signal(SIGPIPE, SIG_IGN);
MPIR_ERR_CHKANDJUMP1(ret == SIG_ERR, mpi_errno, MPI_ERR_OTHER, "**signal", "**signal %s", MPIU_Strerror(errno));
if (ret != SIG_DFL && ret != SIG_IGN) {
/* The app has set its own signal handler. Replace the previous handler. */
ret = signal(SIGPIPE, ret);
MPIR_ERR_CHKANDJUMP1(ret == SIG_ERR, mpi_errno, MPI_ERR_OTHER, "**signal", "**signal %s", MPIU_Strerror(errno));
}
}
#endif
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_INIT);
/* fprintf(stdout, FCNAME " Exit\n"); fflush(stdout); */
return mpi_errno;
fn_fail:
/* fprintf(stdout, "failure. mpi_errno = %d\n", mpi_errno); */
goto fn_exit;
}
static int reinit_pmi(void)
{
int ret;
int has_parent = 0;
int pg_rank, pg_size;
int kvs_name_sz, pg_id_sz;
MPIDI_STATE_DECL(MPID_STATE_REINIT_PMI);
MPIDI_FUNC_ENTER(MPID_STATE_REINIT_PMI);
/* Init pmi and do some sanity checks */
ret = PMI_Init(&has_parent);
CHECK_ERR(ret, "pmi_init");
ret = PMI_Get_rank(&pg_rank);
CHECK_ERR(ret, "pmi_get_rank");
ret = PMI_Get_size(&pg_size);
CHECK_ERR(ret, "pmi_get_size");
CHECK_ERR(pg_size != MPIDI_Process.my_pg->size, "pg size differs after restart");
CHECK_ERR(pg_rank != MPIDI_Process.my_pg_rank, "pg rank differs after restart");
/* get new pg_id */
ret = PMI_KVS_Get_name_length_max(&pg_id_sz);
CHECK_ERR(ret, "pmi_get_id_length_max");
MPIU_Free(MPIDI_Process.my_pg->id);
MPIDI_Process.my_pg->id = MPIU_Malloc(pg_id_sz + 1);
CHECK_ERR(MPIDI_Process.my_pg->id == NULL, "malloc failed");
ret = PMI_KVS_Get_my_name(MPIDI_Process.my_pg->id, pg_id_sz);
CHECK_ERR(ret, "pmi_kvs_get_my_name");
/* get new kvsname */
ret = PMI_KVS_Get_name_length_max(&kvs_name_sz);
CHECK_ERR(ret, "PMI_KVS_Get_name_length_max");
MPIU_Free(MPIDI_Process.my_pg->connData);
MPIDI_Process.my_pg->connData = MPIU_Malloc(kvs_name_sz + 1);
CHECK_ERR(MPIDI_Process.my_pg->connData == NULL, "malloc failed");
ret = PMI_KVS_Get_my_name(MPIDI_Process.my_pg->connData, kvs_name_sz);
CHECK_ERR(ret, "PMI_Get_my_name");
MPIDI_FUNC_EXIT(MPID_STATE_REINIT_PMI);
return 0;
}
void MPIDI_CH3_Rendezvouz_r3_ack_recv(MPIDI_VC_t * vc,
MPIDI_CH3_Pkt_rndv_r3_ack_t *r3ack_pkt)
{
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_RNDV_R3_ACK_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_RNDV_R3_ACK_RECV);
DEBUG_PRINT("Received R3 Ack %d\n", r3ack_pkt->ack_data);
vc->ch.pending_r3_data -= r3ack_pkt->ack_data;
MPIU_Assert(vc->ch.pending_r3_data == 0);
PUSH_FLOWLIST(vc);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_RNDV_R3_ACK_RECV);
}
int MPID_nem_tcp_module_lmt_post_recv (MPIDI_VC_t *vc, MPID_Request *req)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_POST_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_ib_lmt_done_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV);
dprintf("lmt_done_recv,enter,rreq=%p,head=%p\n", rreq, MPID_nem_ib_lmtq.head);
int is_contig;
MPID_Datatype_is_contig(rreq->dev.datatype, &is_contig);
if (!is_contig) {
dprintf("lmt_done_recv,copying noncontiguous data to user buffer\n");
/* see MPIDI_CH3U_Request_unpack_uebuf (in /src/mpid/ch3/src/ch3u_request.c) */
/* or MPIDI_CH3U_Receive_data_found (in src/mpid/ch3/src/ch3u_handle_recv_pkt.c) */
MPIDI_msg_sz_t unpack_sz = rreq->ch.lmt_data_sz;
MPID_Segment seg;
MPI_Aint last;
MPID_Segment_init(rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &seg, 0);
last = unpack_sz;
MPID_Segment_unpack(&seg, 0, &last, REQ_FIELD(rreq, lmt_pack_buf));
if (last != unpack_sz) {
/* --BEGIN ERROR HANDLING-- */
/* received data was not entirely consumed by unpack()
* because too few bytes remained to fill the next basic
* datatype */
MPIR_STATUS_SET_COUNT(rreq->status, last);
rreq->status.MPI_ERROR =
MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TYPE, "**MPID_nem_ib_lmt_done_recv", 0);
/* --END ERROR HANDLING-- */
}
//MPIU_Free(REQ_FIELD(rreq, lmt_pack_buf));
MPID_nem_ib_stfree(REQ_FIELD(rreq, lmt_pack_buf), (size_t) rreq->ch.lmt_data_sz);
}
dprintf("lmt_done_recv,1,req=%p,pcc=%d\n", rreq, MPIDI_CH3I_progress_completion_count.v);
MPIDI_CH3U_Request_complete(rreq);
dprintf("lmt_done_recv,complete,req=%p\n", rreq);
dprintf("lmt_done_recv,2,pcc=%d\n", MPIDI_CH3I_progress_completion_count.v);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_DONE_RECV);
return mpi_errno;
//fn_fail:
goto fn_exit;
}
static int handler_recv_dequeue_complete(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *const rreq = e->user_ptr;
int is_contig;
MPI_Aint last;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr ATTRIBUTE((unused));
MPIDI_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIU_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, is_contig, data_sz, dt_ptr, dt_true_lb);
dequeue_req(e);
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
/* unpack the data from unexpected buffer */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "is_contig = %d", is_contig);
if (is_contig) {
MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, e->start);
if (last != e->mlength)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
} else {
/* Data was placed directly into the user buffer, so datatype mismatch
is harder to detect. We use a simple check ensuring the received bytes
are a multiple of a single basic element. Currently, we do not detect
mismatches with datatypes constructed of more than one basic type */
MPI_Datatype dt_basic_type;
MPID_Datatype_get_basic_type(rreq->dev.datatype, dt_basic_type);
if (dt_basic_type != MPI_DATATYPE_NULL && (e->mlength % MPID_Datatype_get_basic_size(dt_basic_type)) != 0)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
mpi_errno = handler_recv_complete(e);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_nem_ckpt_finish(void)
{
int mpi_errno = MPI_SUCCESS;
int i;
int ret;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
/* Since we're checkpointing the shared memory region (i.e., the
channels between local procs), we don't have to flush those
channels, just make sure no one is sending or receiving during
the checkpoint */
mpi_errno = MPID_nem_barrier();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
ret = sem_post(&ckpt_sem);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**sem_post", "**sem_post %s", MPIU_Strerror(errno));
ret = sem_wait(&cont_sem);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**sem_wait", "**sem_wait %s", MPIU_Strerror(errno));
mpi_errno = MPID_nem_barrier();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (ckpt_result == CKPT_CONTINUE) {
for (i = 0; i < MPIDI_Process.my_pg->size; ++i) {
MPIDI_VC_t *vc;
MPIDI_CH3I_VC *vc_ch;
/* We didn't send a marker to ourselves. */
if (i == MPIDI_Process.my_pg_rank)
continue;
MPIDI_PG_Get_vc(MPIDI_Process.my_pg, i, &vc);
vc_ch = VC_CH(vc);
if (!vc_ch->is_local) {
mpi_errno = vc_ch->ckpt_continue_vc(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
}
checkpointing = FALSE;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_PG_Create_from_string(const char * str, MPIDI_PG_t ** pg_pptr,
int *flag)
{
int mpi_errno = MPI_SUCCESS;
const char *p;
int vct_sz;
MPIDI_PG_t *existing_pg, *pg_ptr=0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_CREATE_FROM_STRING);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_CREATE_FROM_STRING);
/*printf( "PgCreateFromString: Creating pg from %s\n", str );
fflush(stdout); */
/* The pg_id is at the beginning of the string, so we can just pass
it to the find routine */
/* printf( "Looking for pg with id %s\n", str );fflush(stdout); */
mpi_errno = MPIDI_PG_Find((void *)str, &existing_pg);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (existing_pg != NULL) {
/* return the existing PG */
*pg_pptr = existing_pg;
*flag = 0;
/* Note that the memory for the pg_id is freed in the exit */
goto fn_exit;
}
*flag = 1;
/* Get the size from the string */
p = str;
while (*p) p++; p++;
vct_sz = atoi(p);
mpi_errno = MPIDI_PG_Create(vct_sz, (void *)str, pg_pptr);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
pg_ptr = *pg_pptr;
pg_ptr->id = MPIU_Strdup( str );
/* Set up the functions to use strings to manage connection information */
MPIDI_PG_InitConnString( pg_ptr );
(*pg_ptr->connInfoFromString)( str, pg_ptr );
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_CREATE_FROM_STRING);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_finalize(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_FINALIZE);
/* this test is not the right one */
/* MPIU_Assert(MPID_nem_queue_empty( MPID_nem_mem_region.RecvQ[MPID_nem_mem_region.rank])); */
/* these are allocated in MPID_nem_mpich_init, not MPID_nem_init */
MPIU_Free(MPID_nem_recv_seqno);
MPIU_Free(MPID_nem_fboxq_elem_list);
/* from MPID_nem_init */
MPIU_Free(MPID_nem_mem_region.FreeQ);
MPIU_Free(MPID_nem_mem_region.RecvQ);
MPIU_Free(MPID_nem_mem_region.local_ranks);
if (MPID_nem_mem_region.ext_procs > 0)
MPIU_Free(MPID_nem_mem_region.ext_ranks);
MPIU_Free(MPID_nem_mem_region.seg);
MPIU_Free(MPID_nem_mem_region.mailboxes.out);
MPIU_Free(MPID_nem_mem_region.mailboxes.in);
MPIU_Free(MPID_nem_mem_region.local_procs);
#ifdef MEM_REGION_IN_HEAP
MPIU_Free(MPID_nem_mem_region_ptr);
#endif /* MEM_REGION_IN_HEAP */
mpi_errno = MPID_nem_netmod_func->finalize();
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
/* free the shared memory segment */
mpi_errno = MPIDI_CH3I_Seg_destroy();
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
#ifdef PAPI_MONITOR
my_papi_close();
#endif /*PAPI_MONITOR */
if (ENABLE_PVAR_NEM) {
MPIU_Free(MPID_nem_fbox_fall_back_to_queue_count);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_FINALIZE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Nem_nd_vc_destroy(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_nem_ptl_anysource_iprobe(int tag, MPID_Comm * comm, int context_offset, int *flag, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IPROBE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IPROBE);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IPROBE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mxm_iSendContig");
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);
MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t));
_dbg_mxm_output(5,
"iSendContig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n",
vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t),
data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area->ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt);
req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t);
if (data_sz) {
req_area->iov_count = 2;
req_area->iov_buf[1].ptr = (void *) data;
req_area->iov_buf[1].length = data_sz;
}
vc_area->pending_sends += 1;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM,
mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
{
int found;
MPID_Time_t timer_start;
MPID_Request * cur_rreq;
MPID_Request * prev_rreq;
int dequeue_failed;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
found = FALSE;
prev_rreq = NULL;
/* MT FIXME is this right? or should the caller do this? */
MPIU_THREAD_CS_ENTER(MSGQUEUE,);
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
cur_rreq = recvq_posted_head;
while (cur_rreq != NULL) {
if (cur_rreq == rreq) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = cur_rreq->dev.next;
}
else {
recvq_posted_head = cur_rreq->dev.next;
}
if (cur_rreq->dev.next == NULL) {
recvq_posted_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
/* Notify channel that rreq has been dequeued and check if
it has already matched rreq, fail if so */
dequeue_failed = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
if (!dequeue_failed)
found = TRUE;
break;
}
prev_rreq = cur_rreq;
cur_rreq = cur_rreq->dev.next;
}
if (!found)
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_failed_matching_postedq);
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
return found;
}
static int handler_recv_dequeue_unpack_large(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *const rreq = e->user_ptr;
MPIDI_VC_t *vc;
MPI_Aint last;
void *buf;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_UNPACK_LARGE);
MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_UNPACK_LARGE);
MPIU_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Comm_get_vc(rreq->comm, NPTL_MATCH_GET_RANK(e->match_bits), &vc);
dequeue_req(e);
if (!(e->hdr_data & NPTL_LARGE)) {
/* all data has already been received; we're done */
mpi_errno = handler_recv_unpack_complete(e);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
goto fn_exit;
}
if (e->type == PTL_EVENT_PUT_OVERFLOW)
buf = e->start;
else
buf = REQ_PTL(rreq)->chunk_buffer[0];
MPIU_Assert(e->mlength == PTL_LARGE_THRESHOLD);
last = PTL_LARGE_THRESHOLD;
MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, buf);
MPIU_Assert(last == PTL_LARGE_THRESHOLD);
rreq->dev.segment_first += PTL_LARGE_THRESHOLD;
MPIU_Free(REQ_PTL(rreq)->chunk_buffer[0]);
MPIU_CHKPMEM_MALLOC(REQ_PTL(rreq)->chunk_buffer[0], void *, rreq->dev.segment_size - rreq->dev.segment_first,
mpi_errno, "chunk_buffer");
big_get(REQ_PTL(rreq)->chunk_buffer[0], rreq->dev.segment_size - rreq->dev.segment_first, vc, e->match_bits, rreq);
fn_exit:
MPIU_CHKPMEM_COMMIT();
fn_exit2:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_UNPACK_LARGE);
return mpi_errno;
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit2;
}
int MPID_Nem_nd_ckpt_shutdown()
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
mpi_errno = MPID_Nem_nd_finalize();
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_nem_mxm_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MXM_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MXM_CONNECT_TO_ROOT);
MPIU_ERR_SETFATAL(mpi_errno, MPI_ERR_OTHER, "**notimpl");
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MXM_CONNECT_TO_ROOT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_lmt_dma_vc_terminated(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_DMA_VC_TERMINATED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_DMA_VC_TERMINATED);
/* Do nothing. KNEM should abort any ops with dead processes. */
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_DMA_VC_TERMINATED);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline int connection_post_recv_pkt(MPIDI_CH3I_Connection_t * conn)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_CONNECTION_POST_RECV_PKT);
MPIDI_FUNC_ENTER(MPID_STATE_CONNECTION_POST_RECV_PKT);
mpi_errno = MPIDU_Sock_post_read(conn->sock, &conn->pkt, sizeof(conn->pkt), sizeof(conn->pkt), NULL);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SET(mpi_errno,MPI_ERR_OTHER, "**fail");
}
MPIDI_FUNC_EXIT(MPID_STATE_CONNECTION_POST_RECV_PKT);
return mpi_errno;
}
int MPID_Nem_nd_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_Nem_nd_connect_to_root(const char *business_card, MPIDI_VC_t *new_vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPIDI_CH3U_Receive_data_unexpected(MPID_Request * rreq, char *buf, MPIDI_msg_sz_t *buflen, int *complete)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPIU_Malloc(rreq->dev.recv_data_sz);
if (!rreq->dev.tmpbuf) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
/* if all of the data has already been received, copy it
now, otherwise build an iov and let the channel copy it */
if (rreq->dev.recv_data_sz <= *buflen)
{
MPIU_Memcpy(rreq->dev.tmpbuf, buf, rreq->dev.recv_data_sz);
*buflen = rreq->dev.recv_data_sz;
rreq->dev.recv_pending_count = 1;
*complete = TRUE;
}
else
{
rreq->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)((char *)rreq->dev.tmpbuf);
rreq->dev.iov[0].MPID_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.recv_pending_count = 2;
*buflen = 0;
*complete = FALSE;
}
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_ADD(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
return mpi_errno;
}
int MPID_nem_ptl_anysource_improbe(int tag, MPID_Comm * comm, int context_offset, int *flag, MPID_Request **message,
MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IMPROBE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IMPROBE);
return MPID_nem_ptl_improbe(NULL, MPI_ANY_SOURCE, tag, comm, context_offset, flag, message, status);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_ANYSOURCE_IMPROBE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
void MPID_nem_mxm_anysource_posted(MPID_Request * req)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
_dbg_mxm_output(5, "Any Source ========> Posting req %p \n", req);
mpi_errno = MPID_nem_mxm_recv(NULL, req);
MPIU_Assert(mpi_errno == MPI_SUCCESS);
_dbg_mxm_out_req(req);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
}
