/* called when a receive should be progressed */
static int
ompi_mtl_portals_recv_progress(ptl_event_t *ev,
struct ompi_mtl_portals_request_t* ptl_request)
{
int ret;
switch (ev->type) {
case PTL_EVENT_PUT_END:
/* make sure the data is in the right place */
ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->md.start, ev->mlength);
/* set the status */
ptl_request->super.ompi_req->req_status.MPI_SOURCE =
PTL_GET_SOURCE(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_TAG =
PTL_GET_TAG(ev->match_bits);
ptl_request->super.ompi_req->req_status.MPI_ERROR =
(ev->rlength > ev->mlength) ?
MPI_ERR_TRUNCATE : MPI_SUCCESS;
ptl_request->super.ompi_req->req_status._count =
ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv complete: 0x%016llx\n", ev->match_bits));
ptl_request->super.completion_callback(&ptl_request->super);
break;
case PTL_EVENT_REPLY_END:
/* make sure the data is in the right place */
ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->md.start, ev->mlength);
ret=PtlMDUnlink(ev->md_handle);
if( ret !=PTL_OK) {
return ompi_common_portals_error_ptl_to_ompi(ret);
}
/* set the status - most of this filled in right after issuing
the PtlGet*/
ptl_request->super.ompi_req->req_status._count =
ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output,
"recv complete: 0x%016llx\n", ev->match_bits));
ptl_request->super.completion_callback(&ptl_request->super);
break;
default:
break;
}
return OMPI_SUCCESS;
}
static void ompi_mtl_mxm_recv_completion_cb(void *context)
{
mca_mtl_mxm_request_t *req = (mca_mtl_mxm_request_t *) context;
struct ompi_request_t *ompi_req = req->super.ompi_req;
mxm_recv_req_t *mxm_recv_req = &req->mxm.recv;
/* Set completion status and envelope */
ompi_mtl_mxm_to_mpi_status(mxm_recv_req->base.error, &ompi_req->req_status);
ompi_req->req_status.MPI_TAG = mxm_recv_req->completion.sender_tag;
ompi_req->req_status.MPI_SOURCE = mxm_recv_req->completion.sender_imm;
ompi_req->req_status._ucount = mxm_recv_req->completion.actual_len;
/* Copy data and free the buffer */
ompi_mtl_datatype_unpack(req->convertor, req->buf,
mxm_recv_req->completion.actual_len);
req->super.completion_callback(&req->super);
}
int ompi_mtl_psm_progress( void ) {
psm_error_t err;
mca_mtl_psm_request_t* mtl_psm_request;
psm_mq_status_t psm_status;
psm_mq_req_t req;
int completed = 1;
do {
err = psm_mq_ipeek(ompi_mtl_psm.mq, &req, NULL);
if (err == PSM_MQ_INCOMPLETE) {
return completed;
} else if (err != PSM_OK) {
goto error;
}
completed++;
err = psm_mq_test(&req, &psm_status);
if (err != PSM_OK) {
goto error;
}
mtl_psm_request = (mca_mtl_psm_request_t*) psm_status.context;
if (mtl_psm_request->type == OMPI_MTL_PSM_IRECV) {
ompi_mtl_datatype_unpack(mtl_psm_request->convertor,
mtl_psm_request->buf,
psm_status.msg_length);
mtl_psm_request->super.ompi_req->req_status.MPI_SOURCE =
PSM_GET_MQRANK(psm_status.msg_tag);
mtl_psm_request->super.ompi_req->req_status.MPI_TAG =
PSM_GET_MQUTAG(psm_status.msg_tag);
mtl_psm_request->super.ompi_req->req_status._ucount =
psm_status.nbytes;
}
if(mtl_psm_request->type == OMPI_MTL_PSM_ISEND) {
if (mtl_psm_request->free_after) {
free(mtl_psm_request->buf);
}
}
switch (psm_status.error_code) {
case PSM_OK:
mtl_psm_request->super.ompi_req->req_status.MPI_ERROR =
OMPI_SUCCESS;
break;
case PSM_MQ_TRUNCATION:
mtl_psm_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_TRUNCATE;
break;
default:
mtl_psm_request->super.ompi_req->req_status.MPI_ERROR =
MPI_ERR_INTERN;
}
mtl_psm_request->super.completion_callback(&mtl_psm_request->super);
}
while (1);
error:
opal_show_help("help-mtl-psm.txt",
"error polling network", true,
psm_error_get_string(err));
return 1;
}
static int
ompi_mtl_portals4_rndv_get_frag_progress(ptl_event_t *ev,
ompi_mtl_portals4_rndv_get_frag_t* rndv_get_frag)
{
int ret;
ompi_mtl_portals4_recv_request_t* ptl_request =
(ompi_mtl_portals4_recv_request_t*) rndv_get_frag->request;
assert(PTL_EVENT_REPLY == ev->type);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got reply event",
ptl_request->opcount, ptl_request->hdr_data));
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_REPLY with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_DROPPED)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"PTL_EVENT_REPLY with ni_fail_type: %u => cannot retry",
(uint32_t)ev->ni_fail_type);
ret = PTL_FAIL;
goto callback_error;
}
if (0 == rndv_get_frag->frag_abs_timeout_usec) {
/* this is the first retry of the frag. start the timer. */
/* instead of recording the start time, record the end time
* and avoid addition on each retry. */
rndv_get_frag->frag_abs_timeout_usec = opal_timer_base_get_usec() + ompi_mtl_portals4.get_retransmit_timeout;
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"setting frag timeout at %lu",
rndv_get_frag->frag_abs_timeout_usec);
} else if (opal_timer_base_get_usec() >= rndv_get_frag->frag_abs_timeout_usec) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"timeout retrying GET");
ret = PTL_FAIL;
goto callback_error;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Rendezvous Get Failed: Reissuing frag #%u", rndv_get_frag->frag_num));
ret = PtlGet(ompi_mtl_portals4.send_md_h,
(ptl_size_t) rndv_get_frag->frag_start,
rndv_get_frag->frag_length,
rndv_get_frag->frag_target,
ompi_mtl_portals4.read_idx,
rndv_get_frag->frag_match_bits,
rndv_get_frag->frag_remote_offset,
rndv_get_frag);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
return OMPI_SUCCESS;
}
/* set the received length in the status, now that we know
exactly how much data was sent. */
ptl_request->super.super.ompi_req->req_status._ucount += ev->mlength;
/* this frag is complete. return to freelist. */
opal_free_list_return (&ompi_mtl_portals4.fl_rndv_get_frag,
&rndv_get_frag->super);
ret = OPAL_THREAD_ADD32(&(ptl_request->pending_reply), -1);
if (ret > 0) {
return OMPI_SUCCESS;
}
assert(ptl_request->pending_reply == 0);
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
#endif
/* make sure the data is in the right place. Use _ucount for
the total length because it will be set correctly for all
three protocols. mlength is only correct for eager, and
delivery_len is the length of the buffer, not the length of
the send. */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ptl_request->delivery_ptr,
ptl_request->super.super.ompi_req->req_status._ucount);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed , reply (pending_reply: %d)",
ptl_request->opcount, ptl_request->hdr_data, ptl_request->pending_reply));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
return OMPI_SUCCESS;
callback_error:
ptl_request->super.super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);
ptl_request->super.super.completion_callback(&ptl_request->super.super);
return OMPI_SUCCESS;
}
/* called when a receive should be progressed */
static int
ompi_mtl_portals4_recv_progress(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t* ptl_base_request)
{
int ret;
ompi_mtl_portals4_recv_request_t* ptl_request =
(ompi_mtl_portals4_recv_request_t*) ptl_base_request;
size_t msg_length = 0;
/* as soon as we've seen any event associated with a request, it's
started */
ptl_request->req_started = true;
switch (ev->type) {
case PTL_EVENT_PUT:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got put event",
ptl_request->opcount, ev->hdr_data));
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_PUT with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
ret = PTL_FAIL;
goto callback_error;
}
ptl_request->me_h = PTL_INVALID_HANDLE;
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (OPAL_UNLIKELY(msg_length > ptl_request->delivery_len)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"truncate expected: %ld %ld",
msg_length, ptl_request->delivery_len);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
if (ev->mlength < msg_length)
OPAL_OUTPUT_VERBOSE((90, ompi_mtl_base_framework.framework_output, "Truncated message, some PtlGet are required (protocol = %d)",
ompi_mtl_portals4.protocol));
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (!MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits) && msg_length > ev->mlength) {
/* If it's not a short message and we're doing rndv and the message is not complete, we
only have the first part of the message. Issue the get
to pull the second part of the message. */
ret = read_msg((char*)ptl_request->delivery_ptr + ev->mlength,
((msg_length > ptl_request->delivery_len) ? ptl_request->delivery_len : msg_length) - ev->mlength,
ev->initiator,
ev->hdr_data,
ev->mlength,
ptl_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
} else {
/* If we're either using the eager protocol or were a
short message, all data has been received, so complete
the message. */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->start,
ev->mlength);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed, expected",
ptl_request->opcount, ptl_request->hdr_data));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
}
break;
case PTL_EVENT_PUT_OVERFLOW:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got put_overflow event",
ptl_request->opcount, ev->hdr_data));
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_PUT_OVERFLOW with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
ret = PTL_FAIL;
goto callback_error;
}
ptl_request->me_h = PTL_INVALID_HANDLE;
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (OPAL_UNLIKELY(msg_length > ptl_request->delivery_len)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"truncate unexpected: %ld %ld %d",
msg_length, ptl_request->delivery_len,
MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits));
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
/* overflow case. Short messages have the buffer stashed
somewhere. Long messages left in buffer at the source */
if (MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits)) {
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (ev->mlength > 0) {
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
iov.iov_base = (char*) ev->start;
iov.iov_len = ev->mlength;
max_data = iov.iov_len;
ret = opal_convertor_unpack(ptl_request->convertor,
&iov, &iov_count,
&max_data );
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
if (OPAL_UNLIKELY(ret < 0)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: opal_convertor_unpack failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
/* if it's a sync, send the ack */
if (MTL_PORTALS4_IS_SYNC_MSG(ev->hdr_data)) {
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) sending sync ack",
ptl_request->opcount, ptl_request->hdr_data));
ret = PtlPut(ompi_mtl_portals4.zero_md_h,
0,
0,
PTL_NO_ACK_REQ,
ev->initiator,
ompi_mtl_portals4.read_idx,
ev->hdr_data,
0,
NULL,
0);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed, unexpected short (0x%lx)",
ptl_request->opcount, ptl_request->hdr_data, (long) ev->start));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
} else {
/* For long messages in the overflow list, ev->mlength = 0 */
ptl_request->super.super.ompi_req->req_status._ucount = 0;
ret = read_msg((char*)ptl_request->delivery_ptr,
(msg_length > ptl_request->delivery_len) ? ptl_request->delivery_len : msg_length,
ev->initiator,
ev->hdr_data,
0,
ptl_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
}
break;
case PTL_EVENT_LINK:
break;
default:
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"Unhandled receive callback with event type %d",
ev->type);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
callback_error:
ptl_request->super.super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);
ptl_request->super.super.completion_callback(&ptl_request->super.super);
return OMPI_SUCCESS;
}
int ompi_mtl_mx_progress( void ) {
mx_return_t mx_return;
mx_request_t mx_request;
mx_status_t mx_status;
uint32_t result;
mca_mtl_mx_request_t* mtl_mx_request;
int completed = 0;
while(1){
mx_return = mx_ipeek(ompi_mtl_mx.mx_endpoint,
&mx_request,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_ipeek (error %s)\n", mx_strerror(mx_return));
}
if(result) {
completed++;
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_test (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(0 == result) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in ompi_mtl_mx_progress, mx_ipeek returned a request, mx_test on the request resulted failure.\n");
abort();
}
mtl_mx_request = (mca_mtl_mx_request_t*) mx_status.context;
if(OMPI_MTL_MX_ISEND == mtl_mx_request->type) {
if(mtl_mx_request->free_after) {
free(mtl_mx_request->mx_segment[0].segment_ptr);
}
} else {
assert( OMPI_MTL_MX_IRECV == mtl_mx_request->type );
ompi_mtl_datatype_unpack(mtl_mx_request->convertor,
mtl_mx_request->mx_segment[0].segment_ptr,
mx_status.xfer_length);
/* set the status */
MX_GET_SRC(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_SOURCE);
MX_GET_TAG(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_TAG);
mtl_mx_request->super.ompi_req->req_status._ucount =
mx_status.xfer_length;
}
/* suppose everything went just fine ... */
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
if( OPAL_UNLIKELY(MX_STATUS_SUCCESS != mx_status.code) ) {
if( MX_STATUS_TRUNCATED == mx_status.code ) {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
} else {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_INTERN;
}
return completed;
}
mtl_mx_request->super.completion_callback(&mtl_mx_request->super);
return completed;
} else {
return completed;
}
}
}
/* called when a receive should be progressed */
static int
ompi_mtl_portals4_recv_progress(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t* ptl_base_request)
{
int ret;
ompi_mtl_portals4_recv_request_t* ptl_request =
(ompi_mtl_portals4_recv_request_t*) ptl_base_request;
size_t msg_length = 0;
/* as soon as we've seen any event associated with a request, it's
started */
ptl_request->req_started = true;
switch (ev->type) {
case PTL_EVENT_PUT:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got put event",
ptl_request->opcount, ev->hdr_data));
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_PUT with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
goto callback_error;
}
ptl_request->me_h = PTL_INVALID_HANDLE;
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (OPAL_UNLIKELY(msg_length > ptl_request->delivery_len)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"truncate expected: %ld %ld",
msg_length, ptl_request->delivery_len);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
if (!MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits) && ompi_mtl_portals4.protocol == rndv) {
/* If it's not a short message and we're doing rndv, we
only have the first part of the message. Issue the get
to pull the second part of the message. */
ret = read_msg((char*) ptl_request->delivery_ptr + ompi_mtl_portals4.eager_limit,
((msg_length > ptl_request->delivery_len) ?
ptl_request->delivery_len : msg_length) - ompi_mtl_portals4.eager_limit,
ev->initiator,
ev->hdr_data,
ompi_mtl_portals4.eager_limit,
ptl_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
} else {
/* If we're either using the eager protocol or were a
short message, all data has been received, so complete
the message. */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->start,
ev->mlength);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed, expected",
ptl_request->opcount, ptl_request->hdr_data));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
}
break;
case PTL_EVENT_REPLY:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got reply event",
ptl_request->opcount, ptl_request->hdr_data));
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_REPLY with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
PtlMDRelease(ptl_request->md_h);
goto callback_error;
}
/* set the received length in the status, now that we know
excatly how much data was sent. */
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (ompi_mtl_portals4.protocol == rndv) {
ptl_request->super.super.ompi_req->req_status._ucount +=
ompi_mtl_portals4.eager_limit;
}
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
#endif
/* make sure the data is in the right place. Use _ucount for
the total length because it will be set correctly for all
three protocols. mlength is only correct for eager, and
delivery_len is the length of the buffer, not the length of
the send. */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ptl_request->delivery_ptr,
ptl_request->super.super.ompi_req->req_status._ucount);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
PtlMDRelease(ptl_request->md_h);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed, reply",
ptl_request->opcount, ptl_request->hdr_data));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
break;
case PTL_EVENT_PUT_OVERFLOW:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) got put_overflow event",
ptl_request->opcount, ev->hdr_data));
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PTL_EVENT_PUT_OVERFLOW with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
goto callback_error;
}
ptl_request->me_h = PTL_INVALID_HANDLE;
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (OPAL_UNLIKELY(msg_length > ptl_request->delivery_len)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"truncate unexpected: %ld %ld %d",
msg_length, ptl_request->delivery_len,
MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits));
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
/* overflow case. Short messages have the buffer stashed
somewhere. Long messages left in buffer at the source */
if (MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits)) {
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (ev->mlength > 0) {
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
iov.iov_base = (char*) ev->start;
iov.iov_len = ev->mlength;
max_data = iov.iov_len;
ret = opal_convertor_unpack(ptl_request->convertor,
&iov, &iov_count,
&max_data );
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
if (OPAL_UNLIKELY(ret < 0)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: opal_convertor_unpack failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
/* if it's a sync, send the ack */
if (MTL_PORTALS4_IS_SYNC_MSG(ev->hdr_data)) {
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) sending sync ack",
ptl_request->opcount, ptl_request->hdr_data));
ret = PtlPut(ompi_mtl_portals4.zero_md_h,
0,
0,
PTL_NO_ACK_REQ,
ev->initiator,
ompi_mtl_portals4.read_idx,
ev->hdr_data,
0,
NULL,
0);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Recv %lu (0x%lx) completed, unexpected short (0x%lx)",
ptl_request->opcount, ptl_request->hdr_data, (long) ev->start));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
} else {
if (ev->mlength > 0) {
/* if rndv or triggered, copy the eager part to the right place */
memcpy(ptl_request->delivery_ptr, ev->start, ev->mlength);
}
ret = read_msg((char*) ptl_request->delivery_ptr + ev->mlength,
((msg_length > ptl_request->delivery_len) ?
ptl_request->delivery_len : msg_length) - ev->mlength,
ev->initiator,
ev->hdr_data,
ev->mlength,
ptl_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
}
break;
case PTL_EVENT_LINK:
break;
default:
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"Unhandled receive callback with event type %d",
ev->type);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
callback_error:
ptl_request->super.super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);
ptl_request->super.super.completion_callback(&ptl_request->super.super);
return OMPI_SUCCESS;
}
/* called when a receive should be progressed */
int
ompi_mtl_portals4_recv_progress(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t* ptl_base_request)
{
int ret;
ompi_mtl_portals4_recv_request_t* ptl_request =
(ompi_mtl_portals4_recv_request_t*) ptl_base_request;
size_t msg_length = 0;
switch (ev->type) {
case PTL_EVENT_PUT:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) got put event",
ptl_request->opcount, ev->hdr_data));
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PTL_EVENT_PUT with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
goto callback_error;
}
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (msg_length > ptl_request->delivery_len) {
opal_output_verbose(1, ompi_mtl_base_output, "truncate expected: %ld %ld",
msg_length, ptl_request->delivery_len);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
if (!MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits) && ompi_mtl_portals4.protocol == rndv) {
ptl_md_t md;
md.start = (char*) ptl_request->delivery_ptr + ompi_mtl_portals4.eager_limit;
md.length = ((msg_length > ptl_request->delivery_len) ?
ptl_request->delivery_len : msg_length) - ompi_mtl_portals4.eager_limit;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
ret = PtlGet(ptl_request->md_h,
0,
md.length,
ev->initiator,
ompi_mtl_portals4.read_idx,
ev->hdr_data,
ompi_mtl_portals4.eager_limit,
ptl_request);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PtlGet failed: %d",
__FILE__, __LINE__, ret);
PtlMDRelease(ptl_request->md_h);
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
goto callback_error;
}
} else {
/* make sure the data is in the right place */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ev->start,
ev->mlength);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) completed, expected",
ptl_request->opcount, ptl_request->hdr_data));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
}
break;
case PTL_EVENT_REPLY:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) got reply event",
ptl_request->opcount, ptl_request->hdr_data));
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PTL_EVENT_REPLY with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
PtlMDRelease(ptl_request->md_h);
goto callback_error;
}
/* set the status - most of this filled in right after issuing
the PtlGet */
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (ompi_mtl_portals4.protocol == rndv) {
ptl_request->super.super.ompi_req->req_status._ucount +=
ompi_mtl_portals4.eager_limit;
}
/* make sure the data is in the right place. Use _ucount for
the total length because it will be set correctly for all
three protocols. mlength is only correct for eager, and
delivery_len is the length of the buffer, not the length of
the send. */
ret = ompi_mtl_datatype_unpack(ptl_request->convertor,
ptl_request->delivery_ptr,
ptl_request->super.super.ompi_req->req_status._ucount);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: ompi_mtl_datatype_unpack failed: %d",
__FILE__, __LINE__, ret);
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
}
PtlMDRelease(ptl_request->md_h);
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) completed, reply",
ptl_request->opcount, ptl_request->hdr_data));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
break;
case PTL_EVENT_PUT_OVERFLOW:
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) got put_overflow event",
ptl_request->opcount, ev->hdr_data));
if (ev->ni_fail_type != PTL_NI_OK) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PTL_EVENT_PUT_OVERFLOW with ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
goto callback_error;
}
msg_length = MTL_PORTALS4_GET_LENGTH(ev->hdr_data);
ptl_request->super.super.ompi_req->req_status.MPI_SOURCE =
MTL_PORTALS4_GET_SOURCE(ev->match_bits);
ptl_request->super.super.ompi_req->req_status.MPI_TAG =
MTL_PORTALS4_GET_TAG(ev->match_bits);
if (msg_length > ptl_request->delivery_len) {
opal_output_verbose(1, ompi_mtl_base_output, "truncate unexpected: %ld %ld %d",
msg_length, ptl_request->delivery_len, MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits));
ptl_request->super.super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
}
#if OPAL_ENABLE_DEBUG
ptl_request->hdr_data = ev->hdr_data;
#endif
/* overflow case. Short messages have the buffer stashed
somewhere. Long messages left in buffer at the source */
if (MTL_PORTALS4_IS_SHORT_MSG(ev->match_bits)) {
ptl_request->super.super.ompi_req->req_status._ucount = ev->mlength;
if (ev->mlength > 0) {
struct iovec iov;
uint32_t iov_count = 1;
size_t max_data;
iov.iov_base = (char*) ev->start;
iov.iov_len = ev->mlength;
max_data = iov.iov_len;
ret = opal_convertor_unpack(ptl_request->convertor,
&iov, &iov_count,
&max_data );
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
if (ret < 0) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: opal_convertor_unpack failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
/* if it's a sync, send the ack */
if (MTL_PORTALS4_IS_SYNC_MSG(ev->hdr_data)) {
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) sending sync ack",
ptl_request->opcount, ptl_request->hdr_data));
ret = PtlPut(ompi_mtl_portals4.zero_md_h,
0,
0,
PTL_NO_ACK_REQ,
ev->initiator,
ompi_mtl_portals4.read_idx,
ev->hdr_data,
0,
NULL,
0);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PtlPut failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) completed, unexpected short (0x%lx)",
ptl_request->opcount, ptl_request->hdr_data, (long) ev->start));
ptl_request->super.super.completion_callback(&ptl_request->super.super);
} else {
ptl_md_t md;
if (ev->mlength > 0) {
/* if rndv or triggered, copy the eager part to the right place */
memcpy(ptl_request->delivery_ptr, ev->start, ev->mlength);
}
md.start = (char*) ptl_request->delivery_ptr + ev->mlength;
md.length = ((msg_length > ptl_request->delivery_len) ?
ptl_request->delivery_len : msg_length) - ev->mlength;
md.options = 0;
md.eq_handle = ompi_mtl_portals4.eq_h;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(ompi_mtl_portals4.ni_h,
&md,
&ptl_request->md_h);
if (PTL_OK != ret) {
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PtlMDBind failed: %d",
__FILE__, __LINE__, ret);
goto callback_error;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d (0x%lx) getting long data",
ptl_request->opcount, ptl_request->hdr_data));
ret = PtlGet(ptl_request->md_h,
0,
md.length,
ev->initiator,
ompi_mtl_portals4.read_idx,
ev->hdr_data,
ev->mlength,
ptl_request);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_output,
"%s:%d: PtlGet failed: %d",
__FILE__, __LINE__, ret);
if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr);
PtlMDRelease(ptl_request->md_h);
goto callback_error;
}
}
break;
default:
opal_output_verbose(1, ompi_mtl_base_output,
"Unhandled receive callback with event type %d",
ev->type);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
callback_error:
ptl_request->super.super.ompi_req->req_status.MPI_ERROR =
ompi_mtl_portals4_get_error(ret);
ptl_request->super.super.completion_callback(&ptl_request->super.super);
return OMPI_SUCCESS;
}
