static int
flowctl_fanout_callback(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t *ptl_base_request)
{
int ret;
struct timeval tv;
ompi_mtl_portals4.flowctl.flowctl_active = false;
ret = PtlPTEnable(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.recv_idx);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTEnabled failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
gettimeofday(&tv, NULL);
if (((tv.tv_sec * 1000000 + tv.tv_usec) -
(ompi_mtl_portals4.flowctl.tv.tv_sec * 1000000 + ompi_mtl_portals4.flowctl.tv.tv_usec))
< 1000000 * ompi_mtl_portals4.flowctl.backoff_count) {
usleep(++ompi_mtl_portals4.flowctl.backoff_count);
} else {
ompi_mtl_portals4.flowctl.backoff_count = 0;
}
ompi_mtl_portals4.flowctl.tv = tv;
ompi_mtl_portals4_pending_list_progress();
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Exiting flowctl_fanout_callback %ld",
ompi_mtl_portals4.flowctl.epoch_counter));
return OMPI_SUCCESS;
}
static inline int
ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
ompi_mtl_portals4_isend_request_t* ptl_request)
{
int ret= OMPI_SUCCESS;
void *start;
size_t length;
bool free_after;
ptl_process_t ptl_proc;
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
opal_free_list_item_t *item;
ompi_mtl_portals4_pending_request_t *pending;
#endif
if ((ompi_mtl_portals4.use_logical) && (MPI_COMM_WORLD == comm)) {
ptl_proc.rank = dest;
} else {
ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, dest);
ptl_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc));
}
ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
if (OMPI_SUCCESS != ret) return ret;
ptl_request->opcount = OPAL_THREAD_ADD64((int64_t*)&ompi_mtl_portals4.opcount, 1);
ptl_request->buffer_ptr = (free_after) ? start : NULL;
ptl_request->event_count = 0;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"Send %lu to %x,%x of length %ld\n",
ptl_request->opcount,
ptl_proc.phys.nid,
ptl_proc.phys.pid,
(int64_t)length));
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
item = opal_free_list_get (&ompi_mtl_portals4.flowctl.pending_fl);
if (NULL == item) return OMPI_ERR_OUT_OF_RESOURCE;
pending = (ompi_mtl_portals4_pending_request_t*) item;
ptl_request->pending = pending;
pending->mode = mode;
pending->start = start;
pending->length = length;
pending->contextid = comm->c_contextid;
pending->tag = tag;
pending->my_rank = comm->c_my_rank;
pending->fc_notified = 0;
pending->ptl_proc = ptl_proc;
pending->ptl_request = ptl_request;
if (OPAL_UNLIKELY(OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, -1) < 0)) {
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
&pending->super.super);
return OMPI_SUCCESS;
}
if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
&pending->super.super);
ompi_mtl_portals4_pending_list_progress();
return OMPI_SUCCESS;
}
if (OPAL_UNLIKELY(ompi_mtl_portals4.flowctl.flowctl_active)) {
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
&pending->super.super);
return OMPI_SUCCESS;
}
#endif
if (length <= ompi_mtl_portals4.eager_limit) {
ret = ompi_mtl_portals4_short_isend(mode,
start,
length,
comm->c_contextid,
tag,
comm->c_my_rank,
ptl_proc,
ptl_request);
} else {
ret = ompi_mtl_portals4_long_isend(start,
length,
comm->c_contextid,
tag,
comm->c_my_rank,
ptl_proc,
ptl_request);
}
return ret;
}
static inline int
ompi_mtl_portals4_callback(ptl_event_t *ev,
ompi_mtl_portals4_base_request_t* ptl_base_request,
bool *complete)
{
int retval = OMPI_SUCCESS, ret, val, add = 1;
ompi_mtl_portals4_isend_request_t* ptl_request =
(ompi_mtl_portals4_isend_request_t*) ptl_base_request;
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
if (OPAL_UNLIKELY(ev->ni_fail_type == PTL_NI_PT_DISABLED)) {
ompi_mtl_portals4_pending_request_t *pending =
ptl_request->pending;
OPAL_OUTPUT_VERBOSE((10, ompi_mtl_base_framework.framework_output,
"send %lu hit flow control (%d)",
ptl_request->opcount, ev->type));
/* BWB: FIX ME: this is a hack.. */
if (pending->fc_notified) {
return OMPI_SUCCESS;
}
pending->fc_notified = 1;
if (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback PtlMEUnlink returned %d",
__FILE__, __LINE__, ret);
}
}
opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
&pending->super.super);
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
ompi_mtl_portals4_flowctl_trigger();
return OMPI_SUCCESS;
}
#endif
if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback ni_fail_type: %d",
__FILE__, __LINE__, ev->ni_fail_type);
*complete = true;
return OMPI_ERROR;
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"send %lu got event of type %d",
ptl_request->opcount, ev->type));
if ((PTL_EVENT_ACK == ev->type) &&
(PTL_PRIORITY_LIST == ev->ptl_list) &&
(eager == ompi_mtl_portals4.protocol) &&
(!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE))) {
/* long expected messages with the eager protocol won't see a
get event to complete the message. Give them an extra
count to cause the message to complete with just the SEND
and ACK events and remove the ME. (we wait for the counter
to reach 3 events, but short messages start the counter at
1, so they don't need to enter this path) */
ret = PtlMEUnlink(ptl_request->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: send callback PtlMEUnlink returned %d",
__FILE__, __LINE__, ret);
}
add++;
}
val = OPAL_THREAD_ADD32((int32_t*)&ptl_request->event_count, add);
assert(val <= 3);
if (val == 3) {
if (NULL != ptl_request->buffer_ptr) {
free(ptl_request->buffer_ptr);
}
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, "send %lu completed",
ptl_request->opcount));
*complete = true;
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
opal_free_list_return (&ompi_mtl_portals4.flowctl.pending_fl,
&ptl_request->pending->super);
if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
ompi_mtl_portals4_pending_list_progress();
}
#endif
}
return retval;
}
