int mca_pml_ucx_recv(void *buf, size_t count, ompi_datatype_t *datatype, int src,
int tag, struct ompi_communicator_t* comm,
ompi_status_public_t* mpi_status)
{
ucp_tag_t ucp_tag, ucp_tag_mask;
ompi_request_t *req;
PML_UCX_TRACE_RECV("%s", buf, count, datatype, src, tag, comm, "recv");
PML_UCX_MAKE_RECV_TAG(ucp_tag, ucp_tag_mask, tag, src, comm);
req = (ompi_request_t*)ucp_tag_recv_nb(ompi_pml_ucx.ucp_worker, buf, count,
mca_pml_ucx_get_datatype(datatype),
ucp_tag, ucp_tag_mask,
mca_pml_ucx_blocking_recv_completion);
if (UCS_PTR_IS_ERR(req)) {
PML_UCX_ERROR("ucx recv failed: %s", ucs_status_string(UCS_PTR_STATUS(req)));
return OMPI_ERROR;
}
ucp_worker_progress(ompi_pml_ucx.ucp_worker);
while ( !REQUEST_COMPLETE(req) ) {
opal_progress();
}
if (mpi_status != MPI_STATUS_IGNORE) {
*mpi_status = req->req_status;
}
req->req_complete = REQUEST_PENDING;
ucp_request_release(req);
return OMPI_SUCCESS;
}
static void
mca_pml_ucx_blocking_recv_completion(void *request, ucs_status_t status,
ucp_tag_recv_info_t *info)
{
ompi_request_t *req = request;
PML_UCX_VERBOSE(8, "blocking receive request %p completed with status %s tag %"PRIx64" len %zu",
(void*)req, ucs_status_string(status), info->sender_tag,
info->length);
mca_pml_ucx_set_recv_status(&req->req_status, status, info);
PML_UCX_ASSERT( !(REQUEST_COMPLETE(req)));
ompi_request_complete(req,true);
}
static int mca_pml_yalla_send_request_cancel(ompi_request_t *request, int flag)
{
mca_pml_yalla_send_request_t *sreq = (mca_pml_yalla_send_request_t*)request;
mxm_error_t error;
if (REQUEST_COMPLETE(request)) {
/*
* This might be a buffered send request which has completed anyway, so
* we cannot cancel it anymore. Just hope for the best.
*/
PML_YALLA_VERBOSE(7, "not canceling a completed send request %p", (void *)request);
return OMPI_SUCCESS;
}
error = mxm_req_cancel_send(&sreq->mxm);
if ((error != MXM_OK) && (error != MXM_ERR_NO_PROGRESS)) {
PML_YALLA_ERROR("failed to cancel send request %p: %s", (void *)request,
mxm_error_string(error));
return OMPI_ERROR;
}
PML_YALLA_VERBOSE(9, "canceled send request %p", (void *)request);
return OMPI_SUCCESS;
}
int mca_pml_ucx_start(size_t count, ompi_request_t** requests)
{
mca_pml_ucx_persistent_request_t *preq;
ompi_request_t *tmp_req;
size_t i;
for (i = 0; i < count; ++i) {
preq = (mca_pml_ucx_persistent_request_t *)requests[i];
if ((preq == NULL) || (OMPI_REQUEST_PML != preq->ompi.req_type)) {
/* Skip irrelevant requests */
continue;
}
PML_UCX_ASSERT(preq->ompi.req_state != OMPI_REQUEST_INVALID);
preq->ompi.req_state = OMPI_REQUEST_ACTIVE;
mca_pml_ucx_request_reset(&preq->ompi);
if (preq->flags & MCA_PML_UCX_REQUEST_FLAG_SEND) {
/* TODO special care to sync/buffered send */
PML_UCX_VERBOSE(8, "start send request %p", (void*)preq);
tmp_req = (ompi_request_t*)ucp_tag_send_nb(preq->send.ep, preq->buffer,
preq->count, preq->datatype,
preq->tag,
mca_pml_ucx_psend_completion);
} else {
PML_UCX_VERBOSE(8, "start recv request %p", (void*)preq);
tmp_req = (ompi_request_t*)ucp_tag_recv_nb(ompi_pml_ucx.ucp_worker,
preq->buffer, preq->count,
preq->datatype, preq->tag,
preq->recv.tag_mask,
mca_pml_ucx_precv_completion);
}
if (tmp_req == NULL) {
/* Only send can complete immediately */
PML_UCX_ASSERT(preq->flags & MCA_PML_UCX_REQUEST_FLAG_SEND);
PML_UCX_VERBOSE(8, "send completed immediately, completing persistent request %p",
(void*)preq);
mca_pml_ucx_set_send_status(&preq->ompi.req_status, UCS_OK);
ompi_request_complete(&preq->ompi, true);
} else if (!UCS_PTR_IS_ERR(tmp_req)) {
if (REQUEST_COMPLETE(tmp_req)) {
/* tmp_req is already completed */
PML_UCX_VERBOSE(8, "completing persistent request %p", (void*)preq);
mca_pml_ucx_persistent_request_complete(preq, tmp_req);
} else {
/* tmp_req would be completed by callback and trigger completion
* of preq */
PML_UCX_VERBOSE(8, "temporary request %p will complete persistent request %p",
(void*)tmp_req, (void*)preq);
tmp_req->req_complete_cb_data = preq;
preq->tmp_req = tmp_req;
}
} else {
PML_UCX_ERROR("ucx %s failed: %s",
(preq->flags & MCA_PML_UCX_REQUEST_FLAG_SEND) ? "send" : "recv",
ucs_status_string(UCS_PTR_STATUS(tmp_req)));
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
int ompi_request_default_wait_any(size_t count,
ompi_request_t ** requests,
int *index,
ompi_status_public_t * status)
{
size_t i, completed = count, num_requests_null_inactive = 0;
int rc = OMPI_SUCCESS;
ompi_request_t *request=NULL;
ompi_wait_sync_t sync;
if (OPAL_UNLIKELY(0 == count)) {
*index = MPI_UNDEFINED;
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, 1);
num_requests_null_inactive = 0;
for (i = 0; i < count; i++) {
void *_tmp_ptr = REQUEST_PENDING;
request = requests[i];
/* Check for null or completed persistent request. For
* MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync) ) {
assert(REQUEST_COMPLETE(request));
completed = i;
*index = i;
goto after_sync_wait;
}
}
if(num_requests_null_inactive == count) {
*index = MPI_UNDEFINED;
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
/* No signal-in-flight can be in this case */
WAIT_SYNC_RELEASE_NOWAIT(&sync);
return rc;
}
SYNC_WAIT(&sync);
after_sync_wait:
/* recheck the complete status and clean up the sync primitives.
* Do it backward to return the earliest complete request to the
* user.
*/
for(i = completed-1; (i+1) > 0; i--) {
void *tmp_ptr = &sync;
request = requests[i];
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
continue;
}
/* Atomically mark the request as pending. If this succeed then
* the request was not completed, and it is now marked as pending.
* Otherwise, the request has been completed meanwhile, and it
* has been atomically marked as REQUEST_COMPLETE.
*/
if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &tmp_ptr, REQUEST_PENDING) ) {
*index = i;
}
}
if( *index == (int)completed ) {
/* Only one request has triggered. There was no in-flight
* completions. Drop the signalled flag so we won't block
* in WAIT_SYNC_RELEASE
*/
WAIT_SYNC_SIGNALLED(&sync);
}
request = requests[*index];
assert( REQUEST_COMPLETE(request) );
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled ) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
#endif
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set status->MPI_ERROR here! See MPI-1.1 doc,
sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
rc = request->req_status.MPI_ERROR;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
/* If there's an error while freeing the request,
assume that the request is still there. Otherwise,
Bad Things will happen later! */
rc = ompi_request_free(&requests[*index]);
}
WAIT_SYNC_RELEASE(&sync);
return rc;
}
int ompi_request_default_wait_some(size_t count,
ompi_request_t ** requests,
int * outcount,
int * indices,
ompi_status_public_t * statuses)
{
size_t num_requests_null_inactive=0, num_requests_done=0;
int rc = MPI_SUCCESS;
ompi_request_t **rptr = NULL;
ompi_request_t *request = NULL;
ompi_wait_sync_t sync;
size_t sync_sets = 0, sync_unsets = 0;
if (OPAL_UNLIKELY(0 == count)) {
*outcount = MPI_UNDEFINED;
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, 1);
*outcount = 0;
rptr = requests;
num_requests_null_inactive = 0;
num_requests_done = 0;
for (size_t i = 0; i < count; i++, rptr++) {
void *_tmp_ptr = REQUEST_PENDING;
request = *rptr;
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
indices[i] = OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync);
if( !indices[i] ) {
/* If the request is completed go ahead and mark it as such */
assert( REQUEST_COMPLETE(request) );
num_requests_done++;
}
}
sync_sets = count - num_requests_null_inactive - num_requests_done;
if(num_requests_null_inactive == count) {
*outcount = MPI_UNDEFINED;
/* nobody will signall us */
WAIT_SYNC_RELEASE_NOWAIT(&sync);
return rc;
}
if( 0 == num_requests_done ) {
/* One completed request is enough to satisfy the some condition */
SYNC_WAIT(&sync);
}
/* Do the final counting and */
/* Clean up the synchronization primitives */
rptr = requests;
num_requests_done = 0;
for (size_t i = 0; i < count; i++, rptr++) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
continue;
}
/* Here we have 3 possibilities:
* a) request was found completed in the first loop
* => ( indices[i] == 0 )
* b) request was completed between first loop and this check
* => ( indices[i] == 1 ) and we can NOT atomically mark the
* request as pending.
* c) request wasn't finished yet
* => ( indices[i] == 1 ) and we CAN atomically mark the
* request as pending.
* NOTE that in any case (i >= num_requests_done) as latter grows
* either slowly (in case of partial completion)
* OR in parallel with `i` (in case of full set completion)
*/
if( !indices[i] ){
indices[num_requests_done++] = i;
} else if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING) ) {
indices[num_requests_done++] = i;
}
}
sync_unsets = count - num_requests_null_inactive - num_requests_done;
if( sync_sets == sync_unsets ){
/* nobody knows about us,
* set signa-in-progress flag to false
*/
WAIT_SYNC_SIGNALLED(&sync);
}
WAIT_SYNC_RELEASE(&sync);
*outcount = num_requests_done;
for (size_t i = 0; i < num_requests_done; i++) {
request = requests[indices[i]];
assert( REQUEST_COMPLETE(request) );
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
#endif
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUSES_IGNORE != statuses) {
statuses[i] = request->req_status;
}
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
rc = MPI_ERR_IN_STATUS;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else {
/* Only free the request if there was no error */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp;
tmp = ompi_request_free(&(requests[indices[i]]));
if (OMPI_SUCCESS != tmp) {
return tmp;
}
}
}
}
return rc;
}
int ompi_request_default_wait_all( size_t count,
ompi_request_t ** requests,
ompi_status_public_t * statuses )
{
size_t i, completed = 0, failed = 0;
ompi_request_t **rptr;
ompi_request_t *request;
int mpi_error = OMPI_SUCCESS;
ompi_wait_sync_t sync;
if (OPAL_UNLIKELY(0 == count)) {
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, count);
rptr = requests;
for (i = 0; i < count; i++) {
void *_tmp_ptr = REQUEST_PENDING;
request = *rptr++;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
completed++;
continue;
}
if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync)) {
if( OPAL_UNLIKELY( MPI_SUCCESS != request->req_status.MPI_ERROR ) ) {
failed++;
}
completed++;
}
}
if( failed > 0 ) {
goto finish;
}
if( 0 != completed ) {
wait_sync_update(&sync, completed, OPAL_SUCCESS);
}
/* wait until all requests complete or until an error is triggered. */
mpi_error = SYNC_WAIT(&sync);
if( OPAL_SUCCESS != mpi_error ) {
/* if we are in an error case, increase the failed to ensure
proper cleanup during the requests completion. */
failed++;
}
finish:
rptr = requests;
if (MPI_STATUSES_IGNORE != statuses) {
/* fill out status and free request if required */
for( i = 0; i < count; i++, rptr++ ) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
statuses[i] = ompi_status_empty;
continue;
}
if( OPAL_UNLIKELY(0 < failed) ) {
/* if we have failed requests we skipped the waiting on the sync. Thus,
* some of the requests might not be properly completed, in which case
* we must detach all requests from the sync. However, if we can succesfully
* mark the request as pending then it is neither failed nor complete, and
* we must stop altering it.
*/
if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
/*
* Per MPI 2.2 p 60:
* Allows requests to be marked as MPI_ERR_PENDING if they are
* "neither failed nor completed." Which can only happen if
* there was an error in one of the other requests.
*/
statuses[i].MPI_ERROR = MPI_ERR_PENDING;
mpi_error = MPI_ERR_IN_STATUS;
continue;
}
}
assert( REQUEST_COMPLETE(request) );
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
statuses[i] = request->req_status;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
continue;
}
/* Only free the request if there is no error on it */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
/* If there's an error while freeing the request,
assume that the request is still there.
Otherwise, Bad Things will happen later! */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
if( statuses[i].MPI_ERROR != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
} else {
int rc;
/* free request if required */
for( i = 0; i < count; i++, rptr++ ) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
rc = ompi_status_empty.MPI_ERROR;
goto absorb_error_and_continue;
}
/*
* Assert only if no requests were failed.
* Since some may still be pending.
*/
if( OPAL_UNLIKELY(0 < failed) ) {
/* If the request is still pending due to a failed request
* then skip it in this loop.
*/
if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
/*
* Per MPI 2.2 p 60:
* Allows requests to be marked as MPI_ERR_PENDING if they are
* "neither failed nor completed." Which can only happen if
* there was an error in one of the other requests.
*/
rc = MPI_ERR_PENDING;
goto absorb_error_and_continue;
}
}
assert( REQUEST_COMPLETE(request) );
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
/* Per note above, we have to call gen request query_fn
even if STATUSES_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
rc = request->req_status.MPI_ERROR;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
absorb_error_and_continue:
/*
* Per MPI 2.2 p34:
* "It is possible for an MPI function to return MPI_ERR_IN_STATUS
* even when MPI_STATUS_IGNORE or MPI_STATUSES_IGNORE has been
* passed to that function."
* So we should do so here as well.
*/
if( OMPI_SUCCESS == mpi_error && rc != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
}
WAIT_SYNC_RELEASE(&sync);
return mpi_error;
}
/* progresses a request
*
* to be called *only* from the progress thread !!! */
int NBC_Progress(NBC_Handle *handle) {
int res, ret=NBC_CONTINUE;
bool flag;
unsigned long size = 0;
char *delim;
if (handle->nbc_complete) {
return NBC_OK;
}
flag = true;
if ((handle->req_count > 0) && (handle->req_array != NULL)) {
NBC_DEBUG(50, "NBC_Progress: testing for %i requests\n", handle->req_count);
#ifdef NBC_TIMING
Test_time -= MPI_Wtime();
#endif
/* don't call ompi_request_test_all as it causes a recursive call into opal_progress */
while (handle->req_count) {
ompi_request_t *subreq = handle->req_array[handle->req_count - 1];
if (REQUEST_COMPLETE(subreq)) {
if(OPAL_UNLIKELY( OMPI_SUCCESS != subreq->req_status.MPI_ERROR )) {
NBC_Error ("MPI Error in NBC subrequest %p : %d", subreq, subreq->req_status.MPI_ERROR);
/* copy the error code from the underlying request and let the
* round finish */
handle->super.req_status.MPI_ERROR = subreq->req_status.MPI_ERROR;
}
handle->req_count--;
ompi_request_free(&subreq);
} else {
flag = false;
break;
}
}
#ifdef NBC_TIMING
Test_time += MPI_Wtime();
#endif
}
/* a round is finished */
if (flag) {
/* reset handle for next round */
if (NULL != handle->req_array) {
/* free request array */
free (handle->req_array);
handle->req_array = NULL;
}
handle->req_count = 0;
/* previous round had an error */
if (OPAL_UNLIKELY(OMPI_SUCCESS != handle->super.req_status.MPI_ERROR)) {
res = handle->super.req_status.MPI_ERROR;
NBC_Error("NBC_Progress: an error %d was found during schedule %p at row-offset %li - aborting the schedule\n", res, handle->schedule, handle->row_offset);
handle->nbc_complete = true;
if (!handle->super.req_persistent) {
NBC_Free(handle);
}
return res;
}
/* adjust delim to start of current round */
NBC_DEBUG(5, "NBC_Progress: going in schedule %p to row-offset: %li\n", handle->schedule, handle->row_offset);
delim = handle->schedule->data + handle->row_offset;
NBC_DEBUG(10, "delim: %p\n", delim);
nbc_get_round_size(delim, &size);
NBC_DEBUG(10, "size: %li\n", size);
/* adjust delim to end of current round -> delimiter */
delim = delim + size;
if (*delim == 0) {
/* this was the last round - we're done */
NBC_DEBUG(5, "NBC_Progress last round finished - we're done\n");
handle->nbc_complete = true;
if (!handle->super.req_persistent) {
NBC_Free(handle);
}
return NBC_OK;
}
NBC_DEBUG(5, "NBC_Progress round finished - goto next round\n");
/* move delim to start of next round */
/* initializing handle for new virgin round */
handle->row_offset = (intptr_t) (delim + 1) - (intptr_t) handle->schedule->data;
/* kick it off */
res = NBC_Start_round(handle);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Error ("Error in NBC_Start_round() (%i)", res);
return res;
}
}
return ret;
}
