int ompi_request_default_wait(
ompi_request_t ** req_ptr,
ompi_status_public_t * status)
{
ompi_request_t *req = *req_ptr;
ompi_request_wait_completion(req);
#if OPAL_ENABLE_FT_CR == 1
OMPI_CRCP_REQUEST_COMPLETE(req);
#endif
/* return status. If it's a generalized request, we *have* to
invoke the query_fn, even if the user procided STATUS_IGNORE.
MPI-2:8.2. */
if (OMPI_REQUEST_GEN == req->req_type) {
ompi_grequest_invoke_query(req, &req->req_status);
}
if( MPI_STATUS_IGNORE != status ) {
/* Do *NOT* set status->MPI_ERROR here! See MPI-1.1 doc, sec
3.2.5, p.22 */
status->MPI_TAG = req->req_status.MPI_TAG;
status->MPI_SOURCE = req->req_status.MPI_SOURCE;
status->_ucount = req->req_status._ucount;
status->_cancelled = req->req_status._cancelled;
}
if( req->req_persistent ) {
if( req->req_state == OMPI_REQUEST_INACTIVE ) {
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
return OMPI_SUCCESS;
}
req->req_state = OMPI_REQUEST_INACTIVE;
return req->req_status.MPI_ERROR;
}
/* If there was an error, don't free the request -- just return
the single error. */
if (MPI_SUCCESS != req->req_status.MPI_ERROR) {
return req->req_status.MPI_ERROR;
}
/* If there's an error while freeing the request, assume that the
request is still there. Otherwise, Bad Things will happen
later! */
return ompi_request_free(req_ptr);
}
int ompi_request_default_test_any(
size_t count,
ompi_request_t ** requests,
int *index,
int *completed,
ompi_status_public_t * status)
{
size_t i;
size_t num_requests_null_inactive = 0;
ompi_request_t **rptr;
ompi_request_t *request;
opal_atomic_mb();
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if( request->req_complete ) {
OMPI_CRCP_REQUEST_COMPLETE(request);
*index = i;
*completed = true;
/* MPI 2:8.2 says that generalized requests always have
the query function invoked in TEST* / WAIT*
(#@$%@#$%!!! Would have been simpler to call it in
GREQUEST_COMPLETE!), even if the user passed in
STATUS_IGNORE */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set a new value for status->MPI_ERROR
here! See MPI-1.1 doc, sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
} else if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set a new value for status->MPI_ERROR
here! See MPI-1.1 doc, sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
return OMPI_SUCCESS;
}
/* If there is an error on the request, don't free it */
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
return request->req_status.MPI_ERROR;
}
/* If there's an error while freeing the request, assume
that the request is still there. Otherwise, Bad Things
will happen later! */
return ompi_request_free(rptr);
}
}
/* Only fall through here if we found nothing */
*index = MPI_UNDEFINED;
if(num_requests_null_inactive != count) {
*completed = false;
#if OMPI_ENABLE_PROGRESS_THREADS == 0
opal_progress();
#endif
} else {
*completed = true;
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
}
return OMPI_SUCCESS;
}
int ompi_request_default_test( ompi_request_t ** rptr,
int *completed,
ompi_status_public_t * status )
{
ompi_request_t *request = *rptr;
#if OMPI_ENABLE_PROGRESS_THREADS == 0
int do_it_once = 0;
recheck_request_status:
#endif
opal_atomic_mb();
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
*completed = true;
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
return OMPI_SUCCESS;
}
if (request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
*completed = true;
/* For a generalized request, we *have* to call the query_fn
if it completes, even if the user provided
STATUS_IGNORE. See MPI-2:8.2. */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
if (MPI_STATUS_IGNORE != status) {
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
} else if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set a new value for status->MPI_ERROR here!
See MPI-1.1 doc, sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
return request->req_status.MPI_ERROR;
}
/* If there was an error, don't free the request -- just
return the single error. */
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
return request->req_status.MPI_ERROR;
}
/* If there's an error on the request, assume that the request
is still there. Otherwise, Bad Things will happen
later! */
return ompi_request_free(rptr);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
if( 0 == do_it_once ) {
/**
* If we run the opal_progress then check the status of the request before
* leaving. We will call the opal_progress only once per call.
*/
opal_progress();
do_it_once++;
goto recheck_request_status;
}
#endif
*completed = false;
return OMPI_SUCCESS;
}
int ompi_request_default_test_some(
size_t count,
ompi_request_t ** requests,
int * outcount,
int * indices,
ompi_status_public_t * statuses)
{
size_t i, num_requests_null_inactive=0, num_requests_done = 0;
int rc = OMPI_SUCCESS;
ompi_request_t **rptr;
ompi_request_t *request;
opal_atomic_mb();
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if (request->req_state == OMPI_REQUEST_INACTIVE) {
num_requests_null_inactive++;
continue;
}
if (true == request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
indices[num_requests_done++] = i;
}
}
/*
* If there are no active requests, no need to progress
*/
if (num_requests_null_inactive == count) {
*outcount = MPI_UNDEFINED;
return OMPI_SUCCESS;
}
*outcount = num_requests_done;
if (num_requests_done == 0) {
#if OMPI_ENABLE_PROGRESS_THREADS == 0
opal_progress();
#endif
return OMPI_SUCCESS;
}
/* fill out completion status and free request if required */
for( i = 0; i < num_requests_done; i++) {
request = requests[indices[i]];
/* See note above: if a generalized request completes, we
*have* to call the query fn, even if STATUSES_IGNORE
was supplied */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUSES_IGNORE != statuses) {
statuses[i] = request->req_status;
}
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
rc = MPI_ERR_IN_STATUS;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else {
/* Only free the request if there was no error */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp;
tmp = ompi_request_free(&(requests[indices[i]]));
if (OMPI_SUCCESS != tmp) {
return tmp;
}
}
}
}
return rc;
}
int ompi_request_default_test_all(
size_t count,
ompi_request_t ** requests,
int *completed,
ompi_status_public_t * statuses)
{
size_t i, rc;
ompi_request_t **rptr;
size_t num_completed = 0;
ompi_request_t *request;
opal_atomic_mb();
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ||
request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
num_completed++;
}
}
if (num_completed != count) {
*completed = false;
#if OMPI_ENABLE_PROGRESS_THREADS == 0
opal_progress();
#endif
return OMPI_SUCCESS;
}
rptr = requests;
*completed = true;
rc = MPI_SUCCESS;
if (MPI_STATUSES_IGNORE != statuses) {
/* fill out completion status and free request if required */
for( i = 0; i < count; i++, rptr++ ) {
request = *rptr;
/* If the request is OMPI_REQUEST_INACTIVE set the status
* to ompi_status_empty.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
statuses[i] = ompi_status_empty;
continue;
}
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
statuses[i] = request->req_status;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
continue;
}
/* MPI-2:4.5.1 says that we can return MPI_ERR_IN_STATUS
even if MPI_STATUSES_IGNORE was used. Woot! */
/* Only free the request if there was no error on it */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp = ompi_request_free(rptr);
if (tmp != OMPI_SUCCESS) {
return tmp;
}
} else {
rc = MPI_ERR_IN_STATUS;
}
}
} else {
/* free request if required */
for( i = 0; i < count; i++, rptr++ ) {
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE) {
continue;
}
/* See note above: if a generalized request completes, we
*have* to call the query fn, even if STATUSES_IGNORE
was supplied */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
continue;
}
/* Only free the request if there was no error */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp = ompi_request_free(rptr);
if (tmp != OMPI_SUCCESS) {
return tmp;
}
} else {
rc = MPI_ERR_IN_STATUS;
}
}
}
return rc;
}
int ompi_request_default_wait_any(size_t count,
ompi_request_t ** requests,
int *index,
ompi_status_public_t * status)
{
size_t i, completed = count, num_requests_null_inactive = 0;
int rc = OMPI_SUCCESS;
ompi_request_t *request=NULL;
ompi_wait_sync_t sync;
if (OPAL_UNLIKELY(0 == count)) {
*index = MPI_UNDEFINED;
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, 1);
num_requests_null_inactive = 0;
for (i = 0; i < count; i++) {
void *_tmp_ptr = REQUEST_PENDING;
request = requests[i];
/* Check for null or completed persistent request. For
* MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync) ) {
assert(REQUEST_COMPLETE(request));
completed = i;
*index = i;
goto after_sync_wait;
}
}
if(num_requests_null_inactive == count) {
*index = MPI_UNDEFINED;
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
/* No signal-in-flight can be in this case */
WAIT_SYNC_RELEASE_NOWAIT(&sync);
return rc;
}
SYNC_WAIT(&sync);
after_sync_wait:
/* recheck the complete status and clean up the sync primitives.
* Do it backward to return the earliest complete request to the
* user.
*/
for(i = completed-1; (i+1) > 0; i--) {
void *tmp_ptr = &sync;
request = requests[i];
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
continue;
}
/* Atomically mark the request as pending. If this succeed then
* the request was not completed, and it is now marked as pending.
* Otherwise, the request has been completed meanwhile, and it
* has been atomically marked as REQUEST_COMPLETE.
*/
if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &tmp_ptr, REQUEST_PENDING) ) {
*index = i;
}
}
if( *index == (int)completed ) {
/* Only one request has triggered. There was no in-flight
* completions. Drop the signalled flag so we won't block
* in WAIT_SYNC_RELEASE
*/
WAIT_SYNC_SIGNALLED(&sync);
}
request = requests[*index];
assert( REQUEST_COMPLETE(request) );
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled ) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
#endif
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set status->MPI_ERROR here! See MPI-1.1 doc,
sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
rc = request->req_status.MPI_ERROR;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
/* If there's an error while freeing the request,
assume that the request is still there. Otherwise,
Bad Things will happen later! */
rc = ompi_request_free(&requests[*index]);
}
WAIT_SYNC_RELEASE(&sync);
return rc;
}
int ompi_request_default_wait_some(size_t count,
ompi_request_t ** requests,
int * outcount,
int * indices,
ompi_status_public_t * statuses)
{
size_t num_requests_null_inactive=0, num_requests_done=0;
int rc = MPI_SUCCESS;
ompi_request_t **rptr = NULL;
ompi_request_t *request = NULL;
ompi_wait_sync_t sync;
size_t sync_sets = 0, sync_unsets = 0;
if (OPAL_UNLIKELY(0 == count)) {
*outcount = MPI_UNDEFINED;
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, 1);
*outcount = 0;
rptr = requests;
num_requests_null_inactive = 0;
num_requests_done = 0;
for (size_t i = 0; i < count; i++, rptr++) {
void *_tmp_ptr = REQUEST_PENDING;
request = *rptr;
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
indices[i] = OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync);
if( !indices[i] ) {
/* If the request is completed go ahead and mark it as such */
assert( REQUEST_COMPLETE(request) );
num_requests_done++;
}
}
sync_sets = count - num_requests_null_inactive - num_requests_done;
if(num_requests_null_inactive == count) {
*outcount = MPI_UNDEFINED;
/* nobody will signall us */
WAIT_SYNC_RELEASE_NOWAIT(&sync);
return rc;
}
if( 0 == num_requests_done ) {
/* One completed request is enough to satisfy the some condition */
SYNC_WAIT(&sync);
}
/* Do the final counting and */
/* Clean up the synchronization primitives */
rptr = requests;
num_requests_done = 0;
for (size_t i = 0; i < count; i++, rptr++) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
continue;
}
/* Here we have 3 possibilities:
* a) request was found completed in the first loop
* => ( indices[i] == 0 )
* b) request was completed between first loop and this check
* => ( indices[i] == 1 ) and we can NOT atomically mark the
* request as pending.
* c) request wasn't finished yet
* => ( indices[i] == 1 ) and we CAN atomically mark the
* request as pending.
* NOTE that in any case (i >= num_requests_done) as latter grows
* either slowly (in case of partial completion)
* OR in parallel with `i` (in case of full set completion)
*/
if( !indices[i] ){
indices[num_requests_done++] = i;
} else if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING) ) {
indices[num_requests_done++] = i;
}
}
sync_unsets = count - num_requests_null_inactive - num_requests_done;
if( sync_sets == sync_unsets ){
/* nobody knows about us,
* set signa-in-progress flag to false
*/
WAIT_SYNC_SIGNALLED(&sync);
}
WAIT_SYNC_RELEASE(&sync);
*outcount = num_requests_done;
for (size_t i = 0; i < num_requests_done; i++) {
request = requests[indices[i]];
assert( REQUEST_COMPLETE(request) );
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
#endif
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUSES_IGNORE != statuses) {
statuses[i] = request->req_status;
}
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
rc = MPI_ERR_IN_STATUS;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else {
/* Only free the request if there was no error */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp;
tmp = ompi_request_free(&(requests[indices[i]]));
if (OMPI_SUCCESS != tmp) {
return tmp;
}
}
}
}
return rc;
}
int ompi_request_default_wait_all( size_t count,
ompi_request_t ** requests,
ompi_status_public_t * statuses )
{
size_t i, completed = 0, failed = 0;
ompi_request_t **rptr;
ompi_request_t *request;
int mpi_error = OMPI_SUCCESS;
ompi_wait_sync_t sync;
if (OPAL_UNLIKELY(0 == count)) {
return OMPI_SUCCESS;
}
WAIT_SYNC_INIT(&sync, count);
rptr = requests;
for (i = 0; i < count; i++) {
void *_tmp_ptr = REQUEST_PENDING;
request = *rptr++;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
completed++;
continue;
}
if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync)) {
if( OPAL_UNLIKELY( MPI_SUCCESS != request->req_status.MPI_ERROR ) ) {
failed++;
}
completed++;
}
}
if( failed > 0 ) {
goto finish;
}
if( 0 != completed ) {
wait_sync_update(&sync, completed, OPAL_SUCCESS);
}
/* wait until all requests complete or until an error is triggered. */
mpi_error = SYNC_WAIT(&sync);
if( OPAL_SUCCESS != mpi_error ) {
/* if we are in an error case, increase the failed to ensure
proper cleanup during the requests completion. */
failed++;
}
finish:
rptr = requests;
if (MPI_STATUSES_IGNORE != statuses) {
/* fill out status and free request if required */
for( i = 0; i < count; i++, rptr++ ) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
statuses[i] = ompi_status_empty;
continue;
}
if( OPAL_UNLIKELY(0 < failed) ) {
/* if we have failed requests we skipped the waiting on the sync. Thus,
* some of the requests might not be properly completed, in which case
* we must detach all requests from the sync. However, if we can succesfully
* mark the request as pending then it is neither failed nor complete, and
* we must stop altering it.
*/
if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
/*
* Per MPI 2.2 p 60:
* Allows requests to be marked as MPI_ERR_PENDING if they are
* "neither failed nor completed." Which can only happen if
* there was an error in one of the other requests.
*/
statuses[i].MPI_ERROR = MPI_ERR_PENDING;
mpi_error = MPI_ERR_IN_STATUS;
continue;
}
}
assert( REQUEST_COMPLETE(request) );
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
statuses[i] = request->req_status;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
continue;
}
/* Only free the request if there is no error on it */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
/* If there's an error while freeing the request,
assume that the request is still there.
Otherwise, Bad Things will happen later! */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
if( statuses[i].MPI_ERROR != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
} else {
int rc;
/* free request if required */
for( i = 0; i < count; i++, rptr++ ) {
void *_tmp_ptr = &sync;
request = *rptr;
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
rc = ompi_status_empty.MPI_ERROR;
goto absorb_error_and_continue;
}
/*
* Assert only if no requests were failed.
* Since some may still be pending.
*/
if( OPAL_UNLIKELY(0 < failed) ) {
/* If the request is still pending due to a failed request
* then skip it in this loop.
*/
if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
/*
* Per MPI 2.2 p 60:
* Allows requests to be marked as MPI_ERR_PENDING if they are
* "neither failed nor completed." Which can only happen if
* there was an error in one of the other requests.
*/
rc = MPI_ERR_PENDING;
goto absorb_error_and_continue;
}
}
assert( REQUEST_COMPLETE(request) );
if( opal_cr_is_enabled) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
/* Per note above, we have to call gen request query_fn
even if STATUSES_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
rc = request->req_status.MPI_ERROR;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
absorb_error_and_continue:
/*
* Per MPI 2.2 p34:
* "It is possible for an MPI function to return MPI_ERR_IN_STATUS
* even when MPI_STATUS_IGNORE or MPI_STATUSES_IGNORE has been
* passed to that function."
* So we should do so here as well.
*/
if( OMPI_SUCCESS == mpi_error && rc != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
}
WAIT_SYNC_RELEASE(&sync);
return mpi_error;
}
int ompi_request_default_wait_any(
size_t count,
ompi_request_t ** requests,
int *index,
ompi_status_public_t * status)
{
#if OMPI_ENABLE_PROGRESS_THREADS
int c;
#endif
size_t i=0, num_requests_null_inactive=0;
int rc = OMPI_SUCCESS;
int completed = -1;
ompi_request_t **rptr=NULL;
ompi_request_t *request=NULL;
#if OMPI_ENABLE_PROGRESS_THREADS
/* poll for completion */
OPAL_THREAD_ADD32(&ompi_progress_thread_count,1);
for (c = 0; completed < 0 && c < opal_progress_spin_count; c++) {
rptr = requests;
num_requests_null_inactive = 0;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if (true == request->req_complete) {
completed = i;
OPAL_THREAD_ADD32(&ompi_progress_thread_count,-1);
goto finished;
}
}
if( num_requests_null_inactive == count ) {
OPAL_THREAD_ADD32(&ompi_progress_thread_count,-1);
goto finished;
}
opal_progress();
}
OPAL_THREAD_ADD32(&ompi_progress_thread_count,-1);
#endif
/* give up and sleep until completion */
OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request_waiting++;
do {
rptr = requests;
num_requests_null_inactive = 0;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
/* Sanity test */
if( NULL == request) {
continue;
}
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if (request->req_complete == true) {
completed = i;
break;
}
}
if(num_requests_null_inactive == count)
break;
if (completed < 0) {
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
}
} while (completed < 0);
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
#if OMPI_ENABLE_PROGRESS_THREADS
finished:
#endif /* OMPI_ENABLE_PROGRESS_THREADS */
if(num_requests_null_inactive == count) {
*index = MPI_UNDEFINED;
if (MPI_STATUS_IGNORE != status) {
*status = ompi_status_empty;
}
} else {
assert( true == request->req_complete );
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUS_IGNORE != status) {
/* Do *NOT* set status->MPI_ERROR here! See MPI-1.1 doc,
sec 3.2.5, p.22 */
int old_error = status->MPI_ERROR;
*status = request->req_status;
status->MPI_ERROR = old_error;
}
rc = request->req_status.MPI_ERROR;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
/* If there's an error while freeing the request,
assume that the request is still there. Otherwise,
Bad Things will happen later! */
rc = ompi_request_free(rptr);
}
*index = completed;
}
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled) {
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( true == request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
}
}
#endif
return rc;
}
int ompi_request_default_wait_some(
size_t count,
ompi_request_t ** requests,
int * outcount,
int * indices,
ompi_status_public_t * statuses)
{
#if OMPI_ENABLE_PROGRESS_THREADS
int c;
#endif
size_t i, num_requests_null_inactive=0, num_requests_done=0;
int rc = MPI_SUCCESS;
ompi_request_t **rptr=NULL;
ompi_request_t *request=NULL;
*outcount = 0;
for (i = 0; i < count; i++) {
indices[i] = 0;
}
#if OMPI_ENABLE_PROGRESS_THREADS
/* poll for completion */
OPAL_THREAD_ADD32(&ompi_progress_thread_count,1);
for (c = 0; c < opal_progress_spin_count; c++) {
rptr = requests;
num_requests_null_inactive = 0;
num_requests_done = 0;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE
*/
if (request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if (true == request->req_complete) {
indices[i] = 1;
num_requests_done++;
}
}
if (num_requests_null_inactive == count ||
num_requests_done > 0) {
OPAL_THREAD_ADD32(&ompi_progress_thread_count,-1);
goto finished;
}
opal_progress();
}
OPAL_THREAD_ADD32(&ompi_progress_thread_count,-1);
#endif
/*
* We only get here when outcount still is 0.
* give up and sleep until completion
*/
OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request_waiting++;
do {
rptr = requests;
num_requests_null_inactive = 0;
num_requests_done = 0;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
/*
* Check for null or completed persistent request.
* For MPI_REQUEST_NULL, the req_state is always OMPI_REQUEST_INACTIVE.
*/
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
num_requests_null_inactive++;
continue;
}
if (request->req_complete == true) {
indices[i] = 1;
num_requests_done++;
}
}
if (num_requests_null_inactive == count ||
num_requests_done > 0)
break;
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
} while (1);
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
#if OMPI_ENABLE_PROGRESS_THREADS
finished:
#endif /* OMPI_ENABLE_PROGRESS_THREADS */
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled) {
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( true == request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
}
}
#endif
if(num_requests_null_inactive == count) {
*outcount = MPI_UNDEFINED;
} else {
/*
* Compress the index array.
*/
for (i = 0, num_requests_done = 0; i < count; i++) {
if (0 != indices[i]) {
indices[num_requests_done++] = i;
}
}
*outcount = num_requests_done;
for (i = 0; i < num_requests_done; i++) {
request = requests[indices[i]];
assert( true == request->req_complete );
/* Per note above, we have to call gen request query_fn even
if STATUS_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
if (MPI_STATUSES_IGNORE != statuses) {
statuses[i] = request->req_status;
}
if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
rc = MPI_ERR_IN_STATUS;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else {
/* Only free the request if there was no error */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
int tmp;
tmp = ompi_request_free(&(requests[indices[i]]));
if (OMPI_SUCCESS != tmp) {
return tmp;
}
}
}
}
}
return rc;
}
int ompi_request_default_wait_all( size_t count,
ompi_request_t ** requests,
ompi_status_public_t * statuses )
{
size_t completed = 0, i, failed = 0;
ompi_request_t **rptr;
ompi_request_t *request;
int mpi_error = OMPI_SUCCESS;
rptr = requests;
for (i = 0; i < count; i++) {
request = *rptr++;
if (request->req_complete == true) {
if( OPAL_UNLIKELY( MPI_SUCCESS != request->req_status.MPI_ERROR ) ) {
failed++;
}
completed++;
}
}
if( failed > 0 ) {
goto finish;
}
/* if all requests have not completed -- defer acquiring lock
* unless required
*/
if (completed != count) {
/*
* acquire lock and test for completion - if all requests are
* not completed pend on condition variable until a request
* completes
*/
OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request_waiting++;
#if OPAL_ENABLE_MULTI_THREADS
/*
* confirm the status of the pending requests. We have to do it before
* taking the condition or otherwise we can miss some requests completion (the
* one that happpens between our initial test and the aquisition of the lock).
*/
rptr = requests;
for( completed = i = 0; i < count; i++ ) {
request = *rptr++;
if (request->req_complete == true) {
if( MPI_SUCCESS != request->req_status.MPI_ERROR ) {
failed++;
}
completed++;
}
}
if( failed > 0 ) {
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
goto finish;
}
#endif /* OPAL_ENABLE_MULTI_THREADS */
while( completed != count ) {
/* check number of pending requests */
size_t start = ompi_request_completed;
size_t pending = count - completed;
size_t start_failed = ompi_request_failed;
/*
* wait until at least pending requests complete
*/
while (pending > ompi_request_completed - start) {
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
/*
* Check for failed requests. If one request fails, then
* this operation completes in error marking the remaining
* requests as PENDING.
*/
if( OPAL_UNLIKELY( 0 < (ompi_request_failed - start_failed) ) ) {
failed += (ompi_request_failed - start_failed);
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
goto finish;
}
}
/*
* confirm that all pending operations have completed.
*/
rptr = requests;
for( failed = completed = i = 0; i < count; i++ ) {
request = *rptr++;
if (request->req_complete == true) {
if( MPI_SUCCESS != request->req_status.MPI_ERROR ) {
failed++;
}
completed++;
}
}
}
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
}
#if OPAL_ENABLE_FT_CR == 1
if( opal_cr_is_enabled) {
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( true == request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
}
}
#endif
finish:
rptr = requests;
if (MPI_STATUSES_IGNORE != statuses) {
/* fill out status and free request if required */
for( i = 0; i < count; i++, rptr++ ) {
request = *rptr;
/*
* Assert only if no requests were failed.
* Since some may still be pending.
*/
if( 0 >= failed ) {
assert( true == request->req_complete );
}
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
statuses[i] = ompi_status_empty;
continue;
}
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
statuses[i] = request->req_status;
/*
* Per MPI 2.2 p 60:
* Allows requests to be marked as MPI_ERR_PENDING if they are
* "neither failed nor completed." Which can only happen if
* there was an error in one of the other requests.
*/
if( OPAL_UNLIKELY(0 < failed) ) {
if( !request->req_complete ) {
statuses[i].MPI_ERROR = MPI_ERR_PENDING;
mpi_error = MPI_ERR_IN_STATUS;
continue;
}
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
continue;
} else {
/* Only free the request if there is no error on it */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
/* If there's an error while freeing the request,
assume that the request is still there.
Otherwise, Bad Things will happen later! */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
}
if( statuses[i].MPI_ERROR != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
} else {
/* free request if required */
for( i = 0; i < count; i++, rptr++ ) {
int rc;
request = *rptr;
/*
* Assert only if no requests were failed.
* Since some may still be pending.
*/
if( 0 >= failed ) {
assert( true == request->req_complete );
} else {
/* If the request is still pending due to a failed request
* then skip it in this loop.
*/
if( !request->req_complete ) {
continue;
}
}
/* Per note above, we have to call gen request query_fn
even if STATUSES_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
rc = ompi_status_empty.MPI_ERROR;
} else {
rc = request->req_status.MPI_ERROR;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS == mpi_error && OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
/*
* Per MPI 2.2 p34:
* "It is possible for an MPI function to return MPI_ERR_IN_STATUS
* even when MPI_STATUS_IGNORE or MPI_STATUSES_IGNORE has been
* passed to that function."
* So we should do so here as well.
*/
if( OMPI_SUCCESS == mpi_error && rc != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
}
return mpi_error;
}
int ompi_request_default_wait_all( size_t count,
ompi_request_t ** requests,
ompi_status_public_t * statuses )
{
size_t completed = 0, i;
ompi_request_t **rptr;
ompi_request_t *request;
int mpi_error = OMPI_SUCCESS;
rptr = requests;
for (i = 0; i < count; i++) {
request = *rptr++;
if (request->req_complete == true) {
completed++;
}
}
/* if all requests have not completed -- defer acquiring lock
* unless required
*/
if (completed != count) {
/*
* acquire lock and test for completion - if all requests are
* not completed pend on condition variable until a request
* completes
*/
OPAL_THREAD_LOCK(&ompi_request_lock);
ompi_request_waiting++;
#if OMPI_HAVE_THREAD_SUPPORT
/*
* confirm the status of the pending requests. We have to do it before
* taking the condition or otherwise we can miss some requests completion (the
* one that happpens between our initial test and the aquisition of the lock).
*/
rptr = requests;
for( completed = i = 0; i < count; i++ ) {
request = *rptr++;
if (request->req_complete == true) {
completed++;
}
}
#endif /* OMPI_HAVE_THREAD_SUPPORT */
while( completed != count ) {
/* check number of pending requests */
size_t start = ompi_request_completed;
size_t pending = count - completed;
/*
* wait until at least pending requests complete
*/
while (pending > ompi_request_completed - start) {
opal_condition_wait(&ompi_request_cond, &ompi_request_lock);
}
/*
* confirm that all pending operations have completed.
*/
rptr = requests;
for( completed = i = 0; i < count; i++ ) {
request = *rptr++;
if (request->req_complete == true) {
completed++;
}
}
}
ompi_request_waiting--;
OPAL_THREAD_UNLOCK(&ompi_request_lock);
}
#if OPAL_ENABLE_FT == 1
if( opal_cr_is_enabled) {
rptr = requests;
for (i = 0; i < count; i++, rptr++) {
request = *rptr;
if( true == request->req_complete) {
OMPI_CRCP_REQUEST_COMPLETE(request);
}
}
}
#endif
rptr = requests;
if (MPI_STATUSES_IGNORE != statuses) {
/* fill out status and free request if required */
for( i = 0; i < count; i++, rptr++ ) {
request = *rptr;
assert( true == request->req_complete );
if (OMPI_REQUEST_GEN == request->req_type) {
ompi_grequest_invoke_query(request, &request->req_status);
}
statuses[i] = request->req_status;
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else {
/* Only free the request if there is no error on it */
if (MPI_SUCCESS == request->req_status.MPI_ERROR) {
/* If there's an error while freeing the request,
assume that the request is still there.
Otherwise, Bad Things will happen later! */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
}
if( statuses[i].MPI_ERROR != OMPI_SUCCESS) {
mpi_error = MPI_ERR_IN_STATUS;
}
}
} else {
/* free request if required */
for( i = 0; i < count; i++, rptr++ ) {
int rc;
request = *rptr;
assert( true == request->req_complete );
/* Per note above, we have to call gen request query_fn
even if STATUSES_IGNORE was provided */
if (OMPI_REQUEST_GEN == request->req_type) {
rc = ompi_grequest_invoke_query(request, &request->req_status);
}
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
rc = ompi_status_empty.MPI_ERROR;
} else {
rc = request->req_status.MPI_ERROR;
}
if( request->req_persistent ) {
request->req_state = OMPI_REQUEST_INACTIVE;
} else if (MPI_SUCCESS == rc) {
/* Only free the request if there is no error on it */
int tmp = ompi_request_free(rptr);
if (OMPI_SUCCESS != tmp) {
mpi_error = tmp;
}
}
if( rc != OMPI_SUCCESS) {
mpi_error = rc;
}
}
}
return mpi_error;
}
