int MPID_nem_ptl_rptl_ptfini(ptl_pt_index_t pt_index)
{
int i;
int ret = PTL_OK;
struct rptl *rptl;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
/* find the right rptl */
for (rptl = rptl_info.rptl_list; rptl && rptl->data.pt != pt_index; rptl = rptl->next);
assert(rptl);
/* free control portals that were created */
if (rptl->control.pt != PTL_PT_ANY) {
for (i = 0; i < rptl_info.world_size * 2; i++) {
ret = PtlMEUnlink(rptl->control.me[i]);
RPTLU_ERR_POP(ret, "Error unlinking control buffers\n");
}
MPIU_Free(rptl->control.me);
}
MPL_DL_DELETE(rptl_info.rptl_list, rptl);
MPIU_Free(rptl);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_RPTL_PTFINI);
return ret;
fn_fail:
goto fn_exit;
}
int MPID_nem_ptl_rptl_drain_eq(int eq_count, ptl_handle_eq_t *eq)
{
int ret = PTL_OK;
ptl_event_t event;
struct rptl_op_pool_segment *op_segment;
int i;
struct rptl_target *target, *t;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_RPTL_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_RPTL_FINALIZE);
for (target = rptl_info.target_list; target; target = target->next) {
while (target->control_op_list || target->data_op_list) {
for (i = 0; i < eq_count; i++) {
/* read and ignore all events */
ret = MPID_nem_ptl_rptl_eqget(eq[i], &event);
if (ret == PTL_EQ_EMPTY)
ret = PTL_OK;
RPTLU_ERR_POP(ret, "Error calling MPID_nem_ptl_rptl_eqget\n");
}
}
}
for (target = rptl_info.target_list; target;) {
assert(target->data_op_list == NULL);
assert(target->control_op_list == NULL);
while (target->op_segment_list) {
op_segment = target->op_segment_list;
MPL_DL_DELETE(target->op_segment_list, op_segment);
MPIU_Free(op_segment);
}
t = target->next;
MPIU_Free(target);
target = t;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_RPTL_FINALIZE);
return ret;
fn_fail:
goto fn_exit;
}
int MPIR_T_pvar_handle_free_impl(MPI_T_pvar_session session, MPI_T_pvar_handle *handle)
{
int mpi_errno = MPI_SUCCESS;
MPIR_T_pvar_handle_t *hnd = *handle;
MPL_DL_DELETE(session->hlist, hnd);
/* Unlink handle from pvar if it is a watermark */
if (MPIR_T_pvar_is_watermark(hnd)) {
MPIR_T_pvar_watermark_t *mark = (MPIR_T_pvar_watermark_t *)hnd->addr;
if (MPIR_T_pvar_is_first(hnd)) {
mark->first_used = 0;
mark->first_started = 0;
} else {
MPIU_Assert(mark->hlist);
if (mark->hlist == hnd) {
/* hnd happens to be the head */
mark->hlist = hnd->next2;
if (mark->hlist != NULL)
mark->hlist->prev2 = mark->hlist;
} else {
hnd->prev2->next2 = hnd->next2;
if (hnd->next2 != NULL)
hnd->next2->prev2 = hnd->prev2;
}
}
}
MPL_free(hnd);
*handle = MPI_T_PVAR_HANDLE_NULL;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Win_free(MPIR_Win ** win_ptr)
{
int mpi_errno = MPI_SUCCESS;
int in_use;
MPIR_Comm *comm_ptr;
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_WIN_FREE);
MPIR_FUNC_VERBOSE_RMA_ENTER(MPID_STATE_MPID_WIN_FREE);
MPIR_ERR_CHKANDJUMP(((*win_ptr)->states.access_state != MPIDI_RMA_NONE &&
(*win_ptr)->states.access_state != MPIDI_RMA_FENCE_ISSUED &&
(*win_ptr)->states.access_state != MPIDI_RMA_FENCE_GRANTED) ||
((*win_ptr)->states.exposure_state != MPIDI_RMA_NONE),
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
/* 1. Here we must wait until all passive locks are released on this target,
* because for some UNLOCK messages, we do not send ACK back to origin,
* we must wait until lock is released so that we can free window.
* 2. We also need to wait until AT completion counter being zero, because
* this counter is increment everytime we meet a GET-like operation, it is
* possible that when target entering Win_free, passive epoch is not finished
* yet and there are still GETs doing on this target.
* 3. We also need to wait until lock queue becomes empty. It is possible
* that some lock requests is still waiting in the queue when target is
* entering Win_free. */
while ((*win_ptr)->current_lock_type != MPID_LOCK_NONE ||
(*win_ptr)->at_completion_counter != 0 ||
(*win_ptr)->target_lock_queue_head != NULL ||
(*win_ptr)->current_target_lock_data_bytes != 0 || (*win_ptr)->sync_request_cnt != 0) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
mpi_errno = MPIR_Barrier_impl((*win_ptr)->comm_ptr, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Free window resources in lower layer. */
if (MPIDI_CH3U_Win_hooks.win_free != NULL) {
mpi_errno = MPIDI_CH3U_Win_hooks.win_free(win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
/* dequeue window from the global list */
MPIR_Assert((*win_ptr)->active == FALSE);
MPL_DL_DELETE(MPIDI_RMA_Win_inactive_list_head, (*win_ptr));
if (MPIDI_RMA_Win_inactive_list_head == NULL && MPIDI_RMA_Win_active_list_head == NULL) {
/* this is the last window, de-register RMA progress hook */
mpi_errno = MPID_Progress_deregister_hook(MPIDI_CH3I_RMA_Progress_hook_id);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
comm_ptr = (*win_ptr)->comm_ptr;
mpi_errno = MPIR_Comm_free_impl(comm_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if ((*win_ptr)->basic_info_table != NULL)
MPL_free((*win_ptr)->basic_info_table);
MPL_free((*win_ptr)->op_pool_start);
MPL_free((*win_ptr)->target_pool_start);
MPL_free((*win_ptr)->slots);
MPL_free((*win_ptr)->target_lock_entry_pool_start);
MPIR_Assert((*win_ptr)->current_target_lock_data_bytes == 0);
/* Free the attached buffer for windows created with MPI_Win_allocate() */
if ((*win_ptr)->create_flavor == MPI_WIN_FLAVOR_ALLOCATE ||
(*win_ptr)->create_flavor == MPI_WIN_FLAVOR_SHARED) {
if ((*win_ptr)->shm_allocated == FALSE && (*win_ptr)->size > 0) {
MPL_free((*win_ptr)->base);
}
}
MPIR_Object_release_ref(*win_ptr, &in_use);
/* MPI windows don't have reference count semantics, so this should always be true */
MPIR_Assert(!in_use);
MPIR_Handle_obj_free(&MPIR_Win_mem, *win_ptr);
fn_exit:
MPIR_FUNC_VERBOSE_RMA_EXIT(MPID_STATE_MPID_WIN_FREE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
int *is_able_to_issue, int *made_progress)
{
int rank = win_ptr->comm_ptr->rank;
int mpi_errno = MPI_SUCCESS;
(*made_progress) = 0;
(*is_able_to_issue) = 0;
if (target == NULL)
goto fn_exit;
/* When user event happens, move op in user pending list to network pending list */
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK || target->win_complete_flag) {
MPIDI_RMA_Op_t *user_op = target->pending_user_ops_list_head;
if (user_op != NULL) {
if (target->pending_net_ops_list_head == NULL)
win_ptr->num_targets_with_pending_net_ops++;
MPL_DL_DELETE(target->pending_user_ops_list_head, user_op);
MPL_DL_APPEND(target->pending_net_ops_list_head, user_op);
if (target->next_op_to_issue == NULL)
target->next_op_to_issue = user_op;
}
}
switch (target->access_state) {
case MPIDI_RMA_LOCK_CALLED:
if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if ((target->pending_net_ops_list_head == NULL ||
!target->pending_net_ops_list_head->piggyback_lock_candidate) &&
(target->pending_user_ops_list_head == NULL ||
!target->pending_user_ops_list_head->piggyback_lock_candidate)) {
/* issue lock request */
target->access_state = MPIDI_RMA_LOCK_ISSUED;
if (target->target_rank == rank) {
mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
else {
mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_net_ops_list_head == NULL) {
/* No RMA operation has ever been posted to this target,
* finish issuing, no need to acquire the lock. Cleanup
* function will clean it up. */
target->access_state = MPIDI_RMA_LOCK_GRANTED;
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
else {
/* if we reach WIN_UNLOCK and there is still operation existing
* in pending list, this operation must be the only operation
* and it is prepared to piggyback LOCK and UNLOCK. */
MPIU_Assert(MPIR_CVAR_CH3_RMA_DELAY_ISSUING_FOR_PIGGYBACKING);
MPIU_Assert(target->pending_net_ops_list_head->next == NULL);
MPIU_Assert(target->pending_net_ops_list_head->piggyback_lock_candidate);
}
}
break;
case MPIDI_RMA_LOCK_GRANTED:
case MPIDI_RMA_NONE:
if (target->win_complete_flag) {
if (target->pending_net_ops_list_head == NULL) {
MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE;
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH &&
target->num_ops_flush_not_issued > 0) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_decr_at_cnt_msg(target->target_rank, win_ptr, flags);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if (target->pending_net_ops_list_head == NULL) {
if (target->target_rank != rank) {
if (target->num_ops_flush_not_issued > 0) {
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
mpi_errno = send_flush_msg(target->target_rank, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_net_ops_list_head == NULL) {
if (target->target_rank == rank) {
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
else {
MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
if (target->num_ops_flush_not_issued == 0) {
flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
}
else {
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
break;
default:
break;
} /* end of switch */
if (target->access_state != MPIDI_RMA_LOCK_ISSUED) {
(*is_able_to_issue) = 1;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
