int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr, MPID_Request * ureq) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t orig_data_sz, target_data_sz; int dt_contig ATTRIBUTE((unused)), rank; MPI_Aint dt_true_lb ATTRIBUTE((unused)); MPID_Datatype *dtp; MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL; int made_progress = 0; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET); MPIR_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); if (target_rank == MPI_PROC_NULL) { goto fn_exit; } MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, orig_data_sz, dtp, dt_true_lb); if (orig_data_sz == 0) { goto fn_exit; } rank = win_ptr->comm_ptr->rank; if (win_ptr->shm_allocated == TRUE && target_rank != rank && win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) { /* check if target is local and shared memory is allocated on window, * if so, we directly perform this operation on shared memory region. */ /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on * the same node. However, in ch3:sock, even if origin and target are on the same node, they do * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, * which is only set to TRUE when SHM region is allocated in nemesis. * In future we need to figure out a way to check if origin and target are in the same "SHM comm". */ MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); } /* If the get is a local operation, do it here */ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { mpi_errno = MPIDI_CH3I_Shm_get_op(origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count, target_datatype, win_ptr); if (mpi_errno) MPIR_ERR_POP(mpi_errno); if (ureq) { /* Complete user request and release the ch3 ref */ mpi_errno = MPID_Request_complete(ureq); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); } } } else { MPIDI_RMA_Op_t *op_ptr = NULL; MPIDI_CH3_Pkt_get_t *get_pkt = NULL; MPI_Aint target_type_size; int use_immed_resp_pkt = FALSE; int is_origin_contig, is_target_contig; /* queue it up */ mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &op_ptr); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); /******************** Setting operation struct areas ***********************/ /* FIXME: For contig and very short operations, use a streamlined op */ op_ptr->origin_addr = origin_addr; op_ptr->origin_count = origin_count; op_ptr->origin_datatype = origin_datatype; op_ptr->target_rank = target_rank; /* Remember user request */ op_ptr->ureq = ureq; /* if source or target datatypes are derived, increment their * reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); } MPID_Datatype_is_contig(origin_datatype, &is_origin_contig); MPID_Datatype_is_contig(target_datatype, &is_target_contig); MPID_Datatype_get_size_macro(target_datatype, target_type_size); MPIU_Assign_trunc(target_data_sz, target_count * target_type_size, MPIDI_msg_sz_t); /* Judge if we can use IMMED data response packet */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype) && is_origin_contig && is_target_contig) { if (target_data_sz <= MPIDI_RMA_IMMED_BYTES) use_immed_resp_pkt = TRUE; } /* Judge if this operation is an piggyback candidate. */ if (MPIR_DATATYPE_IS_PREDEFINED(origin_datatype) && MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes * for both origin and target data. We should extend this optimization to derived * datatypes as well. */ op_ptr->piggyback_lock_candidate = 1; } /************** Setting packet struct areas in operation ****************/ get_pkt = &(op_ptr->pkt.get); MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET); get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr + win_ptr->basic_info_table[target_rank].disp_unit * target_disp; get_pkt->count = target_count; get_pkt->datatype = target_datatype; get_pkt->info.dataloop_size = 0; get_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle; get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE; if (use_immed_resp_pkt) get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP; MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, op_ptr); if (mpi_errno) MPIR_ERR_POP(mpi_errno); mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 && MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { while (MPIDI_CH3I_RMA_Active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) { mpi_errno = wait_progress_engine(); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } } } fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: goto fn_exit; /* --END ERROR HANDLING-- */ }
int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPID_Win *win_ptr) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t data_sz; int dt_contig ATTRIBUTE((unused)), rank; MPI_Aint dt_true_lb ATTRIBUTE((unused)); MPID_Datatype *dtp; MPIDI_VC_t *orig_vc, *target_vc; MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET); MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET); if (target_rank == MPI_PROC_NULL) { goto fn_exit; } if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) { win_ptr->epoch_state = MPIDI_EPOCH_FENCE; } MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE, mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync"); MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb); if (data_sz == 0) { goto fn_exit; } rank = win_ptr->comm_ptr->rank; if (win_ptr->shm_allocated == TRUE && target_rank != rank && win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) { /* check if target is local and shared memory is allocated on window, if so, we directly perform this operation on shared memory region. */ /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on the same node. However, in ch3:sock, even if origin and target are on the same node, they do not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first, which is only set to TRUE when SHM region is allocated in nemesis. In future we need to figure out a way to check if origin and target are in the same "SHM comm". */ MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc); MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc); } /* If the get is a local operation, do it here */ if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED || (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) { mpi_errno = MPIDI_CH3I_Shm_get_op(origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count, target_datatype, win_ptr); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } else { MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank); MPIDI_RMA_Op_t *new_ptr = NULL; /* queue it up */ MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc); mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr); MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set); /* FIXME: For contig and very short operations, use a streamlined op */ new_ptr->type = MPIDI_RMA_GET; new_ptr->origin_addr = origin_addr; new_ptr->origin_count = origin_count; new_ptr->origin_datatype = origin_datatype; new_ptr->target_rank = target_rank; new_ptr->target_disp = target_disp; new_ptr->target_count = target_count; new_ptr->target_datatype = target_datatype; MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set); /* if source or target datatypes are derived, increment their reference counts */ if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) { MPID_Datatype_get_ptr(origin_datatype, dtp); MPID_Datatype_add_ref(dtp); } if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) { MPID_Datatype_get_ptr(target_datatype, dtp); MPID_Datatype_add_ref(dtp); } } fn_exit: MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET); return mpi_errno; /* --BEGIN ERROR HANDLING-- */ fn_fail: goto fn_exit; /* --END ERROR HANDLING-- */ }