void
MPIDI_WinAtomicCB(pami_context_t context,
void * cookie,
const void * _hdr,
size_t size,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
pami_recv_t * recv)
{
MPIDI_AtomicHeader_t *ahdr = (MPIDI_AtomicHeader_t *) _hdr;
MPID_assert (ahdr != NULL);
MPID_assert (sizeof(MPIDI_AtomicHeader_t) == size);
MPIDI_AtomicHeader_t ack_hdr = *ahdr;
void *dest_addr = ahdr->remote_addr;
int len;
len = MPID_Datatype_get_basic_size (ahdr->datatype);
if (ahdr->atomic_type == MPIDI_WIN_REQUEST_COMPARE_AND_SWAP) {
//overwrite value with result in ack_hdr
MPIU_Memcpy(ack_hdr.buf, dest_addr, len);
if (MPIR_Compare_equal (&ahdr->test, dest_addr, ahdr->datatype))
MPIU_Memcpy(dest_addr, ahdr->buf, len);
}
else if (ahdr->atomic_type == MPIDI_WIN_REQUEST_FETCH_AND_OP) {
//overwrite value with result
MPIU_Memcpy(ack_hdr.buf, dest_addr, len);
MPI_User_function *uop;
int one = 1;
uop = MPIR_OP_HDL_TO_FN(ahdr->op);
if (ahdr->op == MPI_REPLACE)
MPIU_Memcpy(dest_addr, ahdr->buf, len);
else if (ahdr->op == MPI_NO_OP);
else
(*uop) ((void *)ahdr->buf, dest_addr, &one, &ahdr->datatype);
}
else
MPID_abort();
pami_send_immediate_t params = {
.dispatch = MPIDI_Protocols_WinAtomicAck,
.dest = sender,
.header = {
.iov_base = &ack_hdr,
.iov_len = sizeof(MPIDI_AtomicHeader_t),
},
.data = {
.iov_base = NULL,
.iov_len = 0,
},
.hints = {0},
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
MPI_Datatype origin_datatype, void *result_addr, int result_count,
MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win *win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int rank, origin_predefined, result_predefined, target_predefined;
int shm_locked = 0;
int dt_contig ATTRIBUTE((unused));
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPID_Datatype *dtp;
MPIU_CHKLMEM_DECL(2);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);
if (target_rank == MPI_PROC_NULL) {
goto fn_exit;
}
if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
}
MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz,
dtp, dt_true_lb);
if (data_sz == 0) {
goto fn_exit;
}
rank = win_ptr->myrank;
origin_predefined = TRUE; /* quiet uninitialized warnings (b/c goto) */
if (op != MPI_NO_OP) {
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined);
}
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(result_datatype, result_predefined);
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined);
/* Do =! rank first (most likely branch?) */
if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED)
{
MPI_User_function *uop;
void *base;
int disp_unit;
if (win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) {
base = win_ptr->shm_base_addrs[target_rank];
disp_unit = win_ptr->disp_units[target_rank];
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
shm_locked = 1;
}
else {
base = win_ptr->base;
disp_unit = win_ptr->disp_unit;
}
/* Perform the local get first, then the accumulate */
mpi_errno = MPIR_Localcopy((char *) base + disp_unit * target_disp,
target_count, target_datatype,
result_addr, result_count, result_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* NO_OP: Don't perform the accumulate */
if (op == MPI_NO_OP) {
if (shm_locked) {
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
shm_locked = 0;
}
goto fn_exit;
}
if (op == MPI_REPLACE) {
mpi_errno = MPIR_Localcopy(origin_addr, origin_count, origin_datatype,
(char *) base + disp_unit * target_disp,
target_count, target_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
if (shm_locked) {
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
shm_locked = 0;
}
goto fn_exit;
}
MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN),
mpi_errno, MPI_ERR_OP, "**opnotpredefined",
"**opnotpredefined %d", op );
/* get the function by indexing into the op table */
uop = MPIR_OP_HDL_TO_FN(op);
if (origin_predefined && target_predefined) {
/* Cast away const'ness for origin_address in order to
* avoid changing the prototype for MPI_User_function */
(*uop)((void *) origin_addr, (char *) base + disp_unit*target_disp,
&target_count, &target_datatype);
}
else {
/* derived datatype */
MPID_Segment *segp;
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
int vec_len, i, type_size, count;
MPI_Datatype type;
MPI_Aint true_lb, true_extent, extent;
void *tmp_buf=NULL, *target_buf;
const void *source_buf;
if (origin_datatype != target_datatype) {
/* first copy the data into a temporary buffer with
the same datatype as the target. Then do the
accumulate operation. */
MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent);
MPID_Datatype_get_extent_macro(target_datatype, extent);
MPIU_CHKLMEM_MALLOC(tmp_buf, void *,
target_count * (MPIR_MAX(extent,true_extent)),
mpi_errno, "temporary buffer");
/* adjust for potential negative lower bound in datatype */
tmp_buf = (void *)((char*)tmp_buf - true_lb);
mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
origin_datatype, tmp_buf,
target_count, target_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
if (target_predefined) {
/* target predefined type, origin derived datatype */
(*uop)(tmp_buf, (char *) base + disp_unit * target_disp,
&target_count, &target_datatype);
}
else {
segp = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER,
"**nomem","**nomem %s","MPID_Segment_alloc");
MPID_Segment_init(NULL, target_count, target_datatype, segp, 0);
first = 0;
last = SEGMENT_IGNORE_LAST;
MPID_Datatype_get_ptr(target_datatype, dtp);
vec_len = dtp->max_contig_blocks * target_count + 1;
/* +1 needed because Rob says so */
MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *,
vec_len * sizeof(DLOOP_VECTOR),
mpi_errno, "dloop vector");
MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
source_buf = (tmp_buf != NULL) ? tmp_buf : origin_addr;
target_buf = (char *) base + disp_unit * target_disp;
type = dtp->eltype;
type_size = MPID_Datatype_get_basic_size(type);
for (i=0; i<vec_len; i++) {
count = (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size;
(*uop)((char *)source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *)target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
}
MPID_Segment_free(segp);
}
}
int MPIR_Reduce_local_impl(const void *inbuf, void *inoutbuf, int count, MPI_Datatype datatype, MPI_Op op)
{
int mpi_errno = MPI_SUCCESS;
MPID_Op *op_ptr;
MPI_User_function *uop;
#ifdef HAVE_CXX_BINDING
int is_cxx_uop = 0;
#endif
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
int is_f77_uop = 0;
#endif
MPID_THREADPRIV_DECL;
if (count == 0) goto fn_exit;
MPID_THREADPRIV_GET;
MPID_THREADPRIV_FIELD(op_errno) = MPI_SUCCESS;
if (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) {
/* get the function by indexing into the op table */
uop = MPIR_OP_HDL_TO_FN(op);
}
else {
MPID_Op_get_ptr(op, op_ptr);
#ifdef HAVE_CXX_BINDING
if (op_ptr->language == MPID_LANG_CXX) {
uop = (MPI_User_function *) op_ptr->function.c_function;
is_cxx_uop = 1;
}
else
#endif
{
if (op_ptr->language == MPID_LANG_C) {
uop = (MPI_User_function *) op_ptr->function.c_function;
}
else {
uop = (MPI_User_function *) op_ptr->function.f77_function;
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
is_f77_uop = 1;
#endif
}
}
}
/* actually perform the reduction */
#ifdef HAVE_CXX_BINDING
if (is_cxx_uop) {
(*MPIR_Process.cxx_call_op_fn)(inbuf, inoutbuf, count, datatype, uop);
}
else
#endif
{
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
if (is_f77_uop) {
MPI_Fint lcount = (MPI_Fint)count;
MPI_Fint ldtype = (MPI_Fint)datatype;
MPIR_F77_User_function *uop_f77 = (MPIR_F77_User_function *)uop;
(*uop_f77)((void *) inbuf, inoutbuf, &lcount, &ldtype);
}
else {
(*uop)((void *) inbuf, inoutbuf, &count, &datatype);
}
#else
(*uop)((void *) inbuf, inoutbuf, &count, &datatype);
#endif
}
/* --BEGIN ERROR HANDLING-- */
if (MPID_THREADPRIV_FIELD(op_errno))
mpi_errno = MPID_THREADPRIV_FIELD(op_errno);
/* --END ERROR HANDLING-- */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Reduce_local(const void *inbuf, void *inoutbuf, int count, MPI_Datatype datatype,
MPI_Op op)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Op *op_ptr;
MPI_User_function *uop;
#ifdef HAVE_CXX_BINDING
int is_cxx_uop = 0;
#endif
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
int is_f77_uop = 0;
#endif
if (count == 0)
goto fn_exit;
{
MPIR_Per_thread_t *per_thread = NULL;
int err = 0;
MPID_THREADPRIV_KEY_GET_ADDR(MPIR_ThreadInfo.isThreaded, MPIR_Per_thread_key,
MPIR_Per_thread, per_thread, &err);
MPIR_Assert(err == 0);
per_thread->op_errno = MPI_SUCCESS;
}
if (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) {
/* get the function by indexing into the op table */
uop = MPIR_OP_HDL_TO_FN(op);
} else {
MPIR_Op_get_ptr(op, op_ptr);
#ifdef HAVE_CXX_BINDING
if (op_ptr->language == MPIR_LANG__CXX) {
uop = (MPI_User_function *) op_ptr->function.c_function;
is_cxx_uop = 1;
} else
#endif
{
if (op_ptr->language == MPIR_LANG__C) {
uop = (MPI_User_function *) op_ptr->function.c_function;
} else {
uop = (MPI_User_function *) op_ptr->function.f77_function;
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
is_f77_uop = 1;
#endif
}
}
}
/* actually perform the reduction */
#ifdef HAVE_CXX_BINDING
if (is_cxx_uop) {
(*MPIR_Process.cxx_call_op_fn) (inbuf, inoutbuf, count, datatype, uop);
} else
#endif
{
#if defined(HAVE_FORTRAN_BINDING) && !defined(HAVE_FINT_IS_INT)
if (is_f77_uop) {
MPI_Fint lcount = (MPI_Fint) count;
MPI_Fint ldtype = (MPI_Fint) datatype;
MPII_F77_User_function *uop_f77 = (MPII_F77_User_function *) uop;
(*uop_f77) ((void *) inbuf, inoutbuf, &lcount, &ldtype);
} else {
(*uop) ((void *) inbuf, inoutbuf, &count, &datatype);
}
#else
(*uop) ((void *) inbuf, inoutbuf, &count, &datatype);
#endif
}
/* --BEGIN ERROR HANDLING-- */
{
MPIR_Per_thread_t *per_thread = NULL;
int err = 0;
MPID_THREADPRIV_KEY_GET_ADDR(MPIR_ThreadInfo.isThreaded, MPIR_Per_thread_key,
MPIR_Per_thread, per_thread, &err);
MPIR_Assert(err == 0);
if (per_thread->op_errno)
mpi_errno = per_thread->op_errno;
}
/* --END ERROR HANDLING-- */
fn_exit:
return mpi_errno;
}
