/* MPID_Segment_vector_flatten
*
* Notes:
* - this is only called when the starting position is at the beginning
* of a whole block in a vector type.
* - this was a virtual copy of MPID_Segment_pack_to_iov; now it has improvements
* that MPID_Segment_pack_to_iov needs.
* - we return the number of blocks that we did process in region pointed to by
* blocks_p.
*/
static int MPID_Segment_vector_flatten(DLOOP_Offset *blocks_p,
DLOOP_Count count,
DLOOP_Size blksz,
DLOOP_Offset stride,
DLOOP_Type el_type,
DLOOP_Offset rel_off, /* offset into buffer */
void *bufp, /* start of buffer */
void *v_paramp)
{
int i;
DLOOP_Offset size, blocks_left, basic_size;
struct MPID_Segment_piece_params *paramp = v_paramp;
MPIDI_STATE_DECL(MPID_STATE_MPID_SEGMENT_VECTOR_FLATTEN);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_SEGMENT_VECTOR_FLATTEN);
basic_size = (DLOOP_Offset) MPID_Datatype_get_basic_size(el_type);
blocks_left = *blocks_p;
for (i=0; i < count && blocks_left > 0; i++) {
int idx = paramp->u.flatten.index;
if (blocks_left > (DLOOP_Offset) blksz) {
size = ((DLOOP_Offset) blksz) * basic_size;
blocks_left -= (DLOOP_Offset) blksz;
}
else {
/* last pass */
size = blocks_left * basic_size;
blocks_left = 0;
}
if (idx > 0 && ((DLOOP_Offset) MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp + rel_off) ==
((paramp->u.flatten.offp[idx - 1]) + (DLOOP_Offset) paramp->u.flatten.sizep[idx - 1]))
{
/* add this size to the last region rather than using up another one */
paramp->u.flatten.sizep[idx - 1] += size;
}
else if (idx < paramp->u.flatten.length) {
/* take up another region */
paramp->u.flatten.offp[idx] = (DLOOP_Offset) MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp + rel_off;
paramp->u.flatten.sizep[idx] = size;
paramp->u.flatten.index++;
}
else {
/* we tried to add to the end of the last region and failed; add blocks back in */
*blocks_p = *blocks_p - blocks_left + (size / basic_size);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_VECTOR_FLATTEN);
return 1;
}
rel_off += stride;
}
/* --BEGIN ERROR HANDLING-- */
MPIU_Assert(blocks_left == 0);
/* --END ERROR HANDLING-- */
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_VECTOR_FLATTEN);
return 0;
}
void
MPIDI_WinAtomicCB(pami_context_t context,
void * cookie,
const void * _hdr,
size_t size,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
pami_recv_t * recv)
{
MPIDI_AtomicHeader_t *ahdr = (MPIDI_AtomicHeader_t *) _hdr;
MPID_assert (ahdr != NULL);
MPID_assert (sizeof(MPIDI_AtomicHeader_t) == size);
MPIDI_AtomicHeader_t ack_hdr = *ahdr;
void *dest_addr = ahdr->remote_addr;
int len;
len = MPID_Datatype_get_basic_size (ahdr->datatype);
if (ahdr->atomic_type == MPIDI_WIN_REQUEST_COMPARE_AND_SWAP) {
//overwrite value with result in ack_hdr
MPIU_Memcpy(ack_hdr.buf, dest_addr, len);
if (MPIR_Compare_equal (&ahdr->test, dest_addr, ahdr->datatype))
MPIU_Memcpy(dest_addr, ahdr->buf, len);
}
else if (ahdr->atomic_type == MPIDI_WIN_REQUEST_FETCH_AND_OP) {
//overwrite value with result
MPIU_Memcpy(ack_hdr.buf, dest_addr, len);
MPI_User_function *uop;
int one = 1;
uop = MPIR_OP_HDL_TO_FN(ahdr->op);
if (ahdr->op == MPI_REPLACE)
MPIU_Memcpy(dest_addr, ahdr->buf, len);
else if (ahdr->op == MPI_NO_OP);
else
(*uop) ((void *)ahdr->buf, dest_addr, &one, &ahdr->datatype);
}
else
MPID_abort();
pami_send_immediate_t params = {
.dispatch = MPIDI_Protocols_WinAtomicAck,
.dest = sender,
.header = {
.iov_base = &ack_hdr,
.iov_len = sizeof(MPIDI_AtomicHeader_t),
},
.data = {
.iov_base = NULL,
.iov_len = 0,
},
.hints = {0},
/* MPID_Segment_contig_pack_to_iov
*/
static int MPID_Segment_contig_pack_to_iov(DLOOP_Offset *blocks_p,
DLOOP_Type el_type,
DLOOP_Offset rel_off,
void *bufp,
void *v_paramp)
{
int el_size, last_idx;
DLOOP_Offset size;
char *last_end = NULL;
struct MPID_Segment_piece_params *paramp = v_paramp;
MPIDI_STATE_DECL(MPID_STATE_MPID_SEGMENT_CONTIG_PACK_TO_IOV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_SEGMENT_CONTIG_PACK_TO_IOV);
el_size = MPID_Datatype_get_basic_size(el_type);
size = *blocks_p * (DLOOP_Offset) el_size;
MPL_DBG_MSG_FMT(MPIR_DBG_DATATYPE,VERBOSE,(MPL_DBG_FDEST,
" contig to vec: do=" MPI_AINT_FMT_DEC_SPEC ", dp=%p, ind=%d, sz=%d, blksz=" MPI_AINT_FMT_DEC_SPEC,
(MPI_Aint) rel_off,
bufp,
paramp->u.pack_vector.index,
el_size,
(MPI_Aint) *blocks_p));
last_idx = paramp->u.pack_vector.index - 1;
if (last_idx >= 0) {
last_end = ((char *) paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_BUF) +
paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_LEN;
}
MPIU_Ensure_Aint_fits_in_pointer((MPIU_VOID_PTR_CAST_TO_MPI_AINT (bufp)) + rel_off);
if ((last_idx == paramp->u.pack_vector.length-1) &&
(last_end != ((char *) bufp + rel_off)))
{
/* we have used up all our entries, and this region doesn't fit on
* the end of the last one. setting blocks to 0 tells manipulation
* function that we are done (and that we didn't process any blocks).
*/
*blocks_p = 0;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_CONTIG_PACK_TO_IOV);
return 1;
}
else if (last_idx >= 0 && (last_end == ((char *) bufp + rel_off)))
{
/* add this size to the last vector rather than using up another one */
paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_LEN += size;
}
else {
paramp->u.pack_vector.vectorp[last_idx+1].DLOOP_VECTOR_BUF = (char *) bufp + rel_off;
paramp->u.pack_vector.vectorp[last_idx+1].DLOOP_VECTOR_LEN = size;
paramp->u.pack_vector.index++;
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_CONTIG_PACK_TO_IOV);
return 0;
}
static int handler_recv_dequeue_complete(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *const rreq = e->user_ptr;
int is_contig;
MPI_Aint last;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr ATTRIBUTE((unused));
MPIDI_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIU_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, is_contig, data_sz, dt_ptr, dt_true_lb);
dequeue_req(e);
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
/* unpack the data from unexpected buffer */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "is_contig = %d", is_contig);
if (is_contig) {
MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, e->start);
if (last != e->mlength)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
} else {
/* Data was placed directly into the user buffer, so datatype mismatch
is harder to detect. We use a simple check ensuring the received bytes
are a multiple of a single basic element. Currently, we do not detect
mismatches with datatypes constructed of more than one basic type */
MPI_Datatype dt_basic_type;
MPID_Datatype_get_basic_type(rreq->dev.datatype, dt_basic_type);
if (dt_basic_type != MPI_DATATYPE_NULL && (e->mlength % MPID_Datatype_get_basic_size(dt_basic_type)) != 0)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
mpi_errno = handler_recv_complete(e);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* MPID_Segment_contig_flatten
*/
static int MPID_Segment_contig_flatten(DLOOP_Offset *blocks_p,
DLOOP_Type el_type,
DLOOP_Offset rel_off,
void *bufp,
void *v_paramp)
{
int idx, el_size;
DLOOP_Offset size;
struct MPID_Segment_piece_params *paramp = v_paramp;
MPIDI_STATE_DECL(MPID_STATE_MPID_SEGMENT_CONTIG_FLATTEN);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_SEGMENT_CONTIG_FLATTEN);
el_size = MPID_Datatype_get_basic_size(el_type);
size = *blocks_p * (DLOOP_Offset) el_size;
idx = paramp->u.flatten.index;
#ifdef MPID_SP_VERBOSE
MPL_DBG_MSG_FMT(MPIR_DBG_DATATYPE,VERBOSE,(MPL_DBG_FDEST,"\t[contig flatten: idx = %d, loc = (" MPI_AINT_FMT_HEX_SPEC " + " MPI_AINT_FMT_HEX_SPEC ") = " MPI_AINT_FMT_HEX_SPEC ", size = " MPI_AINT_FMT_DEC_SPEC "]\n",
idx,
MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp,
(MPI_Aint) rel_off,
MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp + rel_off,
(MPI_Aint) size));
#endif
if (idx > 0 && ((DLOOP_Offset) MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp + rel_off) ==
((paramp->u.flatten.offp[idx - 1]) +
(DLOOP_Offset) paramp->u.flatten.sizep[idx - 1]))
{
/* add this size to the last vector rather than using up another one */
paramp->u.flatten.sizep[idx - 1] += size;
}
else {
paramp->u.flatten.offp[idx] = ((int64_t) MPIU_VOID_PTR_CAST_TO_MPI_AINT bufp) + (int64_t) rel_off;
paramp->u.flatten.sizep[idx] = size;
paramp->u.flatten.index++;
/* check to see if we have used our entire vector buffer, and if so
* return 1 to stop processing
*/
if (paramp->u.flatten.index == paramp->u.flatten.length)
{
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_CONTIG_FLATTEN);
return 1;
}
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_CONTIG_FLATTEN);
return 0;
}
void MPIR_Type_get_extent_x_impl(MPI_Datatype datatype, MPI_Count *lb, MPI_Count *extent)
{
MPID_Datatype *datatype_ptr = NULL;
MPID_Datatype_get_ptr(datatype, datatype_ptr);
if (HANDLE_GET_KIND(datatype) == HANDLE_KIND_BUILTIN) {
*lb = 0;
*extent = MPID_Datatype_get_basic_size(datatype);
}
else {
*lb = datatype_ptr->lb;
*extent = datatype_ptr->extent; /* derived, should be same as ub - lb */
}
}
void MPIR_Type_get_true_extent_impl(MPI_Datatype datatype, MPI_Aint *true_lb, MPI_Aint *true_extent)
{
MPID_Datatype *datatype_ptr = NULL;
MPID_Datatype_get_ptr(datatype, datatype_ptr);
if (HANDLE_GET_KIND(datatype) == HANDLE_KIND_BUILTIN) {
*true_lb = 0;
*true_extent = MPID_Datatype_get_basic_size(datatype);
}
else {
*true_lb = datatype_ptr->true_lb;
*true_extent = datatype_ptr->true_ub - datatype_ptr->true_lb;
}
}
/*@
MPID_Type_vector - create a vector datatype
Input Parameters:
+ count - number of blocks in vector
. blocklength - number of elements in each block
. stride - distance from beginning of one block to the next (see next
parameter for units)
. strideinbytes - if nonzero, then stride is in bytes, otherwise stride
is in terms of extent of oldtype
- oldtype - type (using handle) of datatype on which vector is based
Output Parameters:
. newtype - handle of new vector datatype
Return Value:
0 on success, MPI error code on failure.
@*/
int MPID_Type_vector(int count,
int blocklength,
MPI_Aint stride,
int strideinbytes,
MPI_Datatype oldtype,
MPI_Datatype *newtype)
{
int mpi_errno = MPI_SUCCESS;
int is_builtin, old_is_contig;
MPI_Aint el_sz, old_sz;
MPI_Datatype el_type;
MPI_Aint old_lb, old_ub, old_extent, old_true_lb, old_true_ub, eff_stride;
MPID_Datatype *new_dtp;
if (count == 0) return MPID_Type_zerolen(newtype);
/* allocate new datatype object and handle */
new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
if (!new_dtp) {
/* --BEGIN ERROR HANDLING-- */
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
"MPID_Type_vector", __LINE__,
MPI_ERR_OTHER, "**nomem", 0);
return mpi_errno;
/* --END ERROR HANDLING-- */
}
/* handle is filled in by MPIU_Handle_obj_alloc() */
MPIU_Object_set_ref(new_dtp, 1);
new_dtp->is_permanent = 0;
new_dtp->is_committed = 0;
new_dtp->attributes = NULL;
new_dtp->cache_id = 0;
new_dtp->name[0] = 0;
new_dtp->contents = NULL;
new_dtp->dataloop = NULL;
new_dtp->dataloop_size = -1;
new_dtp->dataloop_depth = -1;
new_dtp->hetero_dloop = NULL;
new_dtp->hetero_dloop_size = -1;
new_dtp->hetero_dloop_depth = -1;
is_builtin = (HANDLE_GET_KIND(oldtype) == HANDLE_KIND_BUILTIN);
if (is_builtin) {
el_sz = (MPI_Aint) MPID_Datatype_get_basic_size(oldtype);
el_type = oldtype;
old_lb = 0;
old_true_lb = 0;
old_ub = el_sz;
old_true_ub = el_sz;
old_sz = el_sz;
old_extent = el_sz;
old_is_contig = 1;
new_dtp->size = (MPI_Aint) count *
(MPI_Aint) blocklength * el_sz;
new_dtp->has_sticky_lb = 0;
new_dtp->has_sticky_ub = 0;
new_dtp->alignsize = el_sz; /* ??? */
new_dtp->n_builtin_elements = count * blocklength;
new_dtp->builtin_element_size = el_sz;
new_dtp->basic_type = el_type;
new_dtp->max_contig_blocks = count;
eff_stride = (strideinbytes) ? stride : (stride * el_sz);
}
else /* user-defined base type (oldtype) */ {
MPID_Datatype *old_dtp;
MPID_Datatype_get_ptr(oldtype, old_dtp);
el_sz = old_dtp->builtin_element_size;
el_type = old_dtp->basic_type;
old_lb = old_dtp->lb;
old_true_lb = old_dtp->true_lb;
old_ub = old_dtp->ub;
old_true_ub = old_dtp->true_ub;
old_sz = old_dtp->size;
old_extent = old_dtp->extent;
old_is_contig = old_dtp->is_contig;
new_dtp->size = count * blocklength * old_dtp->size;
new_dtp->has_sticky_lb = old_dtp->has_sticky_lb;
new_dtp->has_sticky_ub = old_dtp->has_sticky_ub;
new_dtp->alignsize = old_dtp->alignsize;
new_dtp->n_builtin_elements = count * blocklength * old_dtp->n_builtin_elements;
new_dtp->builtin_element_size = el_sz;
new_dtp->basic_type = el_type;
new_dtp->max_contig_blocks = old_dtp->max_contig_blocks * count * blocklength;
eff_stride = (strideinbytes) ? stride : (stride * old_dtp->extent);
}
MPID_DATATYPE_VECTOR_LB_UB((MPI_Aint) count,
eff_stride,
(MPI_Aint) blocklength,
old_lb,
old_ub,
old_extent,
new_dtp->lb,
new_dtp->ub);
new_dtp->true_lb = new_dtp->lb + (old_true_lb - old_lb);
new_dtp->true_ub = new_dtp->ub + (old_true_ub - old_ub);
new_dtp->extent = new_dtp->ub - new_dtp->lb;
/* new type is only contig for N types if old one was, and
* size and extent of new type are equivalent, and stride is
* equal to blocklength * size of old type.
*/
if ((MPI_Aint)(new_dtp->size) == new_dtp->extent &&
eff_stride == (MPI_Aint) blocklength * old_sz &&
old_is_contig)
{
new_dtp->is_contig = 1;
new_dtp->max_contig_blocks = 1;
}
else {
new_dtp->is_contig = 0;
}
*newtype = new_dtp->handle;
MPIU_DBG_MSG_P(DATATYPE,VERBOSE,"vector type %x created.",
new_dtp->handle);
return mpi_errno;
}
int MPIR_Pack_impl(const void *inbuf,
int incount,
MPI_Datatype datatype,
void *outbuf,
MPI_Aint outsize,
MPI_Aint *position)
{
int mpi_errno = MPI_SUCCESS;
MPI_Aint first, last;
MPID_Segment *segp;
int contig;
MPI_Aint dt_true_lb;
MPI_Aint data_sz;
if (incount == 0) {
goto fn_exit;
}
/* Handle contig case quickly */
if (HANDLE_GET_KIND(datatype) == HANDLE_KIND_BUILTIN) {
contig = TRUE;
dt_true_lb = 0;
data_sz = incount * MPID_Datatype_get_basic_size(datatype);
} else {
MPID_Datatype *dt_ptr;
MPID_Datatype_get_ptr(datatype, dt_ptr);
contig = dt_ptr->is_contig;
dt_true_lb = dt_ptr->true_lb;
data_sz = incount * dt_ptr->size;
}
if (contig) {
MPIU_Memcpy((char *) outbuf + *position, (char *)inbuf + dt_true_lb, data_sz);
*position = (int)((MPI_Aint)*position + data_sz);
goto fn_exit;
}
/* non-contig case */
/* TODO: CHECK RETURN VALUES?? */
/* TODO: SHOULD THIS ALL BE IN A MPID_PACK??? */
segp = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1(segp == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment");
mpi_errno = MPID_Segment_init(inbuf, incount, datatype, segp, 0);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* NOTE: the use of buffer values and positions in MPI_Pack and in
* MPID_Segment_pack are quite different. See code or docs or something.
*/
first = 0;
last = SEGMENT_IGNORE_LAST;
/* Ensure that pointer increment fits in a pointer */
MPID_Ensure_Aint_fits_in_pointer((MPI_VOID_PTR_CAST_TO_MPI_AINT outbuf) +
(MPI_Aint) *position);
MPID_Segment_pack(segp,
first,
&last,
(void *) ((char *) outbuf + *position));
/* Ensure that calculation fits into an int datatype. */
MPID_Ensure_Aint_fits_in_int((MPI_Aint)*position + last);
*position = (int)((MPI_Aint)*position + last);
MPID_Segment_free(segp);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Type_blockindexed(int count,
int blocklength,
void *displacement_array,
int dispinbytes,
MPI_Datatype oldtype,
MPI_Datatype *newtype)
{
int mpi_errno = MPI_SUCCESS, i;
int is_builtin, contig_count, old_is_contig;
MPI_Aint el_sz;
MPI_Datatype el_type;
MPI_Aint old_lb, old_ub, old_extent, old_true_lb, old_true_ub;
MPI_Aint min_lb = 0, max_ub = 0, eff_disp;
MPID_Datatype *new_dtp;
if (count == 0) return MPID_Type_zerolen(newtype);
/* allocate new datatype object and handle */
new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
/* --BEGIN ERROR HANDLING-- */
if (!new_dtp)
{
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
"MPID_Type_vector", __LINE__,
MPI_ERR_OTHER, "**nomem", 0);
return mpi_errno;
}
/* --END ERROR HANDLING-- */
/* handle is filled in by MPIU_Handle_obj_alloc() */
MPIU_Object_set_ref(new_dtp, 1);
new_dtp->is_permanent = 0;
new_dtp->is_committed = 0;
new_dtp->attributes = NULL;
new_dtp->cache_id = 0;
new_dtp->name[0] = 0;
new_dtp->contents = NULL;
new_dtp->dataloop = NULL;
new_dtp->dataloop_size = -1;
new_dtp->dataloop_depth = -1;
new_dtp->hetero_dloop = NULL;
new_dtp->hetero_dloop_size = -1;
new_dtp->hetero_dloop_depth = -1;
is_builtin = (HANDLE_GET_KIND(oldtype) == HANDLE_KIND_BUILTIN);
if (is_builtin)
{
el_sz = (MPI_Aint) MPID_Datatype_get_basic_size(oldtype);
el_type = oldtype;
old_lb = 0;
old_true_lb = 0;
old_ub = el_sz;
old_true_ub = el_sz;
old_extent = el_sz;
old_is_contig = 1;
new_dtp->size = (MPI_Aint) count *
(MPI_Aint) blocklength * el_sz;
new_dtp->has_sticky_lb = 0;
new_dtp->has_sticky_ub = 0;
new_dtp->alignsize = el_sz; /* ??? */
new_dtp->n_elements = count * blocklength;
new_dtp->element_size = el_sz;
new_dtp->eltype = el_type;
new_dtp->max_contig_blocks = count;
}
else
{
/* user-defined base type (oldtype) */
MPID_Datatype *old_dtp;
MPID_Datatype_get_ptr(oldtype, old_dtp);
el_sz = old_dtp->element_size;
el_type = old_dtp->eltype;
old_lb = old_dtp->lb;
old_true_lb = old_dtp->true_lb;
old_ub = old_dtp->ub;
old_true_ub = old_dtp->true_ub;
old_extent = old_dtp->extent;
old_is_contig = old_dtp->is_contig;
new_dtp->size = (MPI_Aint) count *
(MPI_Aint) blocklength *
(MPI_Aint) old_dtp->size;
new_dtp->has_sticky_lb = old_dtp->has_sticky_lb;
new_dtp->has_sticky_ub = old_dtp->has_sticky_ub;
new_dtp->alignsize = old_dtp->alignsize;
new_dtp->n_elements = count * blocklength * old_dtp->n_elements;
new_dtp->element_size = el_sz;
new_dtp->eltype = el_type;
new_dtp->max_contig_blocks = old_dtp->max_contig_blocks * count * blocklength;
}
/* priming for loop */
eff_disp = (dispinbytes) ? ((MPI_Aint *) displacement_array)[0] :
(((MPI_Aint) ((int *) displacement_array)[0]) * old_extent);
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint) blocklength,
eff_disp,
old_lb,
old_ub,
old_extent,
min_lb,
max_ub);
/* determine new min lb and max ub */
for (i=1; i < count; i++)
{
MPI_Aint tmp_lb, tmp_ub;
eff_disp = (dispinbytes) ? ((MPI_Aint *) displacement_array)[i] :
(((MPI_Aint) ((int *) displacement_array)[i]) * old_extent);
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint) blocklength,
eff_disp,
old_lb,
old_ub,
old_extent,
tmp_lb,
tmp_ub);
if (tmp_lb < min_lb) min_lb = tmp_lb;
if (tmp_ub > max_ub) max_ub = tmp_ub;
}
new_dtp->lb = min_lb;
new_dtp->ub = max_ub;
new_dtp->true_lb = min_lb + (old_true_lb - old_lb);
new_dtp->true_ub = max_ub + (old_true_ub - old_ub);
new_dtp->extent = max_ub - min_lb;
/* new type is contig for N types if it is all one big block,
* its size and extent are the same, and the old type was also
* contiguous.
*/
new_dtp->is_contig = 0;
if (old_is_contig)
{
contig_count = MPID_Type_blockindexed_count_contig(count,
blocklength,
displacement_array,
dispinbytes,
old_extent);
new_dtp->max_contig_blocks = contig_count;
if( (contig_count == 1) &&
((MPI_Aint) new_dtp->size == new_dtp->extent) )
{
new_dtp->is_contig = 1;
}
}
*newtype = new_dtp->handle;
return mpi_errno;
}
int MPIDI_Accumulate(void *origin_addr, int origin_count, MPI_Datatype
origin_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPI_Op op,
MPID_Win *win_ptr)
{
int mpi_errno=MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int dt_contig, rank, origin_predefined, target_predefined;
MPI_Aint dt_true_lb;
MPIDI_RMA_ops *new_ptr;
MPID_Datatype *dtp;
MPIU_CHKLMEM_DECL(2);
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_ACCUMULATE);
MPIDI_Datatype_get_info(origin_count, origin_datatype,
dt_contig, data_sz, dtp, dt_true_lb);
if ((data_sz == 0) || (target_rank == MPI_PROC_NULL))
{
goto fn_exit;
}
rank = win_ptr->myrank;
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined);
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined);
/* Do =! rank first (most likely branch?) */
if (target_rank == rank)
{
MPI_User_function *uop;
if (op == MPI_REPLACE)
{
mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
origin_datatype,
(char *) win_ptr->base + win_ptr->disp_unit *
target_disp, target_count, target_datatype);
goto fn_exit;
}
MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN),
mpi_errno, MPI_ERR_OP, "**opnotpredefined",
"**opnotpredefined %d", op );
/* get the function by indexing into the op table */
uop = MPIR_Op_table[((op)&0xf) - 1];
if (origin_predefined && target_predefined)
{
(*uop)(origin_addr, (char *) win_ptr->base + win_ptr->disp_unit *
target_disp, &target_count, &target_datatype);
}
else
{
/* derived datatype */
MPID_Segment *segp;
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
int vec_len, i, type_size, count;
MPI_Datatype type;
MPI_Aint true_lb, true_extent, extent;
void *tmp_buf=NULL, *source_buf, *target_buf;
if (origin_datatype != target_datatype)
{
/* first copy the data into a temporary buffer with
the same datatype as the target. Then do the
accumulate operation. */
MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent);
MPID_Datatype_get_extent_macro(target_datatype, extent);
MPIU_CHKLMEM_MALLOC(tmp_buf, void *,
target_count * (MPIR_MAX(extent,true_extent)),
mpi_errno, "temporary buffer");
/* adjust for potential negative lower bound in datatype */
tmp_buf = (void *)((char*)tmp_buf - true_lb);
mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
origin_datatype, tmp_buf,
target_count, target_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
if (target_predefined) {
/* target predefined type, origin derived datatype */
(*uop)(tmp_buf, (char *) win_ptr->base + win_ptr->disp_unit *
target_disp, &target_count, &target_datatype);
}
else {
segp = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER,
"**nomem","**nomem %s","MPID_Segment_alloc");
MPID_Segment_init(NULL, target_count, target_datatype, segp, 0);
first = 0;
last = SEGMENT_IGNORE_LAST;
MPID_Datatype_get_ptr(target_datatype, dtp);
vec_len = dtp->max_contig_blocks * target_count + 1;
/* +1 needed because Rob says so */
MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *,
vec_len * sizeof(DLOOP_VECTOR),
mpi_errno, "dloop vector");
MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
source_buf = (tmp_buf != NULL) ? tmp_buf : origin_addr;
target_buf = (char *) win_ptr->base +
win_ptr->disp_unit * target_disp;
type = dtp->eltype;
type_size = MPID_Datatype_get_basic_size(type);
for (i=0; i<vec_len; i++)
{
count = (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size;
(*uop)((char *)source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *)target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
}
MPID_Segment_free(segp);
}
}
/* MPID_Type_struct_alignsize
*
* This function guesses at how the C compiler would align a structure
* with the given components.
*
* It uses these configure-time defines to do its magic:
* - HAVE_MAX_INTEGER_ALIGNMENT - maximum byte alignment of integers
* - HAVE_MAX_FP_ALIGNMENT - maximum byte alignment of floating points
* - HAVE_MAX_LONG_DOUBLE_FP_ALIGNMENT - maximum byte alignment with long
* doubles (if different from FP_ALIGNMENT)
* - HAVE_MAX_DOUBLE_FP_ALIGNMENT - maximum byte alignment with doubles (if
* long double is different from FP_ALIGNMENT)
* - HAVE_DOUBLE_POS_ALIGNMENT - indicates that structures with doubles
* are aligned differently if double isn't
* at displacement 0 (e.g. PPC32/64).
* - HAVE_LLINT_POS_ALIGNMENT - same as above, for MPI_LONG_LONG_INT
*
* The different FP, DOUBLE, LONG_DOUBLE alignment case are necessary for
* Cygwin on X86 (because long_double is 12 bytes, so double and long double
* have different natural alignments). Linux on X86, however, does not have
* different rules for this case.
*/
static MPI_Aint MPID_Type_struct_alignsize(int count,
const MPI_Datatype *oldtype_array,
const MPI_Aint *displacement_array)
{
int i;
MPI_Aint max_alignsize = 0, tmp_alignsize, derived_alignsize = 0;
for (i=0; i < count; i++)
{
/* shouldn't be called with an LB or UB, but we'll handle it nicely */
if (oldtype_array[i] == MPI_LB || oldtype_array[i] == MPI_UB) continue;
else if (HANDLE_GET_KIND(oldtype_array[i]) == HANDLE_KIND_BUILTIN)
{
tmp_alignsize = MPID_Datatype_get_basic_size(oldtype_array[i]);
#ifdef HAVE_DOUBLE_ALIGNMENT_EXCEPTION
if (oldtype_array[i] == MPI_DOUBLE) {
tmp_alignsize = HAVE_DOUBLE_ALIGNMENT_EXCEPTION;
}
#endif
switch(oldtype_array[i])
{
case MPI_FLOAT:
case MPI_DOUBLE:
case MPI_LONG_DOUBLE:
#if defined(HAVE_MAX_LONG_DOUBLE_FP_ALIGNMENT) && \
defined(HAVE_MAX_DOUBLE_FP_ALIGNMENT)
if (oldtype_array[i] == MPI_LONG_DOUBLE) {
if (tmp_alignsize > HAVE_MAX_LONG_DOUBLE_FP_ALIGNMENT)
tmp_alignsize = HAVE_MAX_LONG_DOUBLE_FP_ALIGNMENT;
}
else if (oldtype_array[i] == MPI_DOUBLE) {
if (tmp_alignsize > HAVE_MAX_DOUBLE_FP_ALIGNMENT)
tmp_alignsize = HAVE_MAX_DOUBLE_FP_ALIGNMENT;
}
else {
/* HAVE_MAX_FP_ALIGNMENT may not be defined, hence commented */
/*
if (tmp_alignsize > HAVE_MAX_FP_ALIGNMENT)
tmp_alignsize = HAVE_MAX_FP_ALIGNMENT;
*/
}
#elif defined(HAVE_MAX_FP_ALIGNMENT)
if (tmp_alignsize > HAVE_MAX_FP_ALIGNMENT)
tmp_alignsize = HAVE_MAX_FP_ALIGNMENT;
#endif
#ifdef HAVE_DOUBLE_POS_ALIGNMENT
/* sort of a hack, but so is this rule */
if (oldtype_array[i] == MPI_DOUBLE &&
displacement_array[i] != (MPI_Aint) 0)
{
tmp_alignsize = 4;
}
#endif
break;
default:
#ifdef HAVE_MAX_INTEGER_ALIGNMENT
if (tmp_alignsize > HAVE_MAX_INTEGER_ALIGNMENT)
tmp_alignsize = HAVE_MAX_INTEGER_ALIGNMENT;
#endif
break;
#ifdef HAVE_LLINT_POS_ALIGNMENT
if (oldtype_array[i] == MPI_LONG_LONG_INT &&
displacement_array[i] != (MPI_Aint) 0)
{
tmp_alignsize = 4;
}
#endif
}
}
else
{
MPID_Datatype *dtp;
MPID_Datatype_get_ptr(oldtype_array[i], dtp);
tmp_alignsize = dtp->alignsize;
if (derived_alignsize < tmp_alignsize)
derived_alignsize = tmp_alignsize;
}
if (max_alignsize < tmp_alignsize) max_alignsize = tmp_alignsize;
}
return max_alignsize;
}
int MPIR_T_pvar_handle_alloc_impl(MPI_T_pvar_session session, int pvar_index,
void *obj_handle, MPI_T_pvar_handle *handle,int *count)
{
int mpi_errno = MPI_SUCCESS;
int cnt, extra, bytes;
int is_sum, is_watermark;
const pvar_table_entry_t *info;
MPIR_T_pvar_handle_t *hnd;
MPIR_CHKPMEM_DECL(1);
info = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
if (info->get_count == NULL) {
cnt = info->count;
} else {
info->get_count(info->addr, obj_handle, &cnt);
}
bytes = MPID_Datatype_get_basic_size(info->datatype);
is_sum = FALSE;
is_watermark = FALSE;
extra = 0;
if (info->varclass == MPI_T_PVAR_CLASS_COUNTER ||
info->varclass == MPI_T_PVAR_CLASS_AGGREGATE ||
info->varclass == MPI_T_PVAR_CLASS_TIMER)
{
/* Extra memory for accum, offset, current */
is_sum = TRUE;
extra = bytes * cnt * 3;
} else if (info->varclass == MPI_T_PVAR_CLASS_HIGHWATERMARK ||
info->varclass == MPI_T_PVAR_CLASS_LOWWATERMARK)
{
is_watermark = TRUE;
}
/* Allocate memory and bzero it */
MPIR_CHKPMEM_CALLOC(hnd, MPIR_T_pvar_handle_t*, sizeof(*hnd) + extra,
mpi_errno, "performance variable handle");
#ifdef HAVE_ERROR_CHECKING
hnd->kind = MPIR_T_PVAR_HANDLE;
#endif
/* Setup the common fields */
if (is_sum)
hnd->flags |= MPIR_T_PVAR_FLAG_SUM;
else if (is_watermark)
hnd->flags |= MPIR_T_PVAR_FLAG_WATERMARK;
hnd->addr = info->addr;
hnd->datatype = info->datatype;
hnd->count = cnt;
hnd->varclass = info->varclass;
hnd->flags = info->flags;
hnd->session = session;
hnd->info = info;
hnd->obj_handle = obj_handle;
hnd->get_value = info->get_value;
hnd->bytes = bytes;
hnd->count = cnt;
/* Init pointers to cache buffers for a SUM */
if (MPIR_T_pvar_is_sum(hnd)) {
hnd->accum = (char*)(hnd) + sizeof(*hnd);
hnd->offset = (char*)(hnd) + sizeof(*hnd) + bytes*cnt;
hnd->current = (char*)(hnd) + sizeof(*hnd) + bytes*cnt*2;
}
if (MPIR_T_pvar_is_continuous(hnd))
MPIR_T_pvar_set_started(hnd);
/* Set starting value of a continuous SUM */
if (MPIR_T_pvar_is_continuous(hnd) && MPIR_T_pvar_is_sum(hnd)) {
/* Cache current value of a SUM in offset.
* accum is zero since we called CALLOC before.
*/
if (hnd->get_value == NULL)
MPIR_Memcpy(hnd->offset, hnd->addr, bytes*cnt);
else
hnd->get_value(hnd->addr, hnd->obj_handle, hnd->count, hnd->offset);
}
/* Link a WATERMARK handle to its pvar & set starting value if continuous */
if (MPIR_T_pvar_is_watermark(hnd)) {
MPIR_T_pvar_watermark_t *mark = (MPIR_T_pvar_watermark_t *)hnd->addr;
if (!mark->first_used) {
/* Use the special handle slot for optimization if available */
mark->first_used = TRUE;
MPIR_T_pvar_set_first(hnd);
/* Set starting value */
if (MPIR_T_pvar_is_continuous(hnd)) {
mark->first_started = TRUE;
mark->watermark = mark->current;
} else {
mark->first_started = FALSE;
}
} else {
/* If the special handle slot is unavailable, link it to hlist */
if (mark->hlist == NULL) {
hnd->prev2 = hnd;
mark->hlist = hnd;
} else {
hnd->prev2 = hnd;
hnd->next2 = mark->hlist;
mark->hlist->prev2 = hnd;
mark->hlist = hnd;
}
/* Set starting value */
if (MPIR_T_pvar_is_continuous(hnd))
hnd->watermark = mark->current;
}
}
/* Link the handle in its session and return it */
MPL_DL_APPEND(session->hlist, hnd);
*handle = hnd;
*count = cnt;
MPIR_CHKPMEM_COMMIT();
fn_exit:
return mpi_errno;
fn_fail:
MPIR_CHKPMEM_REAP();
goto fn_exit;
}
int MPID_Type_create_resized(MPI_Datatype oldtype,
MPI_Aint lb,
MPI_Aint extent,
MPI_Datatype *newtype_p)
{
MPID_Datatype *new_dtp;
new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
/* --BEGIN ERROR HANDLING-- */
if (!new_dtp) return MPIDI_Type_create_resized_memory_error();
/* --END ERROR HANDLING-- */
/* handle is filled in by MPIU_Handle_obj_alloc() */
MPIU_Object_set_ref(new_dtp, 1);
new_dtp->is_permanent = 0;
new_dtp->is_committed = 0;
new_dtp->attributes = 0;
new_dtp->cache_id = 0;
new_dtp->name[0] = 0;
new_dtp->contents = 0;
new_dtp->dataloop = NULL;
new_dtp->dataloop_size = -1;
new_dtp->dataloop_depth = -1;
new_dtp->hetero_dloop = NULL;
new_dtp->hetero_dloop_size = -1;
new_dtp->hetero_dloop_depth = -1;
/* if oldtype is a basic, we build a contiguous dataloop of count = 1 */
if (HANDLE_GET_KIND(oldtype) == HANDLE_KIND_BUILTIN)
{
int oldsize = MPID_Datatype_get_basic_size(oldtype);
new_dtp->size = oldsize;
new_dtp->has_sticky_ub = 0;
new_dtp->has_sticky_lb = 0;
new_dtp->dataloop_depth = 1;
new_dtp->true_lb = 0;
new_dtp->lb = lb;
new_dtp->true_ub = oldsize;
new_dtp->ub = lb + extent;
new_dtp->extent = extent;
new_dtp->alignsize = oldsize; /* FIXME ??? */
new_dtp->n_builtin_elements = 1;
new_dtp->builtin_element_size = oldsize;
new_dtp->is_contig = (extent == oldsize) ? 1 : 0;
new_dtp->basic_type = oldtype;
new_dtp->max_contig_blocks = 3; /* lb, data, ub */
}
else
{
/* user-defined base type */
MPID_Datatype *old_dtp;
MPID_Datatype_get_ptr(oldtype, old_dtp);
new_dtp->size = old_dtp->size;
new_dtp->has_sticky_ub = 0;
new_dtp->has_sticky_lb = 0;
new_dtp->dataloop_depth = old_dtp->dataloop_depth;
new_dtp->true_lb = old_dtp->true_lb;
new_dtp->lb = lb;
new_dtp->true_ub = old_dtp->true_ub;
new_dtp->ub = lb + extent;
new_dtp->extent = extent;
new_dtp->alignsize = old_dtp->alignsize;
new_dtp->n_builtin_elements = old_dtp->n_builtin_elements;
new_dtp->builtin_element_size = old_dtp->builtin_element_size;
new_dtp->basic_type = old_dtp->basic_type;
new_dtp->is_contig =
(extent == old_dtp->size) ? old_dtp->is_contig : 0;
new_dtp->max_contig_blocks = old_dtp->max_contig_blocks;
}
*newtype_p = new_dtp->handle;
MPIU_DBG_MSG_P(DATATYPE,VERBOSE,"resized type %x created.",
new_dtp->handle);
return MPI_SUCCESS;
}
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
MPI_Datatype origin_datatype, void *result_addr, int result_count,
MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win *win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int rank, origin_predefined, result_predefined, target_predefined;
int shm_locked = 0;
int dt_contig ATTRIBUTE((unused));
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPID_Datatype *dtp;
MPIU_CHKLMEM_DECL(2);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);
if (target_rank == MPI_PROC_NULL) {
goto fn_exit;
}
if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
}
MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");
MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz,
dtp, dt_true_lb);
if (data_sz == 0) {
goto fn_exit;
}
rank = win_ptr->myrank;
origin_predefined = TRUE; /* quiet uninitialized warnings (b/c goto) */
if (op != MPI_NO_OP) {
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(origin_datatype, origin_predefined);
}
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(result_datatype, result_predefined);
MPIDI_CH3I_DATATYPE_IS_PREDEFINED(target_datatype, target_predefined);
/* Do =! rank first (most likely branch?) */
if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED)
{
MPI_User_function *uop;
void *base;
int disp_unit;
if (win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) {
base = win_ptr->shm_base_addrs[target_rank];
disp_unit = win_ptr->disp_units[target_rank];
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
shm_locked = 1;
}
else {
base = win_ptr->base;
disp_unit = win_ptr->disp_unit;
}
/* Perform the local get first, then the accumulate */
mpi_errno = MPIR_Localcopy((char *) base + disp_unit * target_disp,
target_count, target_datatype,
result_addr, result_count, result_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* NO_OP: Don't perform the accumulate */
if (op == MPI_NO_OP) {
if (shm_locked) {
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
shm_locked = 0;
}
goto fn_exit;
}
if (op == MPI_REPLACE) {
mpi_errno = MPIR_Localcopy(origin_addr, origin_count, origin_datatype,
(char *) base + disp_unit * target_disp,
target_count, target_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
if (shm_locked) {
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
shm_locked = 0;
}
goto fn_exit;
}
MPIU_ERR_CHKANDJUMP1((HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN),
mpi_errno, MPI_ERR_OP, "**opnotpredefined",
"**opnotpredefined %d", op );
/* get the function by indexing into the op table */
uop = MPIR_OP_HDL_TO_FN(op);
if (origin_predefined && target_predefined) {
/* Cast away const'ness for origin_address in order to
* avoid changing the prototype for MPI_User_function */
(*uop)((void *) origin_addr, (char *) base + disp_unit*target_disp,
&target_count, &target_datatype);
}
else {
/* derived datatype */
MPID_Segment *segp;
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
int vec_len, i, type_size, count;
MPI_Datatype type;
MPI_Aint true_lb, true_extent, extent;
void *tmp_buf=NULL, *target_buf;
const void *source_buf;
if (origin_datatype != target_datatype) {
/* first copy the data into a temporary buffer with
the same datatype as the target. Then do the
accumulate operation. */
MPIR_Type_get_true_extent_impl(target_datatype, &true_lb, &true_extent);
MPID_Datatype_get_extent_macro(target_datatype, extent);
MPIU_CHKLMEM_MALLOC(tmp_buf, void *,
target_count * (MPIR_MAX(extent,true_extent)),
mpi_errno, "temporary buffer");
/* adjust for potential negative lower bound in datatype */
tmp_buf = (void *)((char*)tmp_buf - true_lb);
mpi_errno = MPIR_Localcopy(origin_addr, origin_count,
origin_datatype, tmp_buf,
target_count, target_datatype);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
if (target_predefined) {
/* target predefined type, origin derived datatype */
(*uop)(tmp_buf, (char *) base + disp_unit * target_disp,
&target_count, &target_datatype);
}
else {
segp = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((!segp), mpi_errno, MPI_ERR_OTHER,
"**nomem","**nomem %s","MPID_Segment_alloc");
MPID_Segment_init(NULL, target_count, target_datatype, segp, 0);
first = 0;
last = SEGMENT_IGNORE_LAST;
MPID_Datatype_get_ptr(target_datatype, dtp);
vec_len = dtp->max_contig_blocks * target_count + 1;
/* +1 needed because Rob says so */
MPIU_CHKLMEM_MALLOC(dloop_vec, DLOOP_VECTOR *,
vec_len * sizeof(DLOOP_VECTOR),
mpi_errno, "dloop vector");
MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
source_buf = (tmp_buf != NULL) ? tmp_buf : origin_addr;
target_buf = (char *) base + disp_unit * target_disp;
type = dtp->eltype;
type_size = MPID_Datatype_get_basic_size(type);
for (i=0; i<vec_len; i++) {
count = (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size;
(*uop)((char *)source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *)target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
}
MPID_Segment_free(segp);
}
}
int MPID_Type_indexed(int count,
const int *blocklength_array,
const void *displacement_array,
int dispinbytes,
MPI_Datatype oldtype,
MPI_Datatype *newtype)
{
int mpi_errno = MPI_SUCCESS;
int is_builtin, old_is_contig;
int i, contig_count;
int el_sz, el_ct, old_ct, old_sz;
MPI_Aint old_lb, old_ub, old_extent, old_true_lb, old_true_ub;
MPI_Aint min_lb = 0, max_ub = 0, eff_disp;
MPI_Datatype el_type;
MPID_Datatype *new_dtp;
if (count == 0) return MPID_Type_zerolen(newtype);
/* sanity check that blocklens are all non-negative */
for (i = 0; i < count; ++i) {
DLOOP_Assert(blocklength_array[i] >= 0);
}
/* allocate new datatype object and handle */
new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
/* --BEGIN ERROR HANDLING-- */
if (!new_dtp)
{
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
"MPID_Type_indexed",
__LINE__,
MPI_ERR_OTHER,
"**nomem",
0);
return mpi_errno;
}
/* --END ERROR HANDLING-- */
/* handle is filled in by MPIU_Handle_obj_alloc() */
MPIU_Object_set_ref(new_dtp, 1);
new_dtp->is_permanent = 0;
new_dtp->is_committed = 0;
new_dtp->attributes = NULL;
new_dtp->cache_id = 0;
new_dtp->name[0] = 0;
new_dtp->contents = NULL;
new_dtp->dataloop = NULL;
new_dtp->dataloop_size = -1;
new_dtp->dataloop_depth = -1;
new_dtp->hetero_dloop = NULL;
new_dtp->hetero_dloop_size = -1;
new_dtp->hetero_dloop_depth = -1;
is_builtin = (HANDLE_GET_KIND(oldtype) == HANDLE_KIND_BUILTIN);
if (is_builtin)
{
/* builtins are handled differently than user-defined types because
* they have no associated dataloop or datatype structure.
*/
el_sz = MPID_Datatype_get_basic_size(oldtype);
old_sz = el_sz;
el_ct = 1;
el_type = oldtype;
old_lb = 0;
old_true_lb = 0;
old_ub = (MPI_Aint) el_sz;
old_true_ub = (MPI_Aint) el_sz;
old_extent = (MPI_Aint) el_sz;
old_is_contig = 1;
new_dtp->has_sticky_ub = 0;
new_dtp->has_sticky_lb = 0;
new_dtp->alignsize = el_sz; /* ??? */
new_dtp->element_size = (MPI_Aint) el_sz;
new_dtp->eltype = el_type;
new_dtp->max_contig_blocks = count;
}
else
{
/* user-defined base type (oldtype) */
MPID_Datatype *old_dtp;
MPID_Datatype_get_ptr(oldtype, old_dtp);
/* Ensure that "element_size" fits into an int datatype. */
MPID_Ensure_Aint_fits_in_int(old_dtp->element_size);
el_sz = old_dtp->element_size;
old_sz = old_dtp->size;
el_ct = old_dtp->n_elements;
el_type = old_dtp->eltype;
old_lb = old_dtp->lb;
old_true_lb = old_dtp->true_lb;
old_ub = old_dtp->ub;
old_true_ub = old_dtp->true_ub;
old_extent = old_dtp->extent;
old_is_contig = old_dtp->is_contig;
new_dtp->has_sticky_lb = old_dtp->has_sticky_lb;
new_dtp->has_sticky_ub = old_dtp->has_sticky_ub;
new_dtp->element_size = (MPI_Aint) el_sz;
new_dtp->eltype = el_type;
new_dtp->max_contig_blocks = 0;
for(i=0; i<count; i++)
new_dtp->max_contig_blocks
+= old_dtp->max_contig_blocks
* ((MPI_Aint ) blocklength_array[i]);
}
/* find the first nonzero blocklength element */
i = 0;
while (i < count && blocklength_array[i] == 0) i++;
if (i == count) {
MPIU_Handle_obj_free(&MPID_Datatype_mem, new_dtp);
return MPID_Type_zerolen(newtype);
}
/* priming for loop */
old_ct = blocklength_array[i];
eff_disp = (dispinbytes) ? ((MPI_Aint *) displacement_array)[i] :
(((MPI_Aint) ((int *) displacement_array)[i]) * old_extent);
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint) blocklength_array[i],
eff_disp,
old_lb,
old_ub,
old_extent,
min_lb,
max_ub);
/* determine min lb, max ub, and count of old types in remaining
* nonzero size blocks
*/
for (i++; i < count; i++)
{
MPI_Aint tmp_lb, tmp_ub;
if (blocklength_array[i] > 0) {
old_ct += blocklength_array[i]; /* add more oldtypes */
eff_disp = (dispinbytes) ? ((MPI_Aint *) displacement_array)[i] :
(((MPI_Aint) ((int *) displacement_array)[i]) * old_extent);
/* calculate ub and lb for this block */
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint)(blocklength_array[i]),
eff_disp,
old_lb,
old_ub,
old_extent,
tmp_lb,
tmp_ub);
if (tmp_lb < min_lb) min_lb = tmp_lb;
if (tmp_ub > max_ub) max_ub = tmp_ub;
}
}
new_dtp->size = old_ct * old_sz;
new_dtp->lb = min_lb;
new_dtp->ub = max_ub;
new_dtp->true_lb = min_lb + (old_true_lb - old_lb);
new_dtp->true_ub = max_ub + (old_true_ub - old_ub);
new_dtp->extent = max_ub - min_lb;
new_dtp->n_elements = old_ct * el_ct;
/* new type is only contig for N types if it's all one big
* block, its size and extent are the same, and the old type
* was also contiguous.
*/
new_dtp->is_contig = 0;
if(old_is_contig)
{
contig_count = MPID_Type_indexed_count_contig(count,
blocklength_array,
displacement_array,
dispinbytes,
old_extent);
new_dtp->max_contig_blocks = contig_count;
if( (contig_count == 1) &&
((MPI_Aint) new_dtp->size == new_dtp->extent))
{
new_dtp->is_contig = 1;
}
}
*newtype = new_dtp->handle;
return mpi_errno;
}
/*@
MPI_Type_ub - Returns the upper bound of a datatype
Input Parameters:
. datatype - datatype (handle)
Output Parameter:
. displacement - displacement of upper bound from origin,
in bytes (address integer)
.N Deprecated
The replacement for this routine is 'MPI_Type_get_extent'
.N SignalSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_TYPE
.N MPI_ERR_ARG
@*/
int MPI_Type_ub(MPI_Datatype datatype, MPI_Aint *displacement)
{
int mpi_errno = MPI_SUCCESS;
MPID_Datatype *datatype_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_TYPE_UB);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_TYPE_UB);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Datatype_get_ptr(datatype, datatype_ptr);
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate datatype_ptr */
MPID_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
if (HANDLE_GET_KIND(datatype) == HANDLE_KIND_BUILTIN)
*displacement = MPID_Datatype_get_basic_size(datatype);
else
*displacement = datatype_ptr->ub;
/* ... end of body of routine ... */
#ifdef HAVE_ERROR_CHECKING
fn_exit:
#endif
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_TYPE_UB);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
fn_fail:
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_type_ub",
"**mpi_type_ub %D %p", datatype, displacement);
}
mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
goto fn_exit;
# endif
/* --END ERROR HANDLING-- */
}
/* MPIR_Type_get_basic_type_elements()
*
* Arguments:
* - bytes_p - input/output byte count
* - count - maximum number of this type to subtract from the bytes; a count
* of -1 indicates use as many as we like
* - datatype - input datatype
*
* Returns number of elements available given the two constraints of number of
* bytes and count of types. Also reduces the byte count by the amount taken
* up by the types.
*
* Assumptions:
* - the type passed to this function must be a basic *or* a pairtype
* (which aren't basic types)
* - the count is not zero (otherwise we can't tell between a "no more
* complete types" case and a "zero count" case)
*
* As per section 4.9.3 of the MPI 1.1 specification, the two-part reduction
* types are to be treated as structs of the constituent types. So we have to
* do something special to handle them correctly in here.
*
* As per section 3.12.5 get_count and get_elements report the same value for
* basic datatypes; I'm currently interpreting this to *not* include these
* reduction types, as they are considered structs.
*/
PMPI_LOCAL MPI_Count MPIR_Type_get_basic_type_elements(MPI_Count *bytes_p,
MPI_Count count,
MPI_Datatype datatype)
{
MPI_Count elements, usable_bytes, used_bytes, type1_sz, type2_sz;
if (count == 0) return 0;
/* determine the maximum number of bytes we should take from the
* byte count.
*/
if (count < 0) {
usable_bytes = *bytes_p;
}
else {
usable_bytes = MPL_MIN(*bytes_p,
count * MPID_Datatype_get_basic_size(datatype));
}
switch (datatype) {
/* we don't get valid fortran datatype handles in all cases... */
#ifdef HAVE_FORTRAN_BINDING
case MPI_2REAL:
type1_sz = type2_sz = MPID_Datatype_get_basic_size(MPI_REAL);
break;
case MPI_2DOUBLE_PRECISION:
type1_sz = type2_sz =
MPID_Datatype_get_basic_size(MPI_DOUBLE_PRECISION);
break;
case MPI_2INTEGER:
type1_sz = type2_sz = MPID_Datatype_get_basic_size(MPI_INTEGER);
break;
#endif
case MPI_2INT:
type1_sz = type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
case MPI_FLOAT_INT:
type1_sz = MPID_Datatype_get_basic_size(MPI_FLOAT);
type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
case MPI_DOUBLE_INT:
type1_sz = MPID_Datatype_get_basic_size(MPI_DOUBLE);
type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
case MPI_LONG_INT:
type1_sz = MPID_Datatype_get_basic_size(MPI_LONG);
type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
case MPI_SHORT_INT:
type1_sz = MPID_Datatype_get_basic_size(MPI_SHORT);
type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
case MPI_LONG_DOUBLE_INT:
type1_sz = MPID_Datatype_get_basic_size(MPI_LONG_DOUBLE);
type2_sz = MPID_Datatype_get_basic_size(MPI_INT);
break;
default:
/* all other types. this is more complicated than
* necessary for handling these types, but it puts us in the
* same code path for all the basics, so we stick with it.
*/
type1_sz = type2_sz = MPID_Datatype_get_basic_size(datatype);
break;
}
/* determine the number of elements in the region */
elements = 2 * (usable_bytes / (type1_sz + type2_sz));
if (usable_bytes % (type1_sz + type2_sz) >= type1_sz) elements++;
/* determine how many bytes we used up with those elements */
used_bytes = ((elements / 2) * (type1_sz + type2_sz));
if (elements % 2 == 1) used_bytes += type1_sz;
*bytes_p -= used_bytes;
return elements;
}
/* MPID_Segment_vector_pack_to_iov
*
* Input Parameters:
* blocks_p - [inout] pointer to a count of blocks (total, for all noncontiguous pieces)
* count - # of noncontiguous regions
* blksz - size of each noncontiguous region
* stride - distance in bytes from start of one region to start of next
* el_type - elemental type (e.g. MPI_INT)
* ...
*
* Note: this is only called when the starting position is at the beginning
* of a whole block in a vector type.
*/
static int MPID_Segment_vector_pack_to_iov(DLOOP_Offset *blocks_p,
DLOOP_Count count,
DLOOP_Size blksz,
DLOOP_Offset stride,
DLOOP_Type el_type,
DLOOP_Offset rel_off, /* offset into buffer */
void *bufp, /* start of buffer */
void *v_paramp)
{
int i;
DLOOP_Offset size, blocks_left, basic_size;
struct MPID_Segment_piece_params *paramp = v_paramp;
MPIDI_STATE_DECL(MPID_STATE_MPID_SEGMENT_VECTOR_PACK_TO_IOV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_SEGMENT_VECTOR_PACK_TO_IOV);
basic_size = (DLOOP_Offset) MPID_Datatype_get_basic_size(el_type);
blocks_left = *blocks_p;
MPL_DBG_MSG_FMT(MPIR_DBG_DATATYPE,VERBOSE,(MPL_DBG_FDEST,
" vector to vec: do=" MPI_AINT_FMT_DEC_SPEC
", dp=%p"
", len=" MPI_AINT_FMT_DEC_SPEC
", ind=" MPI_AINT_FMT_DEC_SPEC
", ct=" MPI_AINT_FMT_DEC_SPEC
", blksz=" MPI_AINT_FMT_DEC_SPEC
", str=" MPI_AINT_FMT_DEC_SPEC
", blks=" MPI_AINT_FMT_DEC_SPEC,
(MPI_Aint) rel_off,
bufp,
(MPI_Aint) paramp->u.pack_vector.length,
(MPI_Aint) paramp->u.pack_vector.index,
count,
blksz,
(MPI_Aint) stride,
(MPI_Aint) *blocks_p));
for (i=0; i < count && blocks_left > 0; i++) {
int last_idx;
char *last_end = NULL;
if (blocks_left > (DLOOP_Offset) blksz) {
size = ((DLOOP_Offset) blksz) * basic_size;
blocks_left -= (DLOOP_Offset) blksz;
}
else {
/* last pass */
size = blocks_left * basic_size;
blocks_left = 0;
}
last_idx = paramp->u.pack_vector.index - 1;
if (last_idx >= 0) {
last_end = ((char *) paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_BUF) +
paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_LEN;
}
MPIU_Ensure_Aint_fits_in_pointer((MPIU_VOID_PTR_CAST_TO_MPI_AINT (bufp)) + rel_off);
if ((last_idx == paramp->u.pack_vector.length-1) &&
(last_end != ((char *) bufp + rel_off)))
{
/* we have used up all our entries, and this one doesn't fit on
* the end of the last one.
*/
*blocks_p -= (blocks_left + (size / basic_size));
#ifdef MPID_SP_VERBOSE
MPL_DBG_MSG_FMT(MPIR_DBG_DATATYPE,VERBOSE,(MPL_DBG_FDEST,"\t[vector to vec exiting (1): next ind = %d, " MPI_AINT_FMT_DEC_SPEC " blocks processed.\n",
paramp->u.pack_vector.index,
(MPI_Aint) *blocks_p));
#endif
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_VECTOR_PACK_TO_IOV);
return 1;
}
else if (last_idx >= 0 && (last_end == ((char *) bufp + rel_off)))
{
/* add this size to the last vector rather than using up new one */
paramp->u.pack_vector.vectorp[last_idx].DLOOP_VECTOR_LEN += size;
}
else {
paramp->u.pack_vector.vectorp[last_idx+1].DLOOP_VECTOR_BUF =
(char *) bufp + rel_off;
paramp->u.pack_vector.vectorp[last_idx+1].DLOOP_VECTOR_LEN = size;
paramp->u.pack_vector.index++;
}
rel_off += stride;
}
#ifdef MPID_SP_VERBOSE
MPL_DBG_MSG_FMT(MPIR_DBG_DATATYPE,VERBOSE,(MPL_DBG_FDEST,"\t[vector to vec exiting (2): next ind = %d, " MPI_AINT_FMT_DEC_SPEC " blocks processed.\n",
paramp->u.pack_vector.index,
(MPI_Aint) *blocks_p));
#endif
/* if we get here then we processed ALL the blocks; don't need to update
* blocks_p
*/
MPIU_Assert(blocks_left == 0);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_SEGMENT_VECTOR_PACK_TO_IOV);
return 0;
}
static inline void
MPIDI_RecvShortCB(pami_context_t context,
const void * _msginfo,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
unsigned isSync)
{
MPID_assert(_msginfo != NULL);
const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo;
MPID_Request * rreq = NULL;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno);
#endif
/* Match not found */
if (unlikely(rreq == NULL))
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request *newreq = MPIDI_Request_create2();
MPID_assert(newreq != NULL);
if (sndlen)
{
newreq->mpid.uebuflen = sndlen;
if (!TOKEN_FLOW_CONTROL_ON)
{
newreq->mpid.uebuf = MPL_malloc(sndlen);
newreq->mpid.uebuf_malloc = mpiuMalloc;
}
else
{
#if TOKEN_FLOW_CONTROL
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen);
newreq->mpid.uebuf_malloc = mpidiBufMM;
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#else
MPID_assert_always(0);
#endif
}
MPID_assert(newreq->mpid.uebuf != NULL);
}
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno);
#endif
if (unlikely(rreq == NULL))
{
MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync);
/* request is always complete now */
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Token_cntr[source].unmatched++;
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_release(newreq);
goto fn_exit_short;
}
else
{
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_discard(newreq);
}
}
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Update_rettoks(source);
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* the receive queue processing has been completed and we found match*/
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, sndlen);
MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE);
MPIDI_Request_cpyPeerRequestH(rreq, msginfo);
MPIDI_Request_setSync (rreq, isSync);
MPIDI_Request_setRzv (rreq, 0);
/* ----------------------------- */
/* Request was already posted. */
/* ----------------------------- */
if (unlikely(isSync))
MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender));
if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN))
{
MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq);
goto fn_exit_short;
}
size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
goto fn_exit_short;
#else
sndlen = dt_size;
#endif
}
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf;
if (sndlen > 0)
{
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf))
{
cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice);
}
else
#endif
memcpy(rcvbuf, sndbuf, sndlen);
}
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
MPIDI_Request_complete(rreq);
fn_exit_short:
#ifdef OUT_OF_ORDER_HANDLING
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#endif
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
/*@
MPID_Type_struct - create a struct datatype
Input Parameters:
+ count - number of blocks in vector
. blocklength_array - number of elements in each block
. displacement_array - offsets of blocks from start of type in bytes
- oldtype_array - types (using handle) of datatypes on which vector is based
Output Parameters:
. newtype - handle of new struct datatype
Return Value:
MPI_SUCCESS on success, MPI errno on failure.
@*/
int MPID_Type_struct(int count,
const int *blocklength_array,
const MPI_Aint *displacement_array,
const MPI_Datatype *oldtype_array,
MPI_Datatype *newtype)
{
int mpi_errno = MPI_SUCCESS;
int i, old_are_contig = 1, definitely_not_contig = 0;
int found_sticky_lb = 0, found_sticky_ub = 0, found_true_lb = 0,
found_true_ub = 0, found_el_type = 0, found_lb=0, found_ub=0;
MPI_Aint el_sz = 0;
MPI_Aint size = 0;
MPI_Datatype el_type = MPI_DATATYPE_NULL;
MPI_Aint true_lb_disp = 0, true_ub_disp = 0, sticky_lb_disp = 0,
sticky_ub_disp = 0, lb_disp = 0, ub_disp = 0;
MPID_Datatype *new_dtp;
if (count == 0) return MPID_Type_zerolen(newtype);
#ifdef MPID_STRUCT_DEBUG
MPIDI_Datatype_printf(oldtype_array[0], 1, displacement_array[0],
blocklength_array[0], 1);
for (i=1; i < count; i++)
{
MPIDI_Datatype_printf(oldtype_array[i], 1, displacement_array[i],
blocklength_array[i], 0);
}
#endif
/* allocate new datatype object and handle */
new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
/* --BEGIN ERROR HANDLING-- */
if (!new_dtp)
{
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
"MPID_Type_struct",
__LINE__, MPI_ERR_OTHER,
"**nomem", 0);
return mpi_errno;
}
/* --END ERROR HANDLING-- */
/* handle is filled in by MPIU_Handle_obj_alloc() */
MPIU_Object_set_ref(new_dtp, 1);
new_dtp->is_permanent = 0;
new_dtp->is_committed = 0;
new_dtp->attributes = NULL;
new_dtp->cache_id = 0;
new_dtp->name[0] = 0;
new_dtp->contents = NULL;
new_dtp->dataloop = NULL;
new_dtp->dataloop_size = -1;
new_dtp->dataloop_depth = -1;
new_dtp->hetero_dloop = NULL;
new_dtp->hetero_dloop_size = -1;
new_dtp->hetero_dloop_depth = -1;
/* check for junk struct with all zero blocks */
for (i=0; i < count; i++) if (blocklength_array[i] != 0) break;
if (i == count)
{
MPIU_Handle_obj_free(&MPID_Datatype_mem, new_dtp);
return MPID_Type_zerolen(newtype);
}
new_dtp->max_contig_blocks = 0;
for (i=0; i < count; i++)
{
int is_builtin =
(HANDLE_GET_KIND(oldtype_array[i]) == HANDLE_KIND_BUILTIN);
MPI_Aint tmp_lb, tmp_ub, tmp_true_lb, tmp_true_ub;
MPI_Aint tmp_el_sz;
MPI_Datatype tmp_el_type;
MPID_Datatype *old_dtp = NULL;
/* Interpreting typemap to not include 0 blklen things, including
* MPI_LB and MPI_UB. -- Rob Ross, 10/31/2005
*/
if (blocklength_array[i] == 0) continue;
if (is_builtin)
{
tmp_el_sz = MPID_Datatype_get_basic_size(oldtype_array[i]);
tmp_el_type = oldtype_array[i];
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint)(blocklength_array[i]),
displacement_array[i],
0,
tmp_el_sz,
tmp_el_sz,
tmp_lb,
tmp_ub);
tmp_true_lb = tmp_lb;
tmp_true_ub = tmp_ub;
size += tmp_el_sz * blocklength_array[i];
new_dtp->max_contig_blocks++;
}
else
{
MPID_Datatype_get_ptr(oldtype_array[i], old_dtp);
/* Ensure that "builtin_element_size" fits into an int datatype. */
MPID_Ensure_Aint_fits_in_int(old_dtp->builtin_element_size);
tmp_el_sz = old_dtp->builtin_element_size;
tmp_el_type = old_dtp->basic_type;
MPID_DATATYPE_BLOCK_LB_UB((MPI_Aint) blocklength_array[i],
displacement_array[i],
old_dtp->lb,
old_dtp->ub,
old_dtp->extent,
tmp_lb,
tmp_ub);
tmp_true_lb = tmp_lb + (old_dtp->true_lb - old_dtp->lb);
tmp_true_ub = tmp_ub + (old_dtp->true_ub - old_dtp->ub);
size += old_dtp->size * blocklength_array[i];
new_dtp->max_contig_blocks += old_dtp->max_contig_blocks;
}
/* element size and type */
if (oldtype_array[i] != MPI_LB && oldtype_array[i] != MPI_UB)
{
if (found_el_type == 0)
{
el_sz = tmp_el_sz;
el_type = tmp_el_type;
found_el_type = 1;
}
else if (el_sz != tmp_el_sz)
{
el_sz = -1;
el_type = MPI_DATATYPE_NULL;
}
else if (el_type != tmp_el_type)
{
/* Q: should we set el_sz = -1 even though the same? */
el_type = MPI_DATATYPE_NULL;
}
}
/* keep lowest sticky lb */
if ((oldtype_array[i] == MPI_LB) ||
(!is_builtin && old_dtp->has_sticky_lb))
{
if (!found_sticky_lb)
{
found_sticky_lb = 1;
sticky_lb_disp = tmp_lb;
}
else if (sticky_lb_disp > tmp_lb)
{
sticky_lb_disp = tmp_lb;
}
}
/* keep highest sticky ub */
if ((oldtype_array[i] == MPI_UB) ||
(!is_builtin && old_dtp->has_sticky_ub))
{
if (!found_sticky_ub)
{
found_sticky_ub = 1;
sticky_ub_disp = tmp_ub;
}
else if (sticky_ub_disp < tmp_ub)
{
sticky_ub_disp = tmp_ub;
}
}
/* keep lowest lb/true_lb and highest ub/true_ub
*
* note: checking for contiguity at the same time, to avoid
* yet another pass over the arrays
*/
if (oldtype_array[i] != MPI_UB && oldtype_array[i] != MPI_LB)
{
if (!found_true_lb)
{
found_true_lb = 1;
true_lb_disp = tmp_true_lb;
}
else if (true_lb_disp > tmp_true_lb)
{
/* element starts before previous */
true_lb_disp = tmp_true_lb;
definitely_not_contig = 1;
}
if (!found_lb)
{
found_lb = 1;
lb_disp = tmp_lb;
}
else if (lb_disp > tmp_lb)
{
/* lb before previous */
lb_disp = tmp_lb;
definitely_not_contig = 1;
}
if (!found_true_ub)
{
found_true_ub = 1;
true_ub_disp = tmp_true_ub;
}
else if (true_ub_disp < tmp_true_ub)
{
true_ub_disp = tmp_true_ub;
}
else {
/* element ends before previous ended */
definitely_not_contig = 1;
}
if (!found_ub)
{
found_ub = 1;
ub_disp = tmp_ub;
}
else if (ub_disp < tmp_ub)
{
ub_disp = tmp_ub;
}
else {
/* ub before previous */
definitely_not_contig = 1;
}
}
if (!is_builtin && !old_dtp->is_contig)
{
old_are_contig = 0;
}
}
new_dtp->n_builtin_elements = -1; /* TODO */
new_dtp->builtin_element_size = el_sz;
new_dtp->basic_type = el_type;
new_dtp->has_sticky_lb = found_sticky_lb;
new_dtp->true_lb = true_lb_disp;
new_dtp->lb = (found_sticky_lb) ? sticky_lb_disp : lb_disp;
new_dtp->has_sticky_ub = found_sticky_ub;
new_dtp->true_ub = true_ub_disp;
new_dtp->ub = (found_sticky_ub) ? sticky_ub_disp : ub_disp;
new_dtp->alignsize = MPID_Type_struct_alignsize(count,
oldtype_array,
displacement_array);
new_dtp->extent = new_dtp->ub - new_dtp->lb;
if ((!found_sticky_lb) && (!found_sticky_ub))
{
/* account for padding */
MPI_Aint epsilon = (new_dtp->alignsize > 0) ?
new_dtp->extent % ((MPI_Aint)(new_dtp->alignsize)) : 0;
if (epsilon)
{
new_dtp->ub += ((MPI_Aint)(new_dtp->alignsize) - epsilon);
new_dtp->extent = new_dtp->ub - new_dtp->lb;
}
}
new_dtp->size = size;
/* new type is contig for N types if its size and extent are the
* same, and the old type was also contiguous, and we didn't see
* something noncontiguous based on true ub/ub.
*/
if (((MPI_Aint)(new_dtp->size) == new_dtp->extent) &&
old_are_contig && (! definitely_not_contig))
{
new_dtp->is_contig = 1;
}
else
{
new_dtp->is_contig = 0;
}
*newtype = new_dtp->handle;
return mpi_errno;
}
