int MPIR_Dist_graph_neighbors_impl(MPIR_Comm * comm_ptr,
int maxindegree, int sources[], int sourceweights[],
int maxoutdegree, int destinations[], int destweights[])
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *topo_ptr = NULL;
topo_ptr = MPIR_Topology_get(comm_ptr);
MPIR_ERR_CHKANDJUMP(!topo_ptr ||
topo_ptr->kind != MPI_DIST_GRAPH, mpi_errno, MPI_ERR_TOPOLOGY,
"**notdistgraphtopo");
MPIR_Memcpy(sources, topo_ptr->topo.dist_graph.in, maxindegree * sizeof(int));
MPIR_Memcpy(destinations, topo_ptr->topo.dist_graph.out, maxoutdegree * sizeof(int));
if (sourceweights != MPI_UNWEIGHTED && topo_ptr->topo.dist_graph.is_weighted) {
MPIR_Memcpy(sourceweights, topo_ptr->topo.dist_graph.in_weights, maxindegree * sizeof(int));
}
if (destweights != MPI_UNWEIGHTED && topo_ptr->topo.dist_graph.is_weighted) {
MPIR_Memcpy(destweights, topo_ptr->topo.dist_graph.out_weights, maxoutdegree * sizeof(int));
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
void MPIDI_Datatype_get_contents_types(MPIDU_Datatype_contents *cp,
MPI_Datatype *user_types)
{
char *ptr;
int align_sz = 8, epsilon;
int struct_sz;
#ifdef HAVE_MAX_STRUCT_ALIGNMENT
if (align_sz > HAVE_MAX_STRUCT_ALIGNMENT) {
align_sz = HAVE_MAX_STRUCT_ALIGNMENT;
}
#endif
struct_sz = sizeof(MPIDU_Datatype_contents);
/* pad the struct, types, and ints before we allocate.
*
* note: it's not necessary that we pad the aints,
* because they are last in the region.
*/
if ((epsilon = struct_sz % align_sz)) {
struct_sz += align_sz - epsilon;
}
ptr = ((char *) cp) + struct_sz;
MPIR_Memcpy(user_types, ptr, cp->nr_types * sizeof(MPI_Datatype));
return;
}
static int MPIDU_Segment_contig_unpack_external32_to_buf(DLOOP_Offset *blocks_p,
DLOOP_Type el_type,
DLOOP_Offset rel_off,
void *bufp,
void *v_paramp)
{
int src_el_size, dest_el_size;
struct MPIDU_Segment_piece_params *paramp = v_paramp;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_SEGMENT_CONTIG_UNPACK_EXTERNAL32_TO_BUF);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_SEGMENT_CONTIG_UNPACK_EXTERNAL32_TO_BUF);
src_el_size = MPIDU_Datatype_get_basic_size(el_type);
dest_el_size = MPIDI_Datatype_get_basic_size_external32(el_type);
MPIR_Assert(dest_el_size);
/*
* h = handle value
* do = datatype buffer offset
* dp = datatype buffer pointer
* up = unpack buffer pointer (current location, incremented as we go)
* sz = size of datatype (guess we could get this from handle value if
* we wanted...)
*/
#ifdef MPID_SP_VERBOSE
dbg_printf("\t[contig unpack [external32]: do=%d, dp=%x, up=%x, "
"src_el_sz=%d, dest_el_sz=%d, blksz=%d]\n",
rel_off,
(unsigned) bufp,
(unsigned) paramp->u.unpack.unpack_buffer,
src_el_size,
dest_el_size,
(int) *blocks_p);
#endif
/* TODO: DEAL WITH CASE WHERE ALL DATA DOESN'T FIT! */
if ((src_el_size == dest_el_size) && (src_el_size == 1))
{
MPIR_Memcpy(((char *)bufp) + rel_off,
paramp->u.unpack.unpack_buffer, *blocks_p);
}
else if (is_float_type(el_type))
{
external32_float_convert(((char *) bufp) + rel_off,
paramp->u.unpack.unpack_buffer,
dest_el_size, src_el_size, *blocks_p);
}
else
{
external32_basic_convert(((char *) bufp) + rel_off,
paramp->u.unpack.unpack_buffer,
dest_el_size, src_el_size, *blocks_p);
}
paramp->u.unpack.unpack_buffer += (dest_el_size * (*blocks_p));
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_SEGMENT_CONTIG_UNPACK_EXTERNAL32_TO_BUF);
return 0;
}
static int handle_mprobe(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *const req = e->user_ptr;
MPIR_CHKPMEM_DECL(1);
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLE_PROBE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLE_PROBE);
if (e->ni_fail_type == PTL_NI_NO_MATCH) {
REQ_PTL(req)->found = FALSE;
goto finish_mprobe;
}
REQ_PTL(req)->found = TRUE;
req->status.MPI_SOURCE = NPTL_MATCH_GET_RANK(e->match_bits);
req->status.MPI_TAG = NPTL_MATCH_GET_TAG(e->match_bits);
MPIR_STATUS_SET_COUNT(req->status, NPTL_HEADER_GET_LENGTH(e->hdr_data));
MPIDI_Request_set_sync_send_flag(req, e->hdr_data & NPTL_SSEND);
MPIR_CHKPMEM_MALLOC(req->dev.tmpbuf, void *, e->mlength, mpi_errno, "tmpbuf", MPL_MEM_BUFFER);
MPIR_Memcpy((char *)req->dev.tmpbuf, e->start, e->mlength);
req->dev.recv_data_sz = e->mlength;
if (!(e->hdr_data & NPTL_LARGE)) {
MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_EAGER_MSG);
}
else {
MPIR_Assert (e->mlength == PTL_LARGE_THRESHOLD);
req->dev.match.parts.tag = req->status.MPI_TAG;
req->dev.match.parts.context_id = NPTL_MATCH_GET_CTX(e->match_bits);
req->dev.match.parts.rank = req->status.MPI_SOURCE;
MPIDI_Request_set_msg_type(req, MPIDI_REQUEST_RNDV_MSG);
}
/* At this point we know the ME is unlinked. Invalidate the handle to
prevent further accesses, e.g. an attempted cancel. */
REQ_PTL(req)->put_me = PTL_INVALID_HANDLE;
req->dev.recv_pending_count = 1;
finish_mprobe:
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
fn_exit:
MPIR_CHKPMEM_COMMIT();
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLE_PROBE);
return mpi_errno;
fn_fail:
MPIR_CHKPMEM_REAP();
goto fn_exit;
}
static int handler_recv_dequeue_complete(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *const rreq = e->user_ptr;
int is_contig;
MPI_Aint last;
MPI_Aint dt_true_lb;
intptr_t data_sz;
MPIDU_Datatype*dt_ptr ATTRIBUTE((unused));
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIR_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, is_contig, data_sz, dt_ptr, dt_true_lb);
dequeue_req(e);
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
/* unpack the data from unexpected buffer */
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "is_contig = %d", is_contig);
if (is_contig) {
MPIR_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPIDU_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, e->start);
if (last != e->mlength)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
} else {
/* Data was placed directly into the user buffer, so datatype mismatch
is harder to detect. We use a simple check ensuring the received bytes
are a multiple of a single basic element. Currently, we do not detect
mismatches with datatypes constructed of more than one basic type */
MPI_Datatype dt_basic_type;
MPIDU_Datatype_get_basic_type(rreq->dev.datatype, dt_basic_type);
if (dt_basic_type != MPI_DATATYPE_NULL && (e->mlength % MPIDU_Datatype_get_basic_size(dt_basic_type)) != 0)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
mpi_errno = handler_recv_complete(e);
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3U_Receive_data_unexpected(MPIR_Request * rreq, void *buf, intptr_t *buflen, int *complete)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPL_malloc(rreq->dev.recv_data_sz, MPL_MEM_BUFFER);
if (!rreq->dev.tmpbuf) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
/* if all of the data has already been received, copy it
now, otherwise build an iov and let the channel copy it */
if (rreq->dev.recv_data_sz <= *buflen)
{
MPIR_Memcpy(rreq->dev.tmpbuf, buf, rreq->dev.recv_data_sz);
*buflen = rreq->dev.recv_data_sz;
rreq->dev.recv_pending_count = 1;
*complete = TRUE;
}
else
{
rreq->dev.iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)((char *)rreq->dev.tmpbuf);
rreq->dev.iov[0].MPL_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.recv_pending_count = 2;
*buflen = 0;
*complete = FALSE;
}
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_PVAR_LEVEL_INC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
fn_fail:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
return mpi_errno;
}
int MPIR_T_pvar_handle_alloc_impl(MPI_T_pvar_session session, int pvar_index,
void *obj_handle, MPI_T_pvar_handle *handle,int *count)
{
int mpi_errno = MPI_SUCCESS;
int cnt, extra, bytes;
int is_sum, is_watermark;
const pvar_table_entry_t *info;
MPIR_T_pvar_handle_t *hnd;
MPIR_CHKPMEM_DECL(1);
info = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
if (info->get_count == NULL) {
cnt = info->count;
} else {
info->get_count(info->addr, obj_handle, &cnt);
}
bytes = MPID_Datatype_get_basic_size(info->datatype);
is_sum = FALSE;
is_watermark = FALSE;
extra = 0;
if (info->varclass == MPI_T_PVAR_CLASS_COUNTER ||
info->varclass == MPI_T_PVAR_CLASS_AGGREGATE ||
info->varclass == MPI_T_PVAR_CLASS_TIMER)
{
/* Extra memory for accum, offset, current */
is_sum = TRUE;
extra = bytes * cnt * 3;
} else if (info->varclass == MPI_T_PVAR_CLASS_HIGHWATERMARK ||
info->varclass == MPI_T_PVAR_CLASS_LOWWATERMARK)
{
is_watermark = TRUE;
}
/* Allocate memory and bzero it */
MPIR_CHKPMEM_CALLOC(hnd, MPIR_T_pvar_handle_t*, sizeof(*hnd) + extra,
mpi_errno, "performance variable handle");
#ifdef HAVE_ERROR_CHECKING
hnd->kind = MPIR_T_PVAR_HANDLE;
#endif
/* Setup the common fields */
if (is_sum)
hnd->flags |= MPIR_T_PVAR_FLAG_SUM;
else if (is_watermark)
hnd->flags |= MPIR_T_PVAR_FLAG_WATERMARK;
hnd->addr = info->addr;
hnd->datatype = info->datatype;
hnd->count = cnt;
hnd->varclass = info->varclass;
hnd->flags = info->flags;
hnd->session = session;
hnd->info = info;
hnd->obj_handle = obj_handle;
hnd->get_value = info->get_value;
hnd->bytes = bytes;
hnd->count = cnt;
/* Init pointers to cache buffers for a SUM */
if (MPIR_T_pvar_is_sum(hnd)) {
hnd->accum = (char*)(hnd) + sizeof(*hnd);
hnd->offset = (char*)(hnd) + sizeof(*hnd) + bytes*cnt;
hnd->current = (char*)(hnd) + sizeof(*hnd) + bytes*cnt*2;
}
if (MPIR_T_pvar_is_continuous(hnd))
MPIR_T_pvar_set_started(hnd);
/* Set starting value of a continuous SUM */
if (MPIR_T_pvar_is_continuous(hnd) && MPIR_T_pvar_is_sum(hnd)) {
/* Cache current value of a SUM in offset.
* accum is zero since we called CALLOC before.
*/
if (hnd->get_value == NULL)
MPIR_Memcpy(hnd->offset, hnd->addr, bytes*cnt);
else
hnd->get_value(hnd->addr, hnd->obj_handle, hnd->count, hnd->offset);
}
/* Link a WATERMARK handle to its pvar & set starting value if continuous */
if (MPIR_T_pvar_is_watermark(hnd)) {
MPIR_T_pvar_watermark_t *mark = (MPIR_T_pvar_watermark_t *)hnd->addr;
if (!mark->first_used) {
/* Use the special handle slot for optimization if available */
mark->first_used = TRUE;
MPIR_T_pvar_set_first(hnd);
/* Set starting value */
if (MPIR_T_pvar_is_continuous(hnd)) {
mark->first_started = TRUE;
mark->watermark = mark->current;
} else {
mark->first_started = FALSE;
}
} else {
/* If the special handle slot is unavailable, link it to hlist */
if (mark->hlist == NULL) {
hnd->prev2 = hnd;
mark->hlist = hnd;
} else {
hnd->prev2 = hnd;
hnd->next2 = mark->hlist;
mark->hlist->prev2 = hnd;
mark->hlist = hnd;
}
/* Set starting value */
if (MPIR_T_pvar_is_continuous(hnd))
hnd->watermark = mark->current;
}
}
/* Link the handle in its session and return it */
MPL_DL_APPEND(session->hlist, hnd);
*handle = hnd;
*count = cnt;
MPIR_CHKPMEM_COMMIT();
fn_exit:
return mpi_errno;
fn_fail:
MPIR_CHKPMEM_REAP();
goto fn_exit;
}
/*@
MPIDU_Datatype_set_contents - store contents information for use in
MPI_Type_get_contents.
Returns MPI_SUCCESS on success, MPI error code on error.
@*/
int MPIDU_Datatype_set_contents(MPIDU_Datatype *new_dtp,
int combiner,
int nr_ints,
int nr_aints,
int nr_types,
int array_of_ints[],
const MPI_Aint array_of_aints[],
const MPI_Datatype array_of_types[])
{
int i, contents_size, align_sz = 8, epsilon, mpi_errno;
int struct_sz, ints_sz, aints_sz, types_sz;
MPIDU_Datatype_contents *cp;
MPIDU_Datatype *old_dtp;
char *ptr;
#ifdef HAVE_MAX_STRUCT_ALIGNMENT
if (align_sz > HAVE_MAX_STRUCT_ALIGNMENT) {
align_sz = HAVE_MAX_STRUCT_ALIGNMENT;
}
#endif
struct_sz = sizeof(MPIDU_Datatype_contents);
types_sz = nr_types * sizeof(MPI_Datatype);
ints_sz = nr_ints * sizeof(int);
aints_sz = nr_aints * sizeof(MPI_Aint);
/* pad the struct, types, and ints before we allocate.
*
* note: it's not necessary that we pad the aints,
* because they are last in the region.
*/
if ((epsilon = struct_sz % align_sz)) {
struct_sz += align_sz - epsilon;
}
if ((epsilon = types_sz % align_sz)) {
types_sz += align_sz - epsilon;
}
if ((epsilon = ints_sz % align_sz)) {
ints_sz += align_sz - epsilon;
}
contents_size = struct_sz + types_sz + ints_sz + aints_sz;
cp = (MPIDU_Datatype_contents *) MPL_malloc(contents_size);
/* --BEGIN ERROR HANDLING-- */
if (cp == NULL) {
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
"MPIDU_Datatype_set_contents",
__LINE__,
MPI_ERR_OTHER,
"**nomem",
0);
return mpi_errno;
}
/* --END ERROR HANDLING-- */
cp->combiner = combiner;
cp->nr_ints = nr_ints;
cp->nr_aints = nr_aints;
cp->nr_types = nr_types;
/* arrays are stored in the following order: types, ints, aints,
* following the structure itself.
*/
ptr = ((char *) cp) + struct_sz;
/* Fortran90 combiner types do not have a "base" type */
if (nr_types > 0) {
MPIR_Memcpy(ptr, array_of_types, nr_types * sizeof(MPI_Datatype));
}
ptr = ((char *) cp) + struct_sz + types_sz;
if (nr_ints > 0) {
MPIR_Memcpy(ptr, array_of_ints, nr_ints * sizeof(int));
}
ptr = ((char *) cp) + struct_sz + types_sz + ints_sz;
if (nr_aints > 0) {
MPIR_Memcpy(ptr, array_of_aints, nr_aints * sizeof(MPI_Aint));
}
new_dtp->contents = cp;
/* increment reference counts on all the derived types used here */
for (i=0; i < nr_types; i++) {
if (HANDLE_GET_KIND(array_of_types[i]) != HANDLE_KIND_BUILTIN) {
MPIDU_Datatype_get_ptr(array_of_types[i], old_dtp);
MPIDU_Datatype_add_ref(old_dtp);
}
}
return MPI_SUCCESS;
}
int MPIDI_CH3U_Receive_data_found(MPIR_Request *rreq, void *buf, intptr_t *buflen, int *complete)
{
int dt_contig;
MPI_Aint dt_true_lb;
intptr_t userbuf_sz;
MPIR_Datatype * dt_ptr = NULL;
intptr_t data_sz;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_FOUND);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_FOUND);
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"posted request found");
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
dt_contig, userbuf_sz, dt_ptr, dt_true_lb);
if (rreq->dev.recv_data_sz <= userbuf_sz) {
data_sz = rreq->dev.recv_data_sz;
}
else {
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"receive buffer too small; message truncated, msg_sz=%" PRIdPTR ", userbuf_sz=%"
PRIdPTR,
rreq->dev.recv_data_sz, userbuf_sz));
rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
"**truncate", "**truncate %d %d %d %d",
rreq->status.MPI_SOURCE, rreq->status.MPI_TAG,
rreq->dev.recv_data_sz, userbuf_sz );
MPIR_STATUS_SET_COUNT(rreq->status, userbuf_sz);
data_sz = userbuf_sz;
}
if (dt_contig && data_sz == rreq->dev.recv_data_sz)
{
/* user buffer is contiguous and large enough to store the
entire message. However, we haven't yet *read* the data
(this code describes how to read the data into the destination) */
/* if all of the data has already been received, unpack it
now, otherwise build an iov and let the channel unpack */
if (*buflen >= data_sz)
{
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"Copying contiguous data to user buffer");
/* copy data out of the receive buffer */
if (rreq->dev.drop_data == FALSE) {
MPIR_Memcpy((char*)(rreq->dev.user_buf) + dt_true_lb, buf, data_sz);
}
*buflen = data_sz;
*complete = TRUE;
}
else
{
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"IOV loaded for contiguous read");
rreq->dev.iov[0].MPL_IOV_BUF =
(MPL_IOV_BUF_CAST)((char*)(rreq->dev.user_buf) + dt_true_lb);
rreq->dev.iov[0].MPL_IOV_LEN = data_sz;
rreq->dev.iov_count = 1;
*buflen = 0;
*complete = FALSE;
}
/* Trigger OnFinal when receiving the last segment */
rreq->dev.OnDataAvail = rreq->dev.OnFinal;
}
else {
/* user buffer is not contiguous or is too small to hold
the entire message */
rreq->dev.segment_ptr = MPIR_Segment_alloc( );
MPIR_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIR_Segment_alloc");
MPIR_Segment_init(rreq->dev.user_buf, rreq->dev.user_count,
rreq->dev.datatype, rreq->dev.segment_ptr);
rreq->dev.segment_first = 0;
rreq->dev.segment_size = data_sz;
/* if all of the data has already been received, and the
message is not truncated, unpack it now, otherwise build an
iov and let the channel unpack */
if (data_sz == rreq->dev.recv_data_sz && *buflen >= data_sz)
{
intptr_t last;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"Copying noncontiguous data to user buffer");
last = data_sz;
MPIR_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first,
&last, buf);
/* --BEGIN ERROR HANDLING-- */
if (last != data_sz)
{
/* If the data can't be unpacked, the we have a
mismatch between the datatype and the amount of
data received. Throw away received data. */
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
MPIR_STATUS_SET_COUNT(rreq->status, rreq->dev.segment_first);
*buflen = data_sz;
*complete = TRUE;
/* FIXME: Set OnDataAvail to 0? If not, why not? */
goto fn_exit;
}
/* --END ERROR HANDLING-- */
*buflen = data_sz;
/* Trigger OnFinal when receiving the last segment */
rreq->dev.OnDataAvail = rreq->dev.OnFinal;
*complete = TRUE;
}
else
{
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"IOV loaded for non-contiguous read");
mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**ch3|loadrecviov");
}
*buflen = 0;
*complete = FALSE;
}
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_FOUND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Unpack_impl(const void *inbuf, MPI_Aint insize, MPI_Aint * position,
void *outbuf, int outcount, MPI_Datatype datatype)
{
int mpi_errno = MPI_SUCCESS;
MPI_Aint first, last;
MPIR_Segment *segp;
int contig;
MPI_Aint dt_true_lb;
MPI_Aint data_sz;
if (insize == 0)
goto fn_exit;
/* Handle contig case quickly */
if (HANDLE_GET_KIND(datatype) == HANDLE_KIND_BUILTIN) {
contig = TRUE;
dt_true_lb = 0;
data_sz = outcount * MPIR_Datatype_get_basic_size(datatype);
} else {
MPIR_Datatype *dt_ptr;
MPIR_Datatype_get_ptr(datatype, dt_ptr);
MPIR_Datatype_is_contig(datatype, &contig);
dt_true_lb = dt_ptr->true_lb;
data_sz = outcount * dt_ptr->size;
}
if (contig) {
MPIR_Memcpy((char *) outbuf + dt_true_lb, (char *) inbuf + *position, data_sz);
*position = (int) ((MPI_Aint) * position + data_sz);
goto fn_exit;
}
/* non-contig case */
segp = MPIR_Segment_alloc();
MPIR_ERR_CHKANDJUMP1(segp == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIR_Segment_alloc");
mpi_errno = MPIR_Segment_init(outbuf, outcount, datatype, segp);
MPIR_Assert(mpi_errno == MPI_SUCCESS);
/* NOTE: the use of buffer values and positions in MPI_Unpack and in
* MPIR_Segment_unpack are quite different. See code or docs or something.
*/
first = 0;
last = SEGMENT_IGNORE_LAST;
/* Ensure that pointer increment fits in a pointer */
MPIR_Ensure_Aint_fits_in_pointer((MPIR_VOID_PTR_CAST_TO_MPI_AINT inbuf) +
(MPI_Aint) * position);
MPIR_Segment_unpack(segp, first, &last, (void *) ((char *) inbuf + *position));
/* Ensure that calculation fits into an int datatype. */
MPIR_Ensure_Aint_fits_in_int((MPI_Aint) * position + last);
*position = (int) ((MPI_Aint) * position + last);
MPIR_Segment_free(segp);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_send_iov(MPIDI_VC_t *vc, MPIR_Request **sreq_ptr, MPL_IOV *iov, int n_iov)
{
int mpi_errno = MPI_SUCCESS;
intptr_t data_sz;
int i;
int iov_data_copied;
MPIR_Request *sreq = *sreq_ptr;
MPL_IOV *data_iov = &iov[1]; /* iov of just the data, not the header */
int data_n_iov = n_iov - 1;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_SEND_IOV);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_SEND_IOV);
if (*sreq_ptr == NULL)
{
/* create a request */
sreq = MPIR_Request_create(MPIR_REQUEST_KIND__UNDEFINED);
MPIR_Assert(sreq != NULL);
MPIR_Object_set_ref(sreq, 2);
sreq->kind = MPIR_REQUEST_KIND__SEND;
sreq->dev.OnDataAvail = 0;
}
data_sz = 0;
for (i = 0; i < data_n_iov; ++i)
data_sz += data_iov[i].MPL_IOV_LEN;
if (!MPIDI_Request_get_srbuf_flag(sreq))
{
MPIDI_CH3U_SRBuf_alloc(sreq, data_sz);
/* --BEGIN ERROR HANDLING-- */
if (sreq->dev.tmpbuf_sz == 0)
{
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,TYPICAL,"SRBuf allocation failure");
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0);
sreq->status.MPI_ERROR = mpi_errno;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
}
MPIR_Assert(sreq->dev.tmpbuf_sz >= data_sz);
iov_data_copied = 0;
for (i = 0; i < data_n_iov; ++i) {
MPIR_Memcpy((char*) sreq->dev.tmpbuf + iov_data_copied, data_iov[i].MPL_IOV_BUF, data_iov[i].MPL_IOV_LEN);
iov_data_copied += data_iov[i].MPL_IOV_LEN;
}
mpi_errno = vc->ch.iSendContig(vc, sreq, iov[0].MPL_IOV_BUF, iov[0].MPL_IOV_LEN, sreq->dev.tmpbuf, data_sz);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
*sreq_ptr = sreq;
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_SEND_IOV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int handler_recv_dequeue_large(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *const rreq = e->user_ptr;
MPIDI_VC_t *vc;
MPID_nem_ptl_vc_area *vc_ptl;
int ret;
int dt_contig;
intptr_t data_sz;
MPIDU_Datatype*dt_ptr;
MPI_Aint dt_true_lb;
MPI_Aint last;
MPIR_CHKPMEM_DECL(1);
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_LARGE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_LARGE);
MPIR_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Comm_get_vc(rreq->comm, NPTL_MATCH_GET_RANK(e->match_bits), &vc);
vc_ptl = VC_PTL(vc);
dequeue_req(e);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
/* unpack data from unexpected buffer first */
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
if (dt_contig) {
MPIR_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPIDU_Segment_unpack(rreq->dev.segment_ptr, 0, &last, e->start);
MPIR_Assert(last == e->mlength);
rreq->dev.segment_first = e->mlength;
}
}
if (!(e->hdr_data & NPTL_LARGE)) {
/* all data has already been received; we're done */
mpi_errno = handler_recv_complete(e);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
goto fn_exit;
}
MPIR_Assert (e->mlength == PTL_LARGE_THRESHOLD);
/* we need to GET the rest of the data from the sender's buffer */
if (dt_contig) {
big_get((char *)rreq->dev.user_buf + dt_true_lb + PTL_LARGE_THRESHOLD, data_sz - PTL_LARGE_THRESHOLD,
vc, e->match_bits, rreq);
goto fn_exit;
}
/* noncontig recv buffer */
last = rreq->dev.segment_size;
rreq->dev.iov_count = MPL_IOV_LIMIT;
MPIDU_Segment_pack_vector(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, rreq->dev.iov, &rreq->dev.iov_count);
if (last == rreq->dev.segment_size && rreq->dev.segment_size <= MPIDI_nem_ptl_ni_limits.max_msg_size + PTL_LARGE_THRESHOLD) {
/* Rest of message fits in one IOV */
ptl_md_t md;
md.start = rreq->dev.iov;
md.length = rreq->dev.iov_count;
md.options = PTL_IOVEC;
md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(rreq)->md);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
REQ_PTL(rreq)->event_handler = handler_recv_complete;
ret = MPID_nem_ptl_rptl_get(REQ_PTL(rreq)->md, 0, rreq->dev.segment_size - rreq->dev.segment_first, vc_ptl->id, vc_ptl->ptg,
e->match_bits, 0, rreq);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlget", "**ptlget %s", MPID_nem_ptl_strerror(ret));
goto fn_exit;
}
/* message won't fit in a single IOV, allocate buffer and unpack when received */
/* FIXME: For now, allocate a single large buffer to hold entire message */
MPIR_CHKPMEM_MALLOC(REQ_PTL(rreq)->chunk_buffer[0], void *, data_sz - PTL_LARGE_THRESHOLD,
mpi_errno, "chunk_buffer");
big_get(REQ_PTL(rreq)->chunk_buffer[0], data_sz - PTL_LARGE_THRESHOLD, vc, e->match_bits, rreq);
fn_exit:
MPIR_CHKPMEM_COMMIT();
fn_exit2:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_LARGE);
return mpi_errno;
fn_fail:
MPIR_CHKPMEM_REAP();
goto fn_exit2;
}