static inline void copy_data (mca_coll_ml_collective_operation_progress_t *coll_op, rank_properties_t *rank_props, int soffset) {
bool rcontig = coll_op->fragment_data.message_descriptor->recv_data_continguous;
size_t total_bytes = coll_op->fragment_data.message_descriptor->n_bytes_total;
size_t pack_len = coll_op->fragment_data.fragment_size;
int doffset = rank_props->rank;
void *dest, *src;
src = (void *) ((uintptr_t)coll_op->fragment_data.buffer_desc->data_addr +
(size_t)coll_op->variable_fn_params.rbuf_offset + soffset * pack_len);
if (rcontig) {
dest = (void *) ((uintptr_t) coll_op->full_message.dest_user_addr +
(uintptr_t) coll_op->fragment_data.offset_into_user_buffer +
doffset * total_bytes);
memcpy(dest, src, pack_len);
} else {
size_t position;
opal_convertor_t *recv_convertor =
&coll_op->fragment_data.message_descriptor->recv_convertor;
position = (size_t) coll_op->fragment_data.offset_into_user_buffer +
doffset * total_bytes;
opal_convertor_set_position(recv_convertor, &position);
mca_coll_ml_convertor_unpack(src, pack_len, recv_convertor);
}
}
static size_t mca_pml_yalla_stream_pack(void *buffer, size_t length, size_t offset,
opal_convertor_t *convertor)
{
uint32_t iov_count;
struct iovec iov;
iov_count = 1;
iov.iov_base = buffer;
iov.iov_len = length;
opal_convertor_set_position(convertor, &offset);
opal_convertor_pack(convertor, &iov, &iov_count, &length);
return length;
}
static size_t ompi_mtl_mxm_stream_send(void *buffer, size_t length, size_t offset, void *context)
{
struct iovec iov;
uint32_t iov_count = 1;
opal_convertor_t *convertor = (opal_convertor_t *) context;
iov.iov_len = length;
iov.iov_base = buffer;
opal_convertor_set_position(convertor, &offset);
opal_convertor_pack(convertor, &iov, &iov_count, &length);
return length;
}
static int
create_segments( ompi_datatype_t* datatype, int count,
size_t segment_length,
ddt_segment_t** segments, int* seg_count )
{
size_t data_size, total_length, position;
opal_convertor_t* convertor;
int i;
ddt_segment_t* segment;
ompi_datatype_type_size( datatype, &data_size );
data_size *= count;
*seg_count = data_size / segment_length;
if( ((*seg_count) * segment_length) != data_size )
*seg_count += 1;
allocate_segments:
*segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) );
convertor = opal_convertor_create( opal_local_arch, 0 );
opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL );
position = 0;
total_length = 0;
for( i = 0; i < (*seg_count); i++ ) {
segment = &((*segments)[i]);
segment->buffer = malloc(segment_length);
segment->position = position;
/* Find the end of the segment */
position += segment_length;
opal_convertor_set_position( convertor, &position );
segment->size = position - segment->position;
total_length += segment->size;
}
OBJ_RELEASE(convertor);
if( total_length != data_size ) {
for( i = 0; i < (*seg_count); i++ ) {
segment = &((*segments)[i]);
free(segment->buffer);
}
free( *segments );
(*seg_count) += 1;
goto allocate_segments;
}
return 0;
}
static size_t ompi_mtl_mxm_stream_unpack(void *buffer, size_t length,
size_t offset, void *context)
{
struct iovec iov;
uint32_t iov_count = 1;
mca_mtl_mxm_request_t *mtl_mxm_request = (mca_mtl_mxm_request_t *) context;
opal_convertor_t *convertor = mtl_mxm_request->convertor;
iov.iov_len = length;
iov.iov_base = buffer;
opal_convertor_set_position(convertor, &offset);
opal_convertor_unpack(convertor, &iov, &iov_count, &length);
return length;
}
static int
unpack_segments( ompi_datatype_t* datatype, int count,
size_t segment_size,
ddt_segment_t* segments, int seg_count,
void* buffer )
{
opal_convertor_t* convertor;
size_t max_size, position;
int i;
uint32_t iov_count;
struct iovec iov;
convertor = opal_convertor_create( opal_local_arch, 0 );
opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer );
for( i = 0; i < seg_count; i++ ) {
iov.iov_len = segments[i].size;
iov.iov_base = segments[i].buffer;
max_size = iov.iov_len;
position = segments[i].position;
opal_convertor_set_position( convertor, &position );
if( position != segments[i].position ) {
opal_output( 0, "Setting position failed (%lu != %lu)\n",
(unsigned long)segments[i].position, (unsigned long)position );
break;
}
iov_count = 1;
opal_convertor_unpack( convertor, &iov, &iov_count, &max_size );
if( max_size != segments[i].size ) {
opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n",
(unsigned long)max_size, (unsigned long)segments[i].size );
opal_output( 0, "Segment %d position %lu size %lu\n", i,
(unsigned long)segments[i].position, segments[i].size );
}
}
OBJ_RELEASE(convertor);
return 0;
}
static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk )
{
OPAL_PTRDIFF_TYPE lb, extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, malloced_size;
int32_t done1 = 0, done2 = 0, errors = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
malloced_size = compute_memory_size(pdt, count);
opal_datatype_get_extent( pdt, &lb, &extent );
odst = (char*)malloc( malloced_size );
osrc = (char*)malloc( malloced_size );
ptemp = malloc( chunk );
{
for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ );
memcpy(odst, osrc, malloced_size);
}
pdst = odst - lb;
psrc = osrc - lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
struct dt_stack_t stack[1+send_convertor->stack_pos];
int i, stack_pos = send_convertor->stack_pos;
size_t pos;
if( 0 == done1 ) {
memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != send_convertor->pStack[i].index )
{errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, send_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != send_convertor->pStack[i].count ) {
if( stack[i].type == send_convertor->pStack[i].type ) {
{errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, send_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) )
{errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)send_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) )
{errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)send_convertor->pStack[i].type, send_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != send_convertor->pStack[i].disp )
{errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
if( 0 == done2 ) {
memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != recv_convertor->pStack[i].index )
{errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, recv_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != recv_convertor->pStack[i].count ) {
if( stack[i].type == recv_convertor->pStack[i].type ) {
{errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, recv_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) )
{errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)recv_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) )
{errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != recv_convertor->pStack[i].disp )
{errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying same data-type using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
if(outputFlags & VALIDATE_DATA) {
for( size_t i = errors = 0; i < malloced_size; i++ ) {
if( odst[i] != osrc[i] ) {
printf("error at position %lu (%d != %d)\n",
(unsigned long)i, (int)(odst[i]), (int)(osrc[i]));
errors++;
if(outputFlags & QUIT_ON_FIRST_ERROR) {
opal_datatype_dump(pdt);
assert(0); exit(-1);
}
}
}
if( 0 == errors ) {
printf("Validation check succesfully passed\n");
} else {
printf("Found %d errors. Giving up!\n", errors);
exit(-1);
}
}
clean_and_return:
if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return (0 == errors ? OPAL_SUCCESS : errors);
}
static int
local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count,
opal_datatype_t const * const recv_type, int recv_count,
int chunk )
{
OPAL_PTRDIFF_TYPE send_lb, send_extent, recv_lb, recv_extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, send_malloced_size, recv_malloced_size;;
int32_t done1 = 0, done2 = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
send_malloced_size = compute_memory_size(send_type, send_count);
recv_malloced_size = compute_memory_size(recv_type, recv_count);
opal_datatype_get_extent( send_type, &send_lb, &send_extent );
opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent );
odst = (char*)malloc( recv_malloced_size );
osrc = (char*)malloc( send_malloced_size );
ptemp = malloc( chunk );
/* fill up the receiver with ZEROS */
{
for( size_t i = 0; i < send_malloced_size; i++ )
osrc[i] = i % 128 + 32;
}
memset( odst, 0, recv_malloced_size );
pdst = odst - recv_lb;
psrc = osrc - send_lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
size_t pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying different data-types using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
clean_and_return:
if( send_convertor != NULL ) {
OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
}
if( recv_convertor != NULL ) {
OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
}
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return OPAL_SUCCESS;
}
int mca_pml_ob1_start(size_t count, ompi_request_t** requests)
{
int rc;
for (size_t i = 0 ; i < count ; ++i) {
mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
if (NULL == pml_request || OMPI_REQUEST_PML != requests[i]->req_type) {
continue;
}
/* If the persistent request is currently active - verify the status
* is incomplete. if the pml layer has not completed the request - mark
* the request as free called - so that it will be freed when the request
* completes - and create a new request.
*/
#if OPAL_ENABLE_MULTI_THREADS
opal_atomic_rmb();
#endif
/* start the request */
switch(pml_request->req_type) {
case MCA_PML_REQUEST_SEND:
{
mca_pml_ob1_send_request_t* sendreq = (mca_pml_ob1_send_request_t*)pml_request;
MEMCHECKER(
memchecker_call(&opal_memchecker_base_isdefined,
pml_request->req_addr, pml_request->req_count,
pml_request->req_datatype);
);
if (!pml_request->req_pml_complete) {
ompi_request_t *request;
/* buffered sends can be mpi complete and pml incomplete. to support this
* case we need to allocate a new request. */
rc = mca_pml_ob1_isend_init (pml_request->req_addr,
pml_request->req_count,
pml_request->req_datatype,
pml_request->req_peer,
pml_request->req_tag,
sendreq->req_send.req_send_mode,
pml_request->req_comm,
&request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
/* copy the callback and callback data to the new requests */
request->req_complete_cb = pml_request->req_ompi.req_complete_cb;
request->req_complete_cb_data = pml_request->req_ompi.req_complete_cb_data;
/* ensure the old request gets released */
pml_request->req_free_called = true;
sendreq = (mca_pml_ob1_send_request_t *) request;
requests[i] = request;
} else if (sendreq->req_send.req_bytes_packed != 0) {
size_t offset = 0;
/**
* Reset the convertor in case we're dealing with the original
* request, which when completed do not reset the convertor.
*/
opal_convertor_set_position (&sendreq->req_send.req_base.req_convertor,
&offset);
}
/* reset the completion flag */
pml_request->req_pml_complete = false;
MCA_PML_OB1_SEND_REQUEST_START(sendreq, rc);
if(rc != OMPI_SUCCESS)
return rc;
break;
}
case MCA_PML_REQUEST_RECV:
{
mca_pml_ob1_recv_request_t* recvreq = (mca_pml_ob1_recv_request_t*)pml_request;
MCA_PML_OB1_RECV_REQUEST_START(recvreq);
break;
}
default:
return OMPI_ERR_REQUEST;
}
}
static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) {
int i, j, errors = 0;
struct iovec a;
unsigned int iov_count;
size_t max_data;
size_t pos;
opal_convertor_t * pConv;
for (j = 0; j < N; ++j) {
pbar[j].i[0] = 123+j;
pbar[j].i[1] = 789+j;
pbar[j].d[0] = 123.456+j;
pbar[j].d[1] = 789.123+j;
memset(&bar[j].i[0], 0xFF, sizeof(int));
memset(&bar[j].i[2], 0xFF, sizeof(int));
bar[j].i[1] = 0;
memset(&bar[j].d[0], 0xFF, sizeof(double));
memset(&bar[j].d[2], 0xFF, sizeof(double));
bar[j].d[1] = 0.0;
}
pConv = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(newtype->super), N, bar ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OMPI_ERROR;
}
for (i=0; arr[i][0] != 0; i++) {
/* add some garbage before and after the source data */
a.iov_base = malloc(arr[i][0]+2048);
if (NULL == a.iov_base) {
printf("cannot malloc iov_base\n");
return 1;
}
memset(a.iov_base, 0xAA, 1024);
memcpy((char*)a.iov_base+1024, (char *)pbar + arr[i][1], arr[i][0]);
memset((char*)a.iov_base+1024+arr[i][0], 0xAA, 1024);
a.iov_base = (char*)a.iov_base + 1024;
a.iov_len = arr[i][0];
iov_count = 1;
max_data = a.iov_len;
pos = arr[i][1];
opal_convertor_set_position(pConv, &pos);
assert(arr[i][1] == pos);
opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
a.iov_base = (char*)a.iov_base - 1024;
free(a.iov_base);
}
for (j = 0; j < N; ++j) {
if (bar[j].i[0] != pbar[j].i[0] ||
bar[j].i[1] != 0 ||
bar[j].i[2] != pbar[j].i[1] ||
bar[j].d[0] != pbar[j].d[0] ||
bar[j].d[1] != 0.0 ||
bar[j].d[2] != pbar[j].d[1]) {
if(0 == errors) {
fprintf(stderr, "ERROR ! count=%d, position=%d, ptr = %p"
" got (%d,%d,%d,%g,%g,%g) expected (%d,%d,%d,%g,%g,%g)\n",
N, j, (void*)&bar[j],
bar[j].i[0],
bar[j].i[1],
bar[j].i[2],
bar[j].d[0],
bar[j].d[1],
bar[j].d[2],
pbar[j].i[0],
0,
pbar[j].i[1],
pbar[j].d[0],
0.0,
pbar[j].d[1]);
print_bar_pbar(&bar[j], &pbar[j]);
}
errors++;
}
}
OBJ_RELEASE( pConv );
return errors;
}
static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_buffer, int source_count,
ompi_datatype_t *source_datatype, void *result_buffer, int result_count,
ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
mca_btl_base_registration_handle_t *target_handle, int target_count,
ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
{
ompi_osc_rdma_module_t *module = sync->module;
struct iovec source_iovec[OMPI_OSC_RDMA_DECODE_MAX], target_iovec[OMPI_OSC_RDMA_DECODE_MAX];
const size_t acc_limit = (mca_osc_rdma_component.buffer_size >> 3);
uint32_t source_primitive_count, target_primitive_count;
opal_convertor_t source_convertor, target_convertor;
uint32_t source_iov_count, target_iov_count;
uint32_t source_iov_index, target_iov_index;
ompi_datatype_t *source_primitive, *target_primitive;
/* needed for opal_convertor_raw but not used */
size_t source_size, target_size;
ompi_osc_rdma_request_t *subreq;
size_t result_position;
ptrdiff_t lb, extent;
int ret, acc_len;
bool done;
(void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
target_address += lb;
/* fast path for accumulate on built-in types */
if (OPAL_LIKELY((!source_count || ompi_datatype_is_predefined (source_datatype)) &&
ompi_datatype_is_predefined (target_datatype) &&
(!result_count || ompi_datatype_is_predefined (result_datatype)) &&
(target_datatype->super.size * target_count <= acc_limit))) {
if (NULL == request) {
OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, request);
request->internal = true;
request->type = result_datatype ? OMPI_OSC_RDMA_TYPE_GET_ACC : OMPI_OSC_RDMA_TYPE_ACC;
}
if (source_datatype) {
(void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
source_buffer = (void *)((intptr_t) source_buffer + lb);
}
if (result_datatype) {
(void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
result_buffer = (void *)((intptr_t) result_buffer + lb);
}
ret = ompi_osc_rdma_gacc_contig (sync, source_buffer, source_count, source_datatype, result_buffer,
result_count, result_datatype, peer, target_address,
target_handle, target_count, target_datatype, op,
request);
if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
return OMPI_SUCCESS;
}
if (source_datatype) {
/* the convertors will handle the lb */
(void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
source_buffer = (void *)((intptr_t) source_buffer - lb);
}
if (result_datatype) {
(void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
result_buffer = (void *)((intptr_t) result_buffer - lb);
}
}
/* the convertor will handle lb from here */
(void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
target_address -= lb;
/* get the primitive datatype info */
ret = ompi_osc_base_get_primitive_type_info (target_datatype, &target_primitive, &target_primitive_count);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
/* target datatype is not made up of a single basic datatype */
return ret;
}
if (source_datatype) {
ret = ompi_osc_base_get_primitive_type_info (source_datatype, &source_primitive, &source_primitive_count);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
/* target datatype is not made up of a single basic datatype */
return ret;
}
if (OPAL_UNLIKELY(source_primitive != target_primitive)) {
return MPI_ERR_TYPE;
}
}
/* prepare convertors for the source and target. these convertors will be used to determine the
* contiguous segments within the source and target. */
/* the source may be NULL if using MPI_OP_NO_OP with MPI_Get_accumulate */
if (source_datatype) {
OBJ_CONSTRUCT(&source_convertor, opal_convertor_t);
ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &source_datatype->super, source_count, source_buffer,
0, &source_convertor);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
}
/* target_datatype can never be NULL */
OBJ_CONSTRUCT(&target_convertor, opal_convertor_t);
ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count,
(void *) (intptr_t) target_address, 0, &target_convertor);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
if (request) {
/* keep the request from completing until all the transfers have started */
request->outstanding_requests = 1;
}
target_iov_index = 0;
target_iov_count = 0;
result_position = 0;
do {
/* decode segments of the source data */
source_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
source_iov_index = 0;
/* opal_convertor_raw returns done when it has reached the end of the data */
if (!source_datatype) {
done = true;
source_iovec[0].iov_len = (size_t) -1;
source_iovec[0].iov_base = NULL;
source_iov_count = 1;
} else {
done = opal_convertor_raw (&source_convertor, source_iovec, &source_iov_count, &source_size);
}
/* loop on the target segments until we have exhaused the decoded source data */
while (source_iov_index != source_iov_count) {
if (target_iov_index == target_iov_count) {
/* decode segments of the target buffer */
target_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
target_iov_index = 0;
(void) opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
}
/* we already checked that the target was large enough. this should be impossible */
assert (0 != target_iov_count);
/* determine how much to put in this operation */
acc_len = min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len);
acc_len = min((size_t) acc_len, acc_limit);
/* execute the get */
OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, subreq);
subreq->internal = true;
subreq->parent_request = request;
if (request) {
(void) OPAL_THREAD_ADD32 (&request->outstanding_requests, 1);
}
if (result_datatype) {
/* prepare a convertor for this part of the result */
opal_convertor_copy_and_prepare_for_recv (ompi_mpi_local_convertor, &result_datatype->super, result_count,
result_buffer, 0, &subreq->convertor);
opal_convertor_set_position (&subreq->convertor, &result_position);
subreq->type = OMPI_OSC_RDMA_TYPE_GET_ACC;
} else {
subreq->type = OMPI_OSC_RDMA_TYPE_ACC;
}
OPAL_OUTPUT_VERBOSE((60, ompi_osc_base_framework.framework_output,
"target index = %d, target = {%p, %lu}, source_index = %d, source = {%p, %lu}, result = %p, result position = %lu, "
"acc_len = %d, count = %lu",
target_iov_index, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len,
source_iov_index, source_iovec[source_iov_index].iov_base, (unsigned long) source_iovec[source_iov_index].iov_len,
result_buffer, (unsigned long) result_position, acc_len, (unsigned long)(acc_len / target_primitive->super.size)));
ret = ompi_osc_rdma_gacc_contig (sync, source_iovec[source_iov_index].iov_base, acc_len / target_primitive->super.size,
target_primitive, NULL, 0, NULL, peer, (uint64_t) (intptr_t) target_iovec[target_iov_index].iov_base,
target_handle, acc_len / target_primitive->super.size, target_primitive, op, subreq);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE != ret)) {
/* something bad happened. need to figure out how to handle these errors */
return ret;
}
/* progress and try again */
ompi_osc_rdma_progress (module);
continue;
}
/* adjust io vectors */
target_iovec[target_iov_index].iov_len -= acc_len;
source_iovec[source_iov_index].iov_len -= acc_len;
target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + acc_len);
source_iovec[source_iov_index].iov_base = (void *)((intptr_t) source_iovec[source_iov_index].iov_base + acc_len);
result_position += acc_len;
source_iov_index += !source_datatype || (0 == source_iovec[source_iov_index].iov_len);
target_iov_index += (0 == target_iovec[target_iov_index].iov_len);
}
} while (!done);
if (request) {
/* release our reference so the request can complete */
(void) OPAL_THREAD_ADD32 (&request->outstanding_requests, -1);
}
if (source_datatype) {
opal_convertor_cleanup (&source_convertor);
OBJ_DESTRUCT(&source_convertor);
}
opal_convertor_cleanup (&target_convertor);
OBJ_DESTRUCT(&target_convertor);
return OMPI_SUCCESS;
}
