static inline void copy_data (mca_coll_ml_collective_operation_progress_t *coll_op, rank_properties_t *rank_props, int soffset) { bool rcontig = coll_op->fragment_data.message_descriptor->recv_data_continguous; size_t total_bytes = coll_op->fragment_data.message_descriptor->n_bytes_total; size_t pack_len = coll_op->fragment_data.fragment_size; int doffset = rank_props->rank; void *dest, *src; src = (void *) ((uintptr_t)coll_op->fragment_data.buffer_desc->data_addr + (size_t)coll_op->variable_fn_params.rbuf_offset + soffset * pack_len); if (rcontig) { dest = (void *) ((uintptr_t) coll_op->full_message.dest_user_addr + (uintptr_t) coll_op->fragment_data.offset_into_user_buffer + doffset * total_bytes); memcpy(dest, src, pack_len); } else { size_t position; opal_convertor_t *recv_convertor = &coll_op->fragment_data.message_descriptor->recv_convertor; position = (size_t) coll_op->fragment_data.offset_into_user_buffer + doffset * total_bytes; opal_convertor_set_position(recv_convertor, &position); mca_coll_ml_convertor_unpack(src, pack_len, recv_convertor); } }
static size_t mca_pml_yalla_stream_pack(void *buffer, size_t length, size_t offset, opal_convertor_t *convertor) { uint32_t iov_count; struct iovec iov; iov_count = 1; iov.iov_base = buffer; iov.iov_len = length; opal_convertor_set_position(convertor, &offset); opal_convertor_pack(convertor, &iov, &iov_count, &length); return length; }
static size_t ompi_mtl_mxm_stream_send(void *buffer, size_t length, size_t offset, void *context) { struct iovec iov; uint32_t iov_count = 1; opal_convertor_t *convertor = (opal_convertor_t *) context; iov.iov_len = length; iov.iov_base = buffer; opal_convertor_set_position(convertor, &offset); opal_convertor_pack(convertor, &iov, &iov_count, &length); return length; }
static int create_segments( ompi_datatype_t* datatype, int count, size_t segment_length, ddt_segment_t** segments, int* seg_count ) { size_t data_size, total_length, position; opal_convertor_t* convertor; int i; ddt_segment_t* segment; ompi_datatype_type_size( datatype, &data_size ); data_size *= count; *seg_count = data_size / segment_length; if( ((*seg_count) * segment_length) != data_size ) *seg_count += 1; allocate_segments: *segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) ); convertor = opal_convertor_create( opal_local_arch, 0 ); opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL ); position = 0; total_length = 0; for( i = 0; i < (*seg_count); i++ ) { segment = &((*segments)[i]); segment->buffer = malloc(segment_length); segment->position = position; /* Find the end of the segment */ position += segment_length; opal_convertor_set_position( convertor, &position ); segment->size = position - segment->position; total_length += segment->size; } OBJ_RELEASE(convertor); if( total_length != data_size ) { for( i = 0; i < (*seg_count); i++ ) { segment = &((*segments)[i]); free(segment->buffer); } free( *segments ); (*seg_count) += 1; goto allocate_segments; } return 0; }
static size_t ompi_mtl_mxm_stream_unpack(void *buffer, size_t length, size_t offset, void *context) { struct iovec iov; uint32_t iov_count = 1; mca_mtl_mxm_request_t *mtl_mxm_request = (mca_mtl_mxm_request_t *) context; opal_convertor_t *convertor = mtl_mxm_request->convertor; iov.iov_len = length; iov.iov_base = buffer; opal_convertor_set_position(convertor, &offset); opal_convertor_unpack(convertor, &iov, &iov_count, &length); return length; }
static int unpack_segments( ompi_datatype_t* datatype, int count, size_t segment_size, ddt_segment_t* segments, int seg_count, void* buffer ) { opal_convertor_t* convertor; size_t max_size, position; int i; uint32_t iov_count; struct iovec iov; convertor = opal_convertor_create( opal_local_arch, 0 ); opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer ); for( i = 0; i < seg_count; i++ ) { iov.iov_len = segments[i].size; iov.iov_base = segments[i].buffer; max_size = iov.iov_len; position = segments[i].position; opal_convertor_set_position( convertor, &position ); if( position != segments[i].position ) { opal_output( 0, "Setting position failed (%lu != %lu)\n", (unsigned long)segments[i].position, (unsigned long)position ); break; } iov_count = 1; opal_convertor_unpack( convertor, &iov, &iov_count, &max_size ); if( max_size != segments[i].size ) { opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n", (unsigned long)max_size, (unsigned long)segments[i].size ); opal_output( 0, "Segment %d position %lu size %lu\n", i, (unsigned long)segments[i].position, segments[i].size ); } } OBJ_RELEASE(convertor); return 0; }
static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk ) { OPAL_PTRDIFF_TYPE lb, extent; void *pdst = NULL, *psrc = NULL, *ptemp = NULL; char *odst, *osrc; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data, length = 0, malloced_size; int32_t done1 = 0, done2 = 0, errors = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; malloced_size = compute_memory_size(pdt, count); opal_datatype_get_extent( pdt, &lb, &extent ); odst = (char*)malloc( malloced_size ); osrc = (char*)malloc( malloced_size ); ptemp = malloc( chunk ); { for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ ); memcpy(odst, osrc, malloced_size); } pdst = odst - lb; psrc = osrc - lb; send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; if( outputFlags & RESET_CONVERTORS ) { struct dt_stack_t stack[1+send_convertor->stack_pos]; int i, stack_pos = send_convertor->stack_pos; size_t pos; if( 0 == done1 ) { memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t)); pos = 0; opal_convertor_set_position(send_convertor, &pos); pos = length; opal_convertor_set_position(send_convertor, &pos); assert(pos == length); for(i = 0; i <= stack_pos; i++ ) { if( stack[i].index != send_convertor->pStack[i].index ) {errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n", i, stack[i].index, send_convertor->pStack[i].index, length, pdt->size * count);} if( stack[i].count != send_convertor->pStack[i].count ) { if( stack[i].type == send_convertor->pStack[i].type ) { {errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n", i, stack[i].count, send_convertor->pStack[i].count, length, pdt->size * count);} } else { if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) ) {errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n", i, (int)stack[i].type, (int)send_convertor->pStack[i].type, length, pdt->size * count);} else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) != (send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) ) {errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n", i, (int)stack[i].type, stack[i].count, (int)send_convertor->pStack[i].type, send_convertor->pStack[i].count, length, pdt->size * count);} } } if( stack[i].disp != send_convertor->pStack[i].disp ) {errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n", i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp, length, pdt->size * count);} if(0 != errors) {assert(0); exit(-1);} } } if( 0 == done2 ) { memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t)); pos = 0; opal_convertor_set_position(recv_convertor, &pos); pos = length; opal_convertor_set_position(recv_convertor, &pos); assert(pos == length); for(i = 0; i <= stack_pos; i++ ) { if( stack[i].index != recv_convertor->pStack[i].index ) {errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n", i, stack[i].index, recv_convertor->pStack[i].index, length, pdt->size * count);} if( stack[i].count != recv_convertor->pStack[i].count ) { if( stack[i].type == recv_convertor->pStack[i].type ) { {errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n", i, stack[i].count, recv_convertor->pStack[i].count, length, pdt->size * count);} } else { if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) ) {errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n", i, (int)stack[i].type, (int)recv_convertor->pStack[i].type, length, pdt->size * count);} else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) != (recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) ) {errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n", i, (int)stack[i].type, stack[i].count, (int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count, length, pdt->size * count);} } } if( stack[i].disp != recv_convertor->pStack[i].disp ) {errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n", i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp, length, pdt->size * count);} if(0 != errors) {assert(0); exit(-1);} } } } } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying same data-type using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); if(outputFlags & VALIDATE_DATA) { for( size_t i = errors = 0; i < malloced_size; i++ ) { if( odst[i] != osrc[i] ) { printf("error at position %lu (%d != %d)\n", (unsigned long)i, (int)(odst[i]), (int)(osrc[i])); errors++; if(outputFlags & QUIT_ON_FIRST_ERROR) { opal_datatype_dump(pdt); assert(0); exit(-1); } } } if( 0 == errors ) { printf("Validation check succesfully passed\n"); } else { printf("Found %d errors. Giving up!\n", errors); exit(-1); } } clean_and_return: if( NULL != send_convertor ) OBJ_RELEASE( send_convertor ); if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor ); if( NULL != odst ) free( odst ); if( NULL != osrc ) free( osrc ); if( NULL != ptemp ) free( ptemp ); return (0 == errors ? OPAL_SUCCESS : errors); }
static int local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count, opal_datatype_t const * const recv_type, int recv_count, int chunk ) { OPAL_PTRDIFF_TYPE send_lb, send_extent, recv_lb, recv_extent; void *pdst = NULL, *psrc = NULL, *ptemp = NULL; char *odst, *osrc; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data, length = 0, send_malloced_size, recv_malloced_size;; int32_t done1 = 0, done2 = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; send_malloced_size = compute_memory_size(send_type, send_count); recv_malloced_size = compute_memory_size(recv_type, recv_count); opal_datatype_get_extent( send_type, &send_lb, &send_extent ); opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent ); odst = (char*)malloc( recv_malloced_size ); osrc = (char*)malloc( send_malloced_size ); ptemp = malloc( chunk ); /* fill up the receiver with ZEROS */ { for( size_t i = 0; i < send_malloced_size; i++ ) osrc[i] = i % 128 + 32; } memset( odst, 0, recv_malloced_size ); pdst = odst - recv_lb; psrc = osrc - send_lb; send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; if( outputFlags & RESET_CONVERTORS ) { size_t pos = 0; opal_convertor_set_position(send_convertor, &pos); pos = length; opal_convertor_set_position(send_convertor, &pos); assert(pos == length); pos = 0; opal_convertor_set_position(recv_convertor, &pos); pos = length; opal_convertor_set_position(recv_convertor, &pos); assert(pos == length); } } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying different data-types using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); clean_and_return: if( send_convertor != NULL ) { OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL ); } if( recv_convertor != NULL ) { OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL ); } if( NULL != odst ) free( odst ); if( NULL != osrc ) free( osrc ); if( NULL != ptemp ) free( ptemp ); return OPAL_SUCCESS; }
int mca_pml_ob1_start(size_t count, ompi_request_t** requests) { int rc; for (size_t i = 0 ; i < count ; ++i) { mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i]; if (NULL == pml_request || OMPI_REQUEST_PML != requests[i]->req_type) { continue; } /* If the persistent request is currently active - verify the status * is incomplete. if the pml layer has not completed the request - mark * the request as free called - so that it will be freed when the request * completes - and create a new request. */ #if OPAL_ENABLE_MULTI_THREADS opal_atomic_rmb(); #endif /* start the request */ switch(pml_request->req_type) { case MCA_PML_REQUEST_SEND: { mca_pml_ob1_send_request_t* sendreq = (mca_pml_ob1_send_request_t*)pml_request; MEMCHECKER( memchecker_call(&opal_memchecker_base_isdefined, pml_request->req_addr, pml_request->req_count, pml_request->req_datatype); ); if (!pml_request->req_pml_complete) { ompi_request_t *request; /* buffered sends can be mpi complete and pml incomplete. to support this * case we need to allocate a new request. */ rc = mca_pml_ob1_isend_init (pml_request->req_addr, pml_request->req_count, pml_request->req_datatype, pml_request->req_peer, pml_request->req_tag, sendreq->req_send.req_send_mode, pml_request->req_comm, &request); if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) { return rc; } /* copy the callback and callback data to the new requests */ request->req_complete_cb = pml_request->req_ompi.req_complete_cb; request->req_complete_cb_data = pml_request->req_ompi.req_complete_cb_data; /* ensure the old request gets released */ pml_request->req_free_called = true; sendreq = (mca_pml_ob1_send_request_t *) request; requests[i] = request; } else if (sendreq->req_send.req_bytes_packed != 0) { size_t offset = 0; /** * Reset the convertor in case we're dealing with the original * request, which when completed do not reset the convertor. */ opal_convertor_set_position (&sendreq->req_send.req_base.req_convertor, &offset); } /* reset the completion flag */ pml_request->req_pml_complete = false; MCA_PML_OB1_SEND_REQUEST_START(sendreq, rc); if(rc != OMPI_SUCCESS) return rc; break; } case MCA_PML_REQUEST_RECV: { mca_pml_ob1_recv_request_t* recvreq = (mca_pml_ob1_recv_request_t*)pml_request; MCA_PML_OB1_RECV_REQUEST_START(recvreq); break; } default: return OMPI_ERR_REQUEST; } }
static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) { int i, j, errors = 0; struct iovec a; unsigned int iov_count; size_t max_data; size_t pos; opal_convertor_t * pConv; for (j = 0; j < N; ++j) { pbar[j].i[0] = 123+j; pbar[j].i[1] = 789+j; pbar[j].d[0] = 123.456+j; pbar[j].d[1] = 789.123+j; memset(&bar[j].i[0], 0xFF, sizeof(int)); memset(&bar[j].i[2], 0xFF, sizeof(int)); bar[j].i[1] = 0; memset(&bar[j].d[0], 0xFF, sizeof(double)); memset(&bar[j].d[2], 0xFF, sizeof(double)); bar[j].d[1] = 0.0; } pConv = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(newtype->super), N, bar ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OMPI_ERROR; } for (i=0; arr[i][0] != 0; i++) { /* add some garbage before and after the source data */ a.iov_base = malloc(arr[i][0]+2048); if (NULL == a.iov_base) { printf("cannot malloc iov_base\n"); return 1; } memset(a.iov_base, 0xAA, 1024); memcpy((char*)a.iov_base+1024, (char *)pbar + arr[i][1], arr[i][0]); memset((char*)a.iov_base+1024+arr[i][0], 0xAA, 1024); a.iov_base = (char*)a.iov_base + 1024; a.iov_len = arr[i][0]; iov_count = 1; max_data = a.iov_len; pos = arr[i][1]; opal_convertor_set_position(pConv, &pos); assert(arr[i][1] == pos); opal_convertor_unpack( pConv, &a, &iov_count, &max_data ); a.iov_base = (char*)a.iov_base - 1024; free(a.iov_base); } for (j = 0; j < N; ++j) { if (bar[j].i[0] != pbar[j].i[0] || bar[j].i[1] != 0 || bar[j].i[2] != pbar[j].i[1] || bar[j].d[0] != pbar[j].d[0] || bar[j].d[1] != 0.0 || bar[j].d[2] != pbar[j].d[1]) { if(0 == errors) { fprintf(stderr, "ERROR ! count=%d, position=%d, ptr = %p" " got (%d,%d,%d,%g,%g,%g) expected (%d,%d,%d,%g,%g,%g)\n", N, j, (void*)&bar[j], bar[j].i[0], bar[j].i[1], bar[j].i[2], bar[j].d[0], bar[j].d[1], bar[j].d[2], pbar[j].i[0], 0, pbar[j].i[1], pbar[j].d[0], 0.0, pbar[j].d[1]); print_bar_pbar(&bar[j], &pbar[j]); } errors++; } } OBJ_RELEASE( pConv ); return errors; }
static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_buffer, int source_count, ompi_datatype_t *source_datatype, void *result_buffer, int result_count, ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address, mca_btl_base_registration_handle_t *target_handle, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request) { ompi_osc_rdma_module_t *module = sync->module; struct iovec source_iovec[OMPI_OSC_RDMA_DECODE_MAX], target_iovec[OMPI_OSC_RDMA_DECODE_MAX]; const size_t acc_limit = (mca_osc_rdma_component.buffer_size >> 3); uint32_t source_primitive_count, target_primitive_count; opal_convertor_t source_convertor, target_convertor; uint32_t source_iov_count, target_iov_count; uint32_t source_iov_index, target_iov_index; ompi_datatype_t *source_primitive, *target_primitive; /* needed for opal_convertor_raw but not used */ size_t source_size, target_size; ompi_osc_rdma_request_t *subreq; size_t result_position; ptrdiff_t lb, extent; int ret, acc_len; bool done; (void) ompi_datatype_get_extent (target_datatype, &lb, &extent); target_address += lb; /* fast path for accumulate on built-in types */ if (OPAL_LIKELY((!source_count || ompi_datatype_is_predefined (source_datatype)) && ompi_datatype_is_predefined (target_datatype) && (!result_count || ompi_datatype_is_predefined (result_datatype)) && (target_datatype->super.size * target_count <= acc_limit))) { if (NULL == request) { OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, request); request->internal = true; request->type = result_datatype ? OMPI_OSC_RDMA_TYPE_GET_ACC : OMPI_OSC_RDMA_TYPE_ACC; } if (source_datatype) { (void) ompi_datatype_get_extent (source_datatype, &lb, &extent); source_buffer = (void *)((intptr_t) source_buffer + lb); } if (result_datatype) { (void) ompi_datatype_get_extent (result_datatype, &lb, &extent); result_buffer = (void *)((intptr_t) result_buffer + lb); } ret = ompi_osc_rdma_gacc_contig (sync, source_buffer, source_count, source_datatype, result_buffer, result_count, result_datatype, peer, target_address, target_handle, target_count, target_datatype, op, request); if (OPAL_LIKELY(OMPI_SUCCESS == ret)) { return OMPI_SUCCESS; } if (source_datatype) { /* the convertors will handle the lb */ (void) ompi_datatype_get_extent (source_datatype, &lb, &extent); source_buffer = (void *)((intptr_t) source_buffer - lb); } if (result_datatype) { (void) ompi_datatype_get_extent (result_datatype, &lb, &extent); result_buffer = (void *)((intptr_t) result_buffer - lb); } } /* the convertor will handle lb from here */ (void) ompi_datatype_get_extent (target_datatype, &lb, &extent); target_address -= lb; /* get the primitive datatype info */ ret = ompi_osc_base_get_primitive_type_info (target_datatype, &target_primitive, &target_primitive_count); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { /* target datatype is not made up of a single basic datatype */ return ret; } if (source_datatype) { ret = ompi_osc_base_get_primitive_type_info (source_datatype, &source_primitive, &source_primitive_count); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { /* target datatype is not made up of a single basic datatype */ return ret; } if (OPAL_UNLIKELY(source_primitive != target_primitive)) { return MPI_ERR_TYPE; } } /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ /* the source may be NULL if using MPI_OP_NO_OP with MPI_Get_accumulate */ if (source_datatype) { OBJ_CONSTRUCT(&source_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &source_datatype->super, source_count, source_buffer, 0, &source_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } } /* target_datatype can never be NULL */ OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *) (intptr_t) target_address, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } if (request) { /* keep the request from completing until all the transfers have started */ request->outstanding_requests = 1; } target_iov_index = 0; target_iov_count = 0; result_position = 0; do { /* decode segments of the source data */ source_iov_count = OMPI_OSC_RDMA_DECODE_MAX; source_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ if (!source_datatype) { done = true; source_iovec[0].iov_len = (size_t) -1; source_iovec[0].iov_base = NULL; source_iov_count = 1; } else { done = opal_convertor_raw (&source_convertor, source_iovec, &source_iov_count, &source_size); } /* loop on the target segments until we have exhaused the decoded source data */ while (source_iov_index != source_iov_count) { if (target_iov_index == target_iov_count) { /* decode segments of the target buffer */ target_iov_count = OMPI_OSC_RDMA_DECODE_MAX; target_iov_index = 0; (void) opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != target_iov_count); /* determine how much to put in this operation */ acc_len = min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len); acc_len = min((size_t) acc_len, acc_limit); /* execute the get */ OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, subreq); subreq->internal = true; subreq->parent_request = request; if (request) { (void) OPAL_THREAD_ADD32 (&request->outstanding_requests, 1); } if (result_datatype) { /* prepare a convertor for this part of the result */ opal_convertor_copy_and_prepare_for_recv (ompi_mpi_local_convertor, &result_datatype->super, result_count, result_buffer, 0, &subreq->convertor); opal_convertor_set_position (&subreq->convertor, &result_position); subreq->type = OMPI_OSC_RDMA_TYPE_GET_ACC; } else { subreq->type = OMPI_OSC_RDMA_TYPE_ACC; } OPAL_OUTPUT_VERBOSE((60, ompi_osc_base_framework.framework_output, "target index = %d, target = {%p, %lu}, source_index = %d, source = {%p, %lu}, result = %p, result position = %lu, " "acc_len = %d, count = %lu", target_iov_index, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len, source_iov_index, source_iovec[source_iov_index].iov_base, (unsigned long) source_iovec[source_iov_index].iov_len, result_buffer, (unsigned long) result_position, acc_len, (unsigned long)(acc_len / target_primitive->super.size))); ret = ompi_osc_rdma_gacc_contig (sync, source_iovec[source_iov_index].iov_base, acc_len / target_primitive->super.size, target_primitive, NULL, 0, NULL, peer, (uint64_t) (intptr_t) target_iovec[target_iov_index].iov_base, target_handle, acc_len / target_primitive->super.size, target_primitive, op, subreq); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE != ret)) { /* something bad happened. need to figure out how to handle these errors */ return ret; } /* progress and try again */ ompi_osc_rdma_progress (module); continue; } /* adjust io vectors */ target_iovec[target_iov_index].iov_len -= acc_len; source_iovec[source_iov_index].iov_len -= acc_len; target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + acc_len); source_iovec[source_iov_index].iov_base = (void *)((intptr_t) source_iovec[source_iov_index].iov_base + acc_len); result_position += acc_len; source_iov_index += !source_datatype || (0 == source_iovec[source_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); if (request) { /* release our reference so the request can complete */ (void) OPAL_THREAD_ADD32 (&request->outstanding_requests, -1); } if (source_datatype) { opal_convertor_cleanup (&source_convertor); OBJ_DESTRUCT(&source_convertor); } opal_convertor_cleanup (&target_convertor); OBJ_DESTRUCT(&target_convertor); return OMPI_SUCCESS; }