/**
* Parse the datatype description from the args and find if the
* datatype is created from a single predefined type. If yes,
* return the type, otherwise return NULL.
*/
ompi_datatype_t* ompi_datatype_get_single_predefined_type_from_args( ompi_datatype_t* type )
{
ompi_datatype_t *predef = NULL, *current_type, *current_predef;
ompi_datatype_args_t* args = (ompi_datatype_args_t*)type->args;
int i;
if( ompi_datatype_is_predefined(type) )
return type;
for( i = 0; i < args->cd; i++ ) {
current_type = args->d[i];
if( ompi_datatype_is_predefined(current_type) ) {
current_predef = current_type;
} else {
current_predef = ompi_datatype_get_single_predefined_type_from_args(current_type);
if( NULL == current_predef ) { /* No single predefined datatype */
return NULL;
}
}
if( NULL == predef ) { /* This is the first iteration */
predef = current_predef;
} else {
/**
* What exactly should we consider as identical types? If they are
* the same MPI level type, or if they map to the same OPAL datatype?
* In other words, MPI_FLOAT and MPI_REAL4 are they identical?
*/
if( predef != current_predef ) {
return NULL;
}
}
}
return predef;
}
int32_t ompi_datatype_destroy( ompi_datatype_t** type)
{
ompi_datatype_t* pData = *type;
if( ompi_datatype_is_predefined(pData) && (pData->super.super.obj_reference_count <= 1) )
return OMPI_ERROR;
OBJ_RELEASE(pData);
*type = NULL;
return OMPI_SUCCESS;
}
int32_t ompi_datatype_get_args( const ompi_datatype_t* pData, int32_t which,
int32_t* ci, int32_t* i,
int32_t* ca, OPAL_PTRDIFF_TYPE* a,
int32_t* cd, ompi_datatype_t** d, int32_t* type)
{
ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)pData->args;
if( NULL == pArgs ) { /* only for predefined datatypes */
if( ompi_datatype_is_predefined(pData) ) {
switch(which){
case 0:
*ci = 0;
*ca = 0;
*cd = 0;
*type = MPI_COMBINER_NAMED;
break;
default:
return MPI_ERR_INTERN;
}
return OMPI_SUCCESS;
}
return MPI_ERR_INTERN;
}
switch(which){
case 0: /* GET THE LENGTHS */
*ci = pArgs->ci;
*ca = pArgs->ca;
*cd = pArgs->cd;
*type = pArgs->create_type;
break;
case 1: /* GET THE ARGUMENTS */
if(*ci < pArgs->ci || *ca < pArgs->ca || *cd < pArgs->cd) {
return MPI_ERR_ARG;
}
if( (NULL != i) && (NULL != pArgs->i) ) {
memcpy( i, pArgs->i, pArgs->ci * sizeof(int) );
}
if( (NULL != a) && (NULL != pArgs->a) ) {
memcpy( a, pArgs->a, pArgs->ca * sizeof(OPAL_PTRDIFF_TYPE) );
}
if( (NULL != d) && (NULL != pArgs->d) ) {
memcpy( d, pArgs->d, pArgs->cd * sizeof(MPI_Datatype) );
}
break;
default:
return MPI_ERR_INTERN;
}
return OMPI_SUCCESS;
}
void ompi_datatype_dump( const ompi_datatype_t* pData )
{
size_t length;
int index = 0;
char* buffer;
length = pData->super.opt_desc.used + pData->super.desc.used;
length = length * 100 + 500;
buffer = (char*)malloc( length );
index += snprintf( buffer, length - index,
"Datatype %p[%s] id %d size %ld align %d opal_id %d length %d used %d\n"
"true_lb %ld true_ub %ld (true_extent %ld) lb %ld ub %ld (extent %ld)\n"
"nbElems %d loops %d flags %X (",
(void*)pData, pData->name, pData->id,
(long)pData->super.size, (int)pData->super.align, pData->super.id, (int)pData->super.desc.length, (int)pData->super.desc.used,
(long)pData->super.true_lb, (long)pData->super.true_ub, (long)(pData->super.true_ub - pData->super.true_lb),
(long)pData->super.lb, (long)pData->super.ub, (long)(pData->super.ub - pData->super.lb),
(int)pData->super.nbElems, (int)pData->super.btypes[OPAL_DATATYPE_LOOP], (int)pData->super.flags );
/* dump the flags */
if( ompi_datatype_is_predefined(pData) ) {
index += snprintf( buffer + index, length - index, "predefined " );
} else {
if( pData->super.flags & OPAL_DATATYPE_FLAG_COMMITED ) index += snprintf( buffer + index, length - index, "commited " );
if( pData->super.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) index += snprintf( buffer + index, length - index, "contiguous " );
}
index += snprintf( buffer + index, length - index, ")" );
index += _ompi_dump_data_flags( pData->super.flags, buffer + index, length - index );
{
index += snprintf( buffer + index, length - index, "\n contain " );
index += opal_datatype_contain_basic_datatypes( &pData->super, buffer + index, length - index );
index += snprintf( buffer + index, length - index, "\n" );
}
if( (pData->super.opt_desc.desc != pData->super.desc.desc) && (NULL != pData->super.opt_desc.desc) ) {
/* If the data is already committed print everything including the last
* fake DT_END_LOOP entry.
*/
index += opal_datatype_dump_data_desc( pData->super.desc.desc, pData->super.desc.used + 1, buffer + index, length - index );
index += snprintf( buffer + index, length - index, "Optimized description \n" );
index += opal_datatype_dump_data_desc( pData->super.opt_desc.desc, pData->super.opt_desc.used + 1, buffer + index, length - index );
} else {
index += opal_datatype_dump_data_desc( pData->super.desc.desc, pData->super.desc.used, buffer + index, length - index );
index += snprintf( buffer + index, length - index, "No optimized description\n" );
}
buffer[index] = '\0'; /* make sure we end the string with 0 */
opal_output( 0, "%s\n", buffer );
ompi_datatype_print_args ( pData );
free(buffer);
}
/* In the dt_add function we increase the reference count for all datatypes
* (except for the predefined ones) that get added to another datatype. This
* insure that they cannot get released until all the references to them
* get removed.
*/
int32_t ompi_datatype_release_args( ompi_datatype_t* pData )
{
int i;
ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)pData->args;
assert( 0 < pArgs->ref_count );
pArgs->ref_count--;
if( 0 == pArgs->ref_count ) {
/* There are some duplicated datatypes around that have a pointer to this
* args. We will release them only when the last datatype will dissapear.
*/
for( i = 0; i < pArgs->cd; i++ ) {
if( !(ompi_datatype_is_predefined(pArgs->d[i])) ) {
OBJ_RELEASE( pArgs->d[i] );
}
}
free( pData->args );
}
pData->args = NULL;
return OMPI_SUCCESS;
}
static ompi_datatype_t* __ompi_datatype_create_from_packed_description( void** packed_buffer,
const struct ompi_proc_t* remote_processor )
{
int* position;
ompi_datatype_t* datatype = NULL;
ompi_datatype_t** array_of_datatype;
OPAL_PTRDIFF_TYPE* array_of_disp;
int* array_of_length;
int number_of_length, number_of_disp, number_of_datatype, data_id;
int create_type, i;
char* next_buffer;
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
bool need_swap = false;
if( (remote_processor->super.proc_arch ^ ompi_proc_local()->super.proc_arch) &
OPAL_ARCH_ISBIGENDIAN ) {
need_swap = true;
}
#endif
next_buffer = (char*)*packed_buffer;
position = (int*)next_buffer;
create_type = position[0];
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (need_swap) {
create_type = opal_swap_bytes4(create_type);
}
#endif
if( MPI_COMBINER_NAMED == create_type ) {
/* there we have a simple predefined datatype */
data_id = position[1];
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (need_swap) {
data_id = opal_swap_bytes4(data_id);
}
#endif
assert( data_id < OMPI_DATATYPE_MAX_PREDEFINED );
*packed_buffer = position + 2;
return (ompi_datatype_t*)ompi_datatype_basicDatatypes[data_id];
}
number_of_length = position[1];
number_of_disp = position[2];
number_of_datatype = position[3];
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (need_swap) {
number_of_length = opal_swap_bytes4(number_of_length);
number_of_disp = opal_swap_bytes4(number_of_disp);
number_of_datatype = opal_swap_bytes4(number_of_datatype);
}
#endif
array_of_datatype = (ompi_datatype_t**)malloc( sizeof(ompi_datatype_t*) *
number_of_datatype );
next_buffer += (4 * sizeof(int)); /* move after the header */
/* description of the displacements (if ANY !) should always be aligned
on MPI_Aint, aka OPAL_PTRDIFF_TYPE */
if (number_of_disp > 0) {
OMPI_DATATYPE_ALIGN_PTR(next_buffer, char*);
}
array_of_disp = (OPAL_PTRDIFF_TYPE*)next_buffer;
next_buffer += number_of_disp * sizeof(OPAL_PTRDIFF_TYPE);
/* the other datatypes */
position = (int*)next_buffer;
next_buffer += number_of_datatype * sizeof(int);
/* the array of lengths (32 bits aligned) */
array_of_length = (int*)next_buffer;
next_buffer += (number_of_length * sizeof(int));
for( i = 0; i < number_of_datatype; i++ ) {
data_id = position[i];
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (need_swap) {
data_id = opal_swap_bytes4(data_id);
}
#endif
if( data_id < OMPI_DATATYPE_MAX_PREDEFINED ) {
array_of_datatype[i] = (ompi_datatype_t*)ompi_datatype_basicDatatypes[data_id];
continue;
}
array_of_datatype[i] =
__ompi_datatype_create_from_packed_description( (void**)&next_buffer,
remote_processor );
if( NULL == array_of_datatype[i] ) {
/* don't cleanup more than required. We can now modify these
* values as we already know we have failed to rebuild the
* datatype.
*/
array_of_datatype[i] = (ompi_datatype_t*)ompi_datatype_basicDatatypes[OPAL_DATATYPE_INT1]; /*XXX TODO */
number_of_datatype = i;
goto cleanup_and_exit;
}
}
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (need_swap) {
for (i = 0 ; i < number_of_length ; ++i) {
array_of_length[i] = opal_swap_bytes4(array_of_length[i]);
}
for (i = 0 ; i < number_of_disp ; ++i) {
#if SIZEOF_PTRDIFF_T == 4
array_of_disp[i] = opal_swap_bytes4(array_of_disp[i]);
#elif SIZEOF_PTRDIFF_T == 8
array_of_disp[i] = (MPI_Aint)opal_swap_bytes8(array_of_disp[i]);
#else
#error "Unknown size of ptrdiff_t"
#endif
}
}
#endif
datatype = __ompi_datatype_create_from_args( array_of_length, array_of_disp,
array_of_datatype, create_type );
*packed_buffer = next_buffer;
cleanup_and_exit:
for( i = 0; i < number_of_datatype; i++ ) {
if( !(ompi_datatype_is_predefined(array_of_datatype[i])) ) {
OBJ_RELEASE(array_of_datatype[i]);
}
}
free( array_of_datatype );
return datatype;
}
int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
const void** packed_buffer )
{
ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args;
int next_index = OMPI_DATATYPE_MAX_PREDEFINED;
void *packed_description = datatype->packed_description;
void* recursive_buffer;
if (NULL == packed_description) {
if (opal_atomic_cmpset (&datatype->packed_description, NULL, (void *) 1)) {
if( ompi_datatype_is_predefined(datatype) ) {
packed_description = malloc(2 * sizeof(int));
} else if( NULL == args ) {
return OMPI_ERROR;
} else {
packed_description = malloc(args->total_pack_size);
}
recursive_buffer = packed_description;
__ompi_datatype_pack_description( datatype, &recursive_buffer, &next_index );
if (!ompi_datatype_is_predefined(datatype)) {
args->total_pack_size = (uintptr_t)((char*)recursive_buffer - (char *) packed_description);
}
opal_atomic_wmb ();
datatype->packed_description = packed_description;
} else {
/* another thread beat us to it */
packed_description = datatype->packed_description;
}
}
if ((void *) 1 == packed_description) {
struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000};
/* wait until the packed description is updated */
while ((void *) 1 == datatype->packed_description) {
nanosleep (&interval, NULL);
}
packed_description = datatype->packed_description;
}
*packed_buffer = (const void *) packed_description;
return OMPI_SUCCESS;
}
size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype )
{
void *packed_description = datatype->packed_description;
if( ompi_datatype_is_predefined(datatype) ) {
return 2 * sizeof(int);
}
if( NULL == packed_description || (void *) 1 == packed_description) {
const void* buf;
int rc;
rc = ompi_datatype_get_pack_description(datatype, &buf);
if( OMPI_SUCCESS != rc ) {
return 0;
}
}
assert( NULL != (ompi_datatype_args_t*)datatype->args );
assert( NULL != (ompi_datatype_args_t*)datatype->packed_description );
return ((ompi_datatype_args_t*)datatype->args)->total_pack_size;
}
static inline int __ompi_datatype_pack_description( ompi_datatype_t* datatype,
void** packed_buffer, int* next_index )
{
int i, *position = (int*)*packed_buffer;
ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args;
char* next_packed = (char*)*packed_buffer;
if( ompi_datatype_is_predefined(datatype) ) {
position[0] = MPI_COMBINER_NAMED;
position[1] = datatype->id; /* On the OMPI - layer, copy the ompi_datatype.id */
next_packed += (2 * sizeof(int));
*packed_buffer = next_packed;
return OMPI_SUCCESS;
}
/* For duplicated datatype we don't have to store all the information */
if( MPI_COMBINER_DUP == args->create_type ) {
ompi_datatype_t* temp_data = args->d[0];
return __ompi_datatype_pack_description(temp_data,
packed_buffer,
next_index );
}
position[0] = args->create_type;
position[1] = args->ci;
position[2] = args->ca;
position[3] = args->cd;
next_packed += (4 * sizeof(int));
/* Spoiler: We will access the data in this storage structure, and thus we
* need to align it to the expected boundaries (special thanks to Sparc64).
* The simplest way is to ensure that prior to each type that must be 64
* bits aligned, we have a pointer that is 64 bits aligned. That will minimize
* the memory requirements in all cases where no displacements are stored.
*/
if( 0 < args->ca ) {
/* description of the displacements must be 64 bits aligned */
OMPI_DATATYPE_ALIGN_PTR(next_packed, char*);
memcpy( next_packed, args->a, sizeof(OPAL_PTRDIFF_TYPE) * args->ca );
next_packed += sizeof(OPAL_PTRDIFF_TYPE) * args->ca;
}
position = (int*)next_packed;
next_packed += sizeof(int) * args->cd;
/* copy the aray of counts (32 bits aligned) */
memcpy( next_packed, args->i, sizeof(int) * args->ci );
next_packed += args->ci * sizeof(int);
/* copy the rest of the data */
for( i = 0; i < args->cd; i++ ) {
ompi_datatype_t* temp_data = args->d[i];
if( ompi_datatype_is_predefined(temp_data) ) {
position[i] = temp_data->id; /* On the OMPI - layer, copy the ompi_datatype.id */
} else {
position[i] = *next_index;
(*next_index)++;
__ompi_datatype_pack_description( temp_data,
(void**)&next_packed,
next_index );
}
}
*packed_buffer = next_packed;
return OMPI_SUCCESS;
}
int32_t ompi_datatype_print_args( const ompi_datatype_t* pData )
{
int32_t i;
ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)pData->args;
if( ompi_datatype_is_predefined(pData) ) {
/* nothing to do for predefined data-types */
return OMPI_SUCCESS;
}
if( pArgs == NULL ) return MPI_ERR_INTERN;
printf( "type %d count ints %d count disp %d count datatype %d\n",
pArgs->create_type, pArgs->ci, pArgs->ca, pArgs->cd );
if( pArgs->i != NULL ) {
printf( "ints: " );
for( i = 0; i < pArgs->ci; i++ ) {
printf( "%d ", pArgs->i[i] );
}
printf( "\n" );
}
if( pArgs->a != NULL ) {
printf( "MPI_Aint: " );
for( i = 0; i < pArgs->ca; i++ ) {
printf( "%ld ", (long)pArgs->a[i] );
}
printf( "\n" );
}
if( pArgs->d != NULL ) {
int count = 1;
ompi_datatype_t *temp, *old;
printf( "types: " );
old = pArgs->d[0];
for( i = 1; i < pArgs->cd; i++ ) {
temp = pArgs->d[i];
if( old == temp ) {
count++;
continue;
}
if( count <= 1 ) {
if( ompi_datatype_is_predefined(old) )
printf( "%s ", old->name );
else
printf( "%p ", (void*)old );
} else {
if( ompi_datatype_is_predefined(old) )
printf( "(%d * %s) ", count, old->name );
else
printf( "(%d * %p) ", count, (void*)old );
}
count = 1;
old = temp;
}
if( count <= 1 ) {
if( ompi_datatype_is_predefined(old) )
printf( "%s ", old->name );
else
printf( "%p ", (void*)old );
} else {
if( ompi_datatype_is_predefined(old) )
printf( "(%d * %s) ", count, old->name );
else
printf( "(%d * %p) ", count, (void*)old );
}
printf( "\n" );
}
return OMPI_SUCCESS;
}
int32_t ompi_datatype_set_args( ompi_datatype_t* pData,
int32_t ci, const int32_t** i,
int32_t ca, const OPAL_PTRDIFF_TYPE* a,
int32_t cd, ompi_datatype_t* const * d, int32_t type)
{
int pos;
ompi_datatype_args_t* pArgs;
assert( NULL == pData->args );
ALLOC_ARGS( pData, ci, ca, cd );
pArgs = (ompi_datatype_args_t*)pData->args;
pArgs->create_type = type;
switch(type) {
case MPI_COMBINER_DUP:
pArgs->total_pack_size = 0; /* store no extra data */
break;
case MPI_COMBINER_CONTIGUOUS:
pArgs->i[0] = i[0][0];
break;
case MPI_COMBINER_VECTOR:
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
pArgs->i[2] = i[2][0];
break;
case MPI_COMBINER_HVECTOR_INTEGER:
case MPI_COMBINER_HVECTOR:
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
break;
case MPI_COMBINER_INDEXED:
pos = 1;
pArgs->i[0] = i[0][0];
memcpy( pArgs->i + pos, i[1], i[0][0] * sizeof(int) );
pos += i[0][0];
memcpy( pArgs->i + pos, i[2], i[0][0] * sizeof(int) );
break;
case MPI_COMBINER_HINDEXED_INTEGER:
case MPI_COMBINER_HINDEXED:
pArgs->i[0] = i[0][0];
memcpy( pArgs->i + 1, i[1], i[0][0] * sizeof(int) );
break;
case MPI_COMBINER_INDEXED_BLOCK:
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
memcpy( pArgs->i + 2, i[2], i[0][0] * sizeof(int) );
break;
case MPI_COMBINER_STRUCT_INTEGER:
case MPI_COMBINER_STRUCT:
pArgs->i[0] = i[0][0];
memcpy( pArgs->i + 1, i[1], i[0][0] * sizeof(int) );
break;
case MPI_COMBINER_SUBARRAY:
pos = 1;
pArgs->i[0] = i[0][0];
memcpy( pArgs->i + pos, i[1], pArgs->i[0] * sizeof(int) );
pos += pArgs->i[0];
memcpy( pArgs->i + pos, i[2], pArgs->i[0] * sizeof(int) );
pos += pArgs->i[0];
memcpy( pArgs->i + pos, i[3], pArgs->i[0] * sizeof(int) );
pos += pArgs->i[0];
pArgs->i[pos] = i[4][0];
break;
case MPI_COMBINER_DARRAY:
pos = 3;
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
pArgs->i[2] = i[2][0];
memcpy( pArgs->i + pos, i[3], i[2][0] * sizeof(int) );
pos += i[2][0];
memcpy( pArgs->i + pos, i[4], i[2][0] * sizeof(int) );
pos += i[2][0];
memcpy( pArgs->i + pos, i[5], i[2][0] * sizeof(int) );
pos += i[2][0];
memcpy( pArgs->i + pos, i[6], i[2][0] * sizeof(int) );
pos += i[2][0];
pArgs->i[pos] = i[7][0];
break;
case MPI_COMBINER_F90_REAL:
case MPI_COMBINER_F90_COMPLEX:
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
break;
case MPI_COMBINER_F90_INTEGER:
pArgs->i[0] = i[0][0];
break;
case MPI_COMBINER_RESIZED:
break;
case MPI_COMBINER_HINDEXED_BLOCK:
pArgs->i[0] = i[0][0];
pArgs->i[1] = i[1][0];
break;
default:
break;
}
/* copy the array of MPI_Aint, aka OPAL_PTRDIFF_TYPE */
if( pArgs->a != NULL )
memcpy( pArgs->a, a, ca * sizeof(OPAL_PTRDIFF_TYPE) );
for( pos = 0; pos < cd; pos++ ) {
pArgs->d[pos] = d[pos];
if( !(ompi_datatype_is_predefined(d[pos])) ) {
/* We handle a user defined datatype. We should make sure that the
* user will not have the oportunity to destroy it before all derived
* datatypes are destroyed. As we keep pointers to every datatype
* (for MPI_Type_get_content and MPI_Type_get_envelope) we have to make
* sure that those datatype will be available if the user ask for them.
* However, there is no easy way to free them in this case ...
*/
OBJ_RETAIN( d[pos] );
pArgs->total_pack_size += ((ompi_datatype_args_t*)d[pos]->args)->total_pack_size;
} else {
pArgs->total_pack_size += 2 * sizeof(int); /* _NAMED + predefined id */
}
}
return OMPI_SUCCESS;
}
int ompi_osc_ucx_get_accumulate(const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_dt,
void *result_addr, int result_count,
struct ompi_datatype_t *result_dt,
int target, ptrdiff_t target_disp,
int target_count, struct ompi_datatype_t *target_dt,
struct ompi_op_t *op, struct ompi_win_t *win) {
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
int ret = OMPI_SUCCESS;
ret = check_sync_state(module, target, false);
if (ret != OMPI_SUCCESS) {
return ret;
}
ret = start_atomicity(module, ep, target);
if (ret != OMPI_SUCCESS) {
return ret;
}
ret = ompi_osc_ucx_get(result_addr, result_count, result_dt, target,
target_disp, target_count, target_dt, win);
if (ret != OMPI_SUCCESS) {
return ret;
}
if (op != &ompi_mpi_op_no_op.op) {
if (op == &ompi_mpi_op_replace.op) {
ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt,
target, target_disp, target_count,
target_dt, win);
if (ret != OMPI_SUCCESS) {
return ret;
}
} else {
void *temp_addr = NULL;
uint32_t temp_count;
ompi_datatype_t *temp_dt;
ptrdiff_t temp_lb, temp_extent;
ucs_status_t status;
bool is_origin_contig = ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count);
if (ompi_datatype_is_predefined(target_dt)) {
temp_dt = target_dt;
temp_count = target_count;
} else {
ret = ompi_osc_base_get_primitive_type_info(target_dt, &temp_dt, &temp_count);
if (ret != OMPI_SUCCESS) {
return ret;
}
}
ompi_datatype_get_true_extent(temp_dt, &temp_lb, &temp_extent);
temp_addr = malloc(temp_extent * temp_count);
if (temp_addr == NULL) {
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
ret = ompi_osc_ucx_get(temp_addr, (int)temp_count, temp_dt,
target, target_disp, target_count, target_dt, win);
if (ret != OMPI_SUCCESS) {
return ret;
}
status = ucp_ep_flush(ep);
if (status != UCS_OK) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: ucp_ep_flush failed: %d\n",
__FILE__, __LINE__, status);
return OMPI_ERROR;
}
if (ompi_datatype_is_predefined(origin_dt) || is_origin_contig) {
ompi_op_reduce(op, (void *)origin_addr, temp_addr, (int)temp_count, temp_dt);
} else {
ucx_iovec_t *origin_ucx_iov = NULL;
uint32_t origin_ucx_iov_count = 0;
uint32_t origin_ucx_iov_idx = 0;
ret = create_iov_list(origin_addr, origin_count, origin_dt,
&origin_ucx_iov, &origin_ucx_iov_count);
if (ret != OMPI_SUCCESS) {
return ret;
}
if ((op != &ompi_mpi_op_maxloc.op && op != &ompi_mpi_op_minloc.op) ||
ompi_datatype_is_contiguous_memory_layout(temp_dt, temp_count)) {
size_t temp_size;
ompi_datatype_type_size(temp_dt, &temp_size);
while (origin_ucx_iov_idx < origin_ucx_iov_count) {
int curr_count = origin_ucx_iov[origin_ucx_iov_idx].len / temp_size;
ompi_op_reduce(op, origin_ucx_iov[origin_ucx_iov_idx].addr,
temp_addr, curr_count, temp_dt);
temp_addr = (void *)((char *)temp_addr + curr_count * temp_size);
origin_ucx_iov_idx++;
}
} else {
int i;
void *curr_origin_addr = origin_ucx_iov[origin_ucx_iov_idx].addr;
for (i = 0; i < (int)temp_count; i++) {
ompi_op_reduce(op, curr_origin_addr,
(void *)((char *)temp_addr + i * temp_extent),
1, temp_dt);
curr_origin_addr = (void *)((char *)curr_origin_addr + temp_extent);
origin_ucx_iov_idx++;
if (curr_origin_addr >= (void *)((char *)origin_ucx_iov[origin_ucx_iov_idx].addr + origin_ucx_iov[origin_ucx_iov_idx].len)) {
origin_ucx_iov_idx++;
curr_origin_addr = origin_ucx_iov[origin_ucx_iov_idx].addr;
}
}
}
free(origin_ucx_iov);
}
ret = ompi_osc_ucx_put(temp_addr, (int)temp_count, temp_dt, target, target_disp,
target_count, target_dt, win);
if (ret != OMPI_SUCCESS) {
return ret;
}
status = ucp_ep_flush(ep);
if (status != UCS_OK) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: ucp_ep_flush failed: %d\n",
__FILE__, __LINE__, status);
return OMPI_ERROR;
}
free(temp_addr);
}
}
ret = end_atomicity(module, ep, target);
return ret;
}
int mca_io_ompio_set_view_internal(mca_io_ompio_file_t *fh,
OMPI_MPI_OFFSET_TYPE disp,
ompi_datatype_t *etype,
ompi_datatype_t *filetype,
char *datarep,
ompi_info_t *info)
{
size_t max_data = 0;
int i;
int num_groups = 0;
contg *contg_groups;
size_t ftype_size;
OPAL_PTRDIFF_TYPE ftype_extent, lb, ub;
ompi_datatype_t *newfiletype;
if ( NULL != fh->f_etype ) {
ompi_datatype_destroy (&fh->f_etype);
}
if ( NULL != fh->f_filetype ) {
ompi_datatype_destroy (&fh->f_filetype);
}
if ( NULL != fh->f_orig_filetype ) {
ompi_datatype_destroy (&fh->f_orig_filetype);
}
if (NULL != fh->f_decoded_iov) {
free (fh->f_decoded_iov);
fh->f_decoded_iov = NULL;
}
if (NULL != fh->f_datarep) {
free (fh->f_datarep);
fh->f_datarep = NULL;
}
/* Reset the flags first */
fh->f_flags = 0;
fh->f_flags |= OMPIO_FILE_VIEW_IS_SET;
fh->f_datarep = strdup (datarep);
ompi_datatype_duplicate (filetype, &fh->f_orig_filetype );
opal_datatype_get_extent(&filetype->super, &lb, &ftype_extent);
opal_datatype_type_size (&filetype->super, &ftype_size);
if ( etype == filetype &&
ompi_datatype_is_predefined (filetype ) &&
ftype_extent == (OPAL_PTRDIFF_TYPE)ftype_size ){
ompi_datatype_create_contiguous(MCA_IO_DEFAULT_FILE_VIEW_SIZE,
&ompi_mpi_byte.dt,
&newfiletype);
ompi_datatype_commit (&newfiletype);
}
else {
newfiletype = filetype;
}
fh->f_iov_count = 0;
fh->f_disp = disp;
fh->f_offset = disp;
fh->f_total_bytes = 0;
ompi_io_ompio_decode_datatype (fh,
newfiletype,
1,
NULL,
&max_data,
&fh->f_decoded_iov,
&fh->f_iov_count);
opal_datatype_get_extent(&newfiletype->super, &lb, &fh->f_view_extent);
opal_datatype_type_ub (&newfiletype->super, &ub);
opal_datatype_type_size (&etype->super, &fh->f_etype_size);
opal_datatype_type_size (&newfiletype->super, &fh->f_view_size);
ompi_datatype_duplicate (etype, &fh->f_etype);
ompi_datatype_duplicate (newfiletype, &fh->f_filetype);
fh->f_cc_size = get_contiguous_chunk_size (fh);
if (opal_datatype_is_contiguous_memory_layout(&etype->super,1)) {
if (opal_datatype_is_contiguous_memory_layout(&filetype->super,1) &&
fh->f_view_extent == (OPAL_PTRDIFF_TYPE)fh->f_view_size ) {
fh->f_flags |= OMPIO_CONTIGUOUS_FVIEW;
}
}
contg_groups = (contg*) calloc ( 1, fh->f_size * sizeof(contg));
if (NULL == contg_groups) {
opal_output (1, "OUT OF MEMORY\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
for( i = 0; i < fh->f_size; i++){
contg_groups[i].procs_in_contg_group = (int*)calloc (1,fh->f_size * sizeof(int));
if(NULL == contg_groups[i].procs_in_contg_group){
int j;
opal_output (1, "OUT OF MEMORY\n");
for(j=0; j<i; j++) {
free(contg_groups[j].procs_in_contg_group);
}
free(contg_groups);
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
if( OMPI_SUCCESS != mca_io_ompio_fview_based_grouping(fh,
&num_groups,
contg_groups)){
opal_output(1, "mca_io_ompio_fview_based_grouping() failed\n");
free(contg_groups);
return OMPI_ERROR;
}
if( !( (fh->f_comm->c_flags & OMPI_COMM_CART) &&
(num_groups == 1 || num_groups == fh->f_size)) ) {
mca_io_ompio_finalize_initial_grouping(fh,
num_groups,
contg_groups);
}
for( i = 0; i < fh->f_size; i++){
free(contg_groups[i].procs_in_contg_group);
}
free(contg_groups);
if ( etype == filetype &&
ompi_datatype_is_predefined (filetype ) &&
ftype_extent == (OPAL_PTRDIFF_TYPE)ftype_size ){
ompi_datatype_destroy ( &newfiletype );
}
if (OMPI_SUCCESS != mca_fcoll_base_file_select (fh, NULL)) {
opal_output(1, "mca_fcoll_base_file_select() failed\n");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_io_ompio_file_set_view (ompi_file_t *fp,
OMPI_MPI_OFFSET_TYPE disp,
ompi_datatype_t *etype,
ompi_datatype_t *filetype,
char *datarep,
ompi_info_t *info)
{
mca_io_ompio_data_t *data;
mca_io_ompio_file_t *fh;
size_t ftype_size;
OPAL_PTRDIFF_TYPE ftype_extent, lb;
data = (mca_io_ompio_data_t *) fp->f_io_selected_data;
fh = &data->ompio_fh;
ompi_datatype_destroy (&fh->f_etype);
ompi_datatype_destroy (&fh->f_filetype);
ompi_datatype_destroy (&fh->f_orig_filetype);
if (NULL != fh->f_decoded_iov) {
free (fh->f_decoded_iov);
fh->f_decoded_iov = NULL;
}
if (NULL != fh->f_datarep) {
free (fh->f_datarep);
fh->f_datarep = NULL;
}
/* Reset the flags first */
fh->f_flags = 0;
fh->f_flags |= OMPIO_FILE_VIEW_IS_SET;
fh->f_datarep = strdup (datarep);
ompi_datatype_duplicate (filetype, &fh->f_orig_filetype );
opal_datatype_get_extent(&filetype->super, &lb, &ftype_extent);
opal_datatype_type_size (&filetype->super, &ftype_size);
if ( etype == filetype &&
ompi_datatype_is_predefined (filetype ) &&
ftype_extent == (OPAL_PTRDIFF_TYPE)ftype_size ) {
ompi_datatype_t *newfiletype;
ompi_datatype_create_contiguous(MCA_IO_DEFAULT_FILE_VIEW_SIZE,
&ompi_mpi_byte.dt,
&newfiletype);
ompi_datatype_commit (&newfiletype);
mca_io_ompio_set_view_internal (fh,
disp,
etype,
newfiletype,
datarep,
info);
ompi_datatype_destroy ( &newfiletype );
}
else {
mca_io_ompio_set_view_internal (fh,
disp,
etype,
filetype,
datarep,
info);
}
if (OMPI_SUCCESS != mca_fcoll_base_file_select (&data->ompio_fh,
NULL)) {
opal_output(1, "mca_fcoll_base_file_select() failed\n");
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_buffer, int source_count,
ompi_datatype_t *source_datatype, void *result_buffer, int result_count,
ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
mca_btl_base_registration_handle_t *target_handle, int target_count,
ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
{
ompi_osc_rdma_module_t *module = sync->module;
struct iovec source_iovec[OMPI_OSC_RDMA_DECODE_MAX], target_iovec[OMPI_OSC_RDMA_DECODE_MAX];
const size_t acc_limit = (mca_osc_rdma_component.buffer_size >> 3);
uint32_t source_primitive_count, target_primitive_count;
opal_convertor_t source_convertor, target_convertor;
uint32_t source_iov_count, target_iov_count;
uint32_t source_iov_index, target_iov_index;
ompi_datatype_t *source_primitive, *target_primitive;
/* needed for opal_convertor_raw but not used */
size_t source_size, target_size;
ompi_osc_rdma_request_t *subreq;
size_t result_position;
ptrdiff_t lb, extent;
int ret, acc_len;
bool done;
(void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
target_address += lb;
/* fast path for accumulate on built-in types */
if (OPAL_LIKELY((!source_count || ompi_datatype_is_predefined (source_datatype)) &&
ompi_datatype_is_predefined (target_datatype) &&
(!result_count || ompi_datatype_is_predefined (result_datatype)) &&
(target_datatype->super.size * target_count <= acc_limit))) {
if (NULL == request) {
OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, request);
request->internal = true;
request->type = result_datatype ? OMPI_OSC_RDMA_TYPE_GET_ACC : OMPI_OSC_RDMA_TYPE_ACC;
}
if (source_datatype) {
(void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
source_buffer = (void *)((intptr_t) source_buffer + lb);
}
if (result_datatype) {
(void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
result_buffer = (void *)((intptr_t) result_buffer + lb);
}
ret = ompi_osc_rdma_gacc_contig (sync, source_buffer, source_count, source_datatype, result_buffer,
result_count, result_datatype, peer, target_address,
target_handle, target_count, target_datatype, op,
request);
if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
return OMPI_SUCCESS;
}
if (source_datatype) {
/* the convertors will handle the lb */
(void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
source_buffer = (void *)((intptr_t) source_buffer - lb);
}
if (result_datatype) {
(void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
result_buffer = (void *)((intptr_t) result_buffer - lb);
}
}
/* the convertor will handle lb from here */
(void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
target_address -= lb;
/* get the primitive datatype info */
ret = ompi_osc_base_get_primitive_type_info (target_datatype, &target_primitive, &target_primitive_count);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
/* target datatype is not made up of a single basic datatype */
return ret;
}
if (source_datatype) {
ret = ompi_osc_base_get_primitive_type_info (source_datatype, &source_primitive, &source_primitive_count);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
/* target datatype is not made up of a single basic datatype */
return ret;
}
if (OPAL_UNLIKELY(source_primitive != target_primitive)) {
return MPI_ERR_TYPE;
}
}
/* prepare convertors for the source and target. these convertors will be used to determine the
* contiguous segments within the source and target. */
/* the source may be NULL if using MPI_OP_NO_OP with MPI_Get_accumulate */
if (source_datatype) {
OBJ_CONSTRUCT(&source_convertor, opal_convertor_t);
ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &source_datatype->super, source_count, source_buffer,
0, &source_convertor);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
}
/* target_datatype can never be NULL */
OBJ_CONSTRUCT(&target_convertor, opal_convertor_t);
ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count,
(void *) (intptr_t) target_address, 0, &target_convertor);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
if (request) {
/* keep the request from completing until all the transfers have started */
request->outstanding_requests = 1;
}
target_iov_index = 0;
target_iov_count = 0;
result_position = 0;
do {
/* decode segments of the source data */
source_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
source_iov_index = 0;
/* opal_convertor_raw returns done when it has reached the end of the data */
if (!source_datatype) {
done = true;
source_iovec[0].iov_len = (size_t) -1;
source_iovec[0].iov_base = NULL;
source_iov_count = 1;
} else {
done = opal_convertor_raw (&source_convertor, source_iovec, &source_iov_count, &source_size);
}
/* loop on the target segments until we have exhaused the decoded source data */
while (source_iov_index != source_iov_count) {
if (target_iov_index == target_iov_count) {
/* decode segments of the target buffer */
target_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
target_iov_index = 0;
(void) opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
}
/* we already checked that the target was large enough. this should be impossible */
assert (0 != target_iov_count);
/* determine how much to put in this operation */
acc_len = min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len);
acc_len = min((size_t) acc_len, acc_limit);
/* execute the get */
OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, subreq);
subreq->internal = true;
subreq->parent_request = request;
if (request) {
(void) OPAL_THREAD_ADD32 (&request->outstanding_requests, 1);
}
if (result_datatype) {
/* prepare a convertor for this part of the result */
opal_convertor_copy_and_prepare_for_recv (ompi_mpi_local_convertor, &result_datatype->super, result_count,
result_buffer, 0, &subreq->convertor);
opal_convertor_set_position (&subreq->convertor, &result_position);
subreq->type = OMPI_OSC_RDMA_TYPE_GET_ACC;
} else {
subreq->type = OMPI_OSC_RDMA_TYPE_ACC;
}
OPAL_OUTPUT_VERBOSE((60, ompi_osc_base_framework.framework_output,
"target index = %d, target = {%p, %lu}, source_index = %d, source = {%p, %lu}, result = %p, result position = %lu, "
"acc_len = %d, count = %lu",
target_iov_index, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len,
source_iov_index, source_iovec[source_iov_index].iov_base, (unsigned long) source_iovec[source_iov_index].iov_len,
result_buffer, (unsigned long) result_position, acc_len, (unsigned long)(acc_len / target_primitive->super.size)));
ret = ompi_osc_rdma_gacc_contig (sync, source_iovec[source_iov_index].iov_base, acc_len / target_primitive->super.size,
target_primitive, NULL, 0, NULL, peer, (uint64_t) (intptr_t) target_iovec[target_iov_index].iov_base,
target_handle, acc_len / target_primitive->super.size, target_primitive, op, subreq);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE != ret)) {
/* something bad happened. need to figure out how to handle these errors */
return ret;
}
/* progress and try again */
ompi_osc_rdma_progress (module);
continue;
}
/* adjust io vectors */
target_iovec[target_iov_index].iov_len -= acc_len;
source_iovec[source_iov_index].iov_len -= acc_len;
target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + acc_len);
source_iovec[source_iov_index].iov_base = (void *)((intptr_t) source_iovec[source_iov_index].iov_base + acc_len);
result_position += acc_len;
source_iov_index += !source_datatype || (0 == source_iovec[source_iov_index].iov_len);
target_iov_index += (0 == target_iovec[target_iov_index].iov_len);
}
} while (!done);
if (request) {
/* release our reference so the request can complete */
(void) OPAL_THREAD_ADD32 (&request->outstanding_requests, -1);
}
if (source_datatype) {
opal_convertor_cleanup (&source_convertor);
OBJ_DESTRUCT(&source_convertor);
}
opal_convertor_cleanup (&target_convertor);
OBJ_DESTRUCT(&target_convertor);
return OMPI_SUCCESS;
}
int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
const void** packed_buffer )
{
ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args;
int next_index = OMPI_DATATYPE_MAX_PREDEFINED;
void *packed_description = (void *) datatype->packed_description;
void* recursive_buffer;
if (NULL == packed_description) {
void *_tmp_ptr = NULL;
if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (intptr_t *) &_tmp_ptr, 1)) {
if( ompi_datatype_is_predefined(datatype) ) {
packed_description = malloc(2 * sizeof(int));
} else if( NULL == args ) {
return OMPI_ERROR;
} else {
packed_description = malloc(args->total_pack_size);
}
recursive_buffer = packed_description;
__ompi_datatype_pack_description( datatype, &recursive_buffer, &next_index );
if (!ompi_datatype_is_predefined(datatype)) {
/* If the precomputed size is not large enough we're already in troubles, we
* have overwritten outside of the allocated buffer. Raise the alarm !
* If not reassess the size of the packed buffer necessary for holding the
* datatype description.
*/
assert(args->total_pack_size >= (uintptr_t)((char*)recursive_buffer - (char *) packed_description));
args->total_pack_size = (uintptr_t)((char*)recursive_buffer - (char *) packed_description);
}
opal_atomic_wmb ();
datatype->packed_description = (intptr_t) packed_description;
} else {
/* another thread beat us to it */
packed_description = (void *) datatype->packed_description;
}
}
if ((void *) 1 == packed_description) {
struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000};
/* wait until the packed description is updated */
while (1 == datatype->packed_description) {
nanosleep (&interval, NULL);
}
packed_description = (void *) datatype->packed_description;
}
*packed_buffer = (const void *) packed_description;
return OMPI_SUCCESS;
}
size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype )
{
void *packed_description = (void *) datatype->packed_description;
if( ompi_datatype_is_predefined(datatype) ) {
return 2 * sizeof(int);
}
if( NULL == packed_description || (void *) 1 == packed_description) {
const void* buf;
int rc;
rc = ompi_datatype_get_pack_description(datatype, &buf);
if( OMPI_SUCCESS != rc ) {
return 0;
}
}
assert( NULL != (ompi_datatype_args_t*)datatype->args );
assert( NULL != (ompi_datatype_args_t*)datatype->packed_description );
return ((ompi_datatype_args_t*)datatype->args)->total_pack_size;
}
