/**
* Main function. Call several tests and print-out the results. It try to stress the convertor
* using difficult data-type constructions as well as strange segment sizes for the conversion.
* Usually, it is able to detect most of the data-type and convertor problems. Any modifications
* on the data-type engine should first pass all the tests from this file, before going into other
* tests.
*/
int main( int argc, char* argv[] )
{
opal_datatype_init();
/**
* By default simulate homogeneous architectures.
*/
remote_arch = opal_local_arch ^ OPAL_ARCH_ISBIGENDIAN;
opal_convertor_t * pConv;
int sbuf[2], rbuf[2];
size_t max_data;
struct iovec a;
uint32_t iov_count;
sbuf[0] = 0x01000000; sbuf[1] = 0x02000000;
printf( "\n\n#\n * TEST UNPACKING 1 int out of 1\n#\n\n" );
pConv = opal_convertor_create( remote_arch, 0 );
rbuf[0] = -1; rbuf[1] = -1;
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 1, rbuf ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OPAL_ERROR;
}
a.iov_base = sbuf;
a.iov_len = 4;
iov_count = 1;
max_data = 4;
opal_unpack_general( pConv, &a, &iov_count, &max_data );
assert(1 == rbuf[0]);
assert(-1 == rbuf[1]);
OBJ_RELEASE(pConv);
printf( "\n\n#\n * TEST UNPACKING 1 int out of 2\n#\n\n" );
pConv = opal_convertor_create( remote_arch, 0 );
rbuf[0] = -1; rbuf[1] = -1;
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 2, rbuf ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OPAL_ERROR;
}
a.iov_base = sbuf;
a.iov_len = 4;
iov_count = 1;
max_data = 4;
opal_unpack_general( pConv, &a, &iov_count, &max_data );
assert(1 == rbuf[0]);
assert(-1 == rbuf[1]);
OBJ_RELEASE(pConv);
/* clean-ups all data allocations */
opal_datatype_finalize();
opal_finalize();
return OPAL_SUCCESS;
}
int32_t ompi_datatype_default_convertors_init( void )
{
/* create the extern32 convertor */
ompi_mpi_external32_convertor = opal_convertor_create( ompi_datatype_external32_arch_id, 0 );
/* create the local convertor */
ompi_mpi_local_convertor = opal_convertor_create( opal_local_arch, 0 );
return OMPI_SUCCESS;
}
int oshmem_proc_init(void)
{
orte_vpid_t i;
OBJ_CONSTRUCT(&oshmem_proc_list, opal_list_t);
OBJ_CONSTRUCT(&oshmem_proc_lock, opal_mutex_t);
oshmem_shmem_local_convertor = opal_convertor_create(opal_local_arch, 0);
size_t ompi_num_procs;
ompi_proc_t **ompi_procs = ompi_proc_world(&ompi_num_procs);
/* create proc structures and find self */
for (i = 0; i < orte_process_info.num_procs; i++) {
oshmem_proc_t *proc = OBJ_NEW(oshmem_proc_t);
opal_list_append(&oshmem_proc_list, (opal_list_item_t*)proc);
proc->super.proc_name = ompi_procs[i]->super.proc_name;
proc->super.proc_arch = ompi_procs[i]->super.proc_arch;
proc->super.proc_flags = ompi_procs[i]->super.proc_flags;
proc->super.proc_hostname = ompi_procs[i]->super.proc_hostname;
if (i == ORTE_PROC_MY_NAME->vpid) {
oshmem_proc_local_proc = proc;
}
}
if (ompi_procs)
free(ompi_procs);
return OSHMEM_SUCCESS;
}
static int
create_segments( ompi_datatype_t* datatype, int count,
size_t segment_length,
ddt_segment_t** segments, int* seg_count )
{
size_t data_size, total_length, position;
opal_convertor_t* convertor;
int i;
ddt_segment_t* segment;
ompi_datatype_type_size( datatype, &data_size );
data_size *= count;
*seg_count = data_size / segment_length;
if( ((*seg_count) * segment_length) != data_size )
*seg_count += 1;
allocate_segments:
*segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) );
convertor = opal_convertor_create( opal_local_arch, 0 );
opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL );
position = 0;
total_length = 0;
for( i = 0; i < (*seg_count); i++ ) {
segment = &((*segments)[i]);
segment->buffer = malloc(segment_length);
segment->position = position;
/* Find the end of the segment */
position += segment_length;
opal_convertor_set_position( convertor, &position );
segment->size = position - segment->position;
total_length += segment->size;
}
OBJ_RELEASE(convertor);
if( total_length != data_size ) {
for( i = 0; i < (*seg_count); i++ ) {
segment = &((*segments)[i]);
free(segment->buffer);
}
free( *segments );
(*seg_count) += 1;
goto allocate_segments;
}
return 0;
}
static int test_upper( unsigned int length )
{
ompi_datatype_t *pdt;
opal_convertor_t * pConv;
int rc = OMPI_SUCCESS;
unsigned int i, iov_count, split_chunk, total_length;
size_t max_data;
struct iovec iov[5];
TIMER_DATA_TYPE start, end;
long total_time;
printf( "test upper matrix\n" );
pdt = upper_matrix( length );
/*dt_dump( pdt );*/
total_length = length * (length + 1) * ( sizeof(double) / 2);
pConv = opal_convertor_create( remote_arch, 0 );
if( OMPI_SUCCESS != opal_convertor_prepare_for_send( pConv, &(pdt->super), 1, NULL ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OMPI_ERROR;
}
GET_TIME( start );
split_chunk = (length + 1) * sizeof(double);
/* split_chunk = (total_length + 1) * sizeof(double); */
for( i = total_length; i > 0; ) {
iov_count = 5;
max_data = 0;
opal_convertor_raw( pConv, iov, &iov_count, &max_data );
i -= max_data;
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "complete raw in %ld microsec\n", total_time );
/* test the automatic destruction pf the data */
ompi_datatype_destroy( &pdt );
assert( pdt == NULL );
OBJ_RELEASE( pConv );
return rc;
}
/**
* Conversion function. They deal with data-types in 3 ways, always making local copies.
* In order to allow performance testings, there are 3 functions:
* - one copying directly from one memory location to another one using the
* data-type copy function.
* - one which use a 2 convertors created with the same data-type
* - and one using 2 convertors created from different data-types.
*
*/
static int local_copy_ddt_raw( ompi_datatype_t* pdt, int count, int iov_num )
{
struct iovec* iov;
opal_convertor_t* convertor;
TIMER_DATA_TYPE start, end;
long total_time;
uint32_t iov_count = iov_num;
size_t max_data = 0, remaining_length;
iov = (struct iovec*)malloc(iov_num * sizeof(struct iovec));
convertor = opal_convertor_create( remote_arch, 0 );
if( OMPI_SUCCESS != opal_convertor_prepare_for_send( convertor, &(pdt->super), count, NULL ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OMPI_ERROR;
}
remaining_length = count * pdt->super.size;
GET_TIME( start );
while( 0 == opal_convertor_raw(convertor, iov, &iov_count, &max_data) ) {
#if 0
printf( "New raw extraction (iov_count = %d, max_data = %zu)\n",
iov_count, max_data );
for( i = 0; i < iov_count; i++ ) {
printf( "\t{%p, %d}\n", iov[i].iov_base, iov[i].iov_len );
}
#endif
remaining_length -= max_data;
iov_count = iov_num;
}
remaining_length -= max_data;
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "raw extraction in %ld microsec\n", total_time );
OBJ_RELEASE( convertor );
if( remaining_length != 0 ) {
printf( "Not all raw description was been extracted (%lu bytes missing)\n",
(unsigned long) remaining_length );
}
free(iov);
return OMPI_SUCCESS;
}
/* in some cases, all PE procs are required to do a modex so they
* can (at the least) exchange their architecture. Since we cannot
* know in advance if this was required, we provide a separate function
* to set the arch (instead of doing it inside of oshmem_proc_init) that
* can be called after the modex completes in oshmem_shmem_init. Thus, we
* know that - regardless of how the arch is known, whether via modex
* or dropped in from a local daemon - the arch can be set correctly
* at this time
*/
int oshmem_proc_set_arch(void)
{
oshmem_proc_t *proc = NULL;
opal_list_item_t *item = NULL;
int ret = OSHMEM_SUCCESS;
OPAL_THREAD_LOCK(&oshmem_proc_lock);
for (item = opal_list_get_first(&oshmem_proc_list);
item != opal_list_get_end(&oshmem_proc_list);
item = opal_list_get_next(item)) {
proc = (oshmem_proc_t*) item;
if (OSHMEM_PROC_VPID(proc) != ORTE_PROC_MY_NAME->vpid) {
/* if arch is different than mine, create a new convertor for this proc */
if (proc->super.proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(proc->super.proc_convertor);
proc->super.proc_convertor = opal_convertor_create(proc->super.proc_arch, 0);
#else
orte_show_help("help-shmem-runtime.txt",
"heterogeneous-support-unavailable",
true,
orte_process_info.nodename,
proc->super.proc_hostname == NULL ?
"<hostname unavailable>" :
proc->super.proc_hostname);
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return OSHMEM_ERR_NOT_SUPPORTED;
#endif
}
}
}
/* Set predefined groups */
ret = oshmem_proc_group_init();
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return ret;
}
static int
unpack_segments( ompi_datatype_t* datatype, int count,
size_t segment_size,
ddt_segment_t* segments, int seg_count,
void* buffer )
{
opal_convertor_t* convertor;
size_t max_size, position;
int i;
uint32_t iov_count;
struct iovec iov;
convertor = opal_convertor_create( opal_local_arch, 0 );
opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer );
for( i = 0; i < seg_count; i++ ) {
iov.iov_len = segments[i].size;
iov.iov_base = segments[i].buffer;
max_size = iov.iov_len;
position = segments[i].position;
opal_convertor_set_position( convertor, &position );
if( position != segments[i].position ) {
opal_output( 0, "Setting position failed (%lu != %lu)\n",
(unsigned long)segments[i].position, (unsigned long)position );
break;
}
iov_count = 1;
opal_convertor_unpack( convertor, &iov, &iov_count, &max_size );
if( max_size != segments[i].size ) {
opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n",
(unsigned long)max_size, (unsigned long)segments[i].size );
opal_output( 0, "Segment %d position %lu size %lu\n", i,
(unsigned long)segments[i].position, segments[i].size );
}
}
OBJ_RELEASE(convertor);
return 0;
}
int
ompio_io_ompio_file_open (ompi_communicator_t *comm,
char *filename,
int amode,
ompi_info_t *info,
mca_io_ompio_file_t *ompio_fh, bool use_sharedfp)
{
int ret = OMPI_SUCCESS;
int remote_arch;
if ( ((amode&MPI_MODE_RDONLY)?1:0) + ((amode&MPI_MODE_RDWR)?1:0) +
((amode&MPI_MODE_WRONLY)?1:0) != 1 ) {
return MPI_ERR_AMODE;
}
if ((amode & MPI_MODE_RDONLY) &&
((amode & MPI_MODE_CREATE) || (amode & MPI_MODE_EXCL))) {
return MPI_ERR_AMODE;
}
if ((amode & MPI_MODE_RDWR) && (amode & MPI_MODE_SEQUENTIAL)) {
return MPI_ERR_AMODE;
}
ompio_fh->f_iov_type = MPI_DATATYPE_NULL;
ompio_fh->f_rank = ompi_comm_rank (comm);
ompio_fh->f_size = ompi_comm_size (comm);
remote_arch = opal_local_arch;
ompio_fh->f_convertor = opal_convertor_create (remote_arch, 0);
ret = ompi_comm_dup (comm, &ompio_fh->f_comm);
if ( ret != OMPI_SUCCESS ) {
goto fn_fail;
}
ompio_fh->f_fstype = NONE;
ompio_fh->f_amode = amode;
ompio_fh->f_info = info;
ompio_fh->f_atomicity = 0;
ompi_io_ompio_set_file_defaults (ompio_fh);
ompio_fh->f_filename = filename;
/*Initialize the print_queues queues here!*/
coll_write_time = (print_queue *) malloc (sizeof(print_queue));
coll_read_time = (print_queue *) malloc (sizeof(print_queue));
ompi_io_ompio_initialize_print_queue(coll_write_time);
ompi_io_ompio_initialize_print_queue(coll_read_time);
/*
if (MPI_INFO_NULL != info) {
ret = ompi_info_dup (info, &ompio_fh->f_info);
if (OMPI_SUCCESS != ret) {
goto fn_fail;
}
}
*/
/* This fix is needed for data seiving to work with
two-phase collective I/O */
if ((amode & MPI_MODE_WRONLY)) {
amode -= MPI_MODE_WRONLY;
amode += MPI_MODE_RDWR;
}
/*--------------------------------------------------*/
if (OMPI_SUCCESS != (ret = mca_fs_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fs_base_file_select() failed\n");
goto fn_fail;
}
if (OMPI_SUCCESS != (ret = mca_fbtl_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fbtl_base_file_select() failed\n");
goto fn_fail;
}
if (OMPI_SUCCESS != (ret = mca_fcoll_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fcoll_base_file_select() failed\n");
goto fn_fail;
}
ompio_fh->f_sharedfp_component = NULL; /*component*/
ompio_fh->f_sharedfp = NULL; /*module*/
ompio_fh->f_sharedfp_data = NULL; /*data*/
if (OMPI_SUCCESS != (ret = mca_sharedfp_base_file_select (ompio_fh, NULL))) {
opal_output(1, "mca_sharedfp_base_file_select() failed\n");
goto fn_fail;
}
/*Determine topology information if set*/
if (ompio_fh->f_comm->c_flags & OMPI_COMM_CART) {
ret = mca_io_ompio_cart_based_grouping(ompio_fh);
if(OMPI_SUCCESS != ret ) {
ret = MPI_ERR_FILE;
}
}
ret = ompio_fh->f_fs->fs_file_open (comm,
filename,
amode,
info,
ompio_fh);
if ( OMPI_SUCCESS != ret ) {
ret = MPI_ERR_FILE;
goto fn_fail;
}
/* open the file once more for the shared file pointer if required.
** Per default, the shared file pointer specific actions are however
** only performed on first access of the shared file pointer, except
** for the addproc sharedfp component.
**
** Lazy open does not work for the addproc sharedfp
** component since it starts by spawning a process using MPI_Comm_spawn.
** For this, the first operation has to be collective which we can
** not guarantuee outside of the MPI_File_open operation.
*/
if ( true == use_sharedfp &&
(!mca_io_ompio_sharedfp_lazy_open ||
!strcmp (ompio_fh->f_sharedfp_component->mca_component_name,
"addproc") )) {
ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm,
filename,
amode,
info,
ompio_fh);
if ( OMPI_SUCCESS != ret ) {
goto fn_fail;
}
}
/* If file has been opened in the append mode, move the internal
file pointer of OMPIO to the very end of the file. */
if ( ompio_fh->f_amode & MPI_MODE_APPEND ) {
OMPI_MPI_OFFSET_TYPE current_size;
ompio_fh->f_fs->fs_file_get_size( ompio_fh,
¤t_size);
ompi_io_ompio_set_explicit_offset (ompio_fh, current_size);
}
return OMPI_SUCCESS;
fn_fail:
/* no need to free resources here, since the destructor
* is calling mca_io_ompio_file_close, which actually gets
*rid of all allocated memory items */
return ret;
}
static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk )
{
OPAL_PTRDIFF_TYPE lb, extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, malloced_size;
int32_t done1 = 0, done2 = 0, errors = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
malloced_size = compute_memory_size(pdt, count);
opal_datatype_get_extent( pdt, &lb, &extent );
odst = (char*)malloc( malloced_size );
osrc = (char*)malloc( malloced_size );
ptemp = malloc( chunk );
{
for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ );
memcpy(odst, osrc, malloced_size);
}
pdst = odst - lb;
psrc = osrc - lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
struct dt_stack_t stack[1+send_convertor->stack_pos];
int i, stack_pos = send_convertor->stack_pos;
size_t pos;
if( 0 == done1 ) {
memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != send_convertor->pStack[i].index )
{errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, send_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != send_convertor->pStack[i].count ) {
if( stack[i].type == send_convertor->pStack[i].type ) {
{errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, send_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) )
{errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)send_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) )
{errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)send_convertor->pStack[i].type, send_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != send_convertor->pStack[i].disp )
{errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
if( 0 == done2 ) {
memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != recv_convertor->pStack[i].index )
{errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, recv_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != recv_convertor->pStack[i].count ) {
if( stack[i].type == recv_convertor->pStack[i].type ) {
{errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, recv_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) )
{errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)recv_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) )
{errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != recv_convertor->pStack[i].disp )
{errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying same data-type using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
if(outputFlags & VALIDATE_DATA) {
for( size_t i = errors = 0; i < malloced_size; i++ ) {
if( odst[i] != osrc[i] ) {
printf("error at position %lu (%d != %d)\n",
(unsigned long)i, (int)(odst[i]), (int)(osrc[i]));
errors++;
if(outputFlags & QUIT_ON_FIRST_ERROR) {
opal_datatype_dump(pdt);
assert(0); exit(-1);
}
}
}
if( 0 == errors ) {
printf("Validation check succesfully passed\n");
} else {
printf("Found %d errors. Giving up!\n", errors);
exit(-1);
}
}
clean_and_return:
if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return (0 == errors ? OPAL_SUCCESS : errors);
}
static int
local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count,
opal_datatype_t const * const recv_type, int recv_count,
int chunk )
{
OPAL_PTRDIFF_TYPE send_lb, send_extent, recv_lb, recv_extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, send_malloced_size, recv_malloced_size;;
int32_t done1 = 0, done2 = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
send_malloced_size = compute_memory_size(send_type, send_count);
recv_malloced_size = compute_memory_size(recv_type, recv_count);
opal_datatype_get_extent( send_type, &send_lb, &send_extent );
opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent );
odst = (char*)malloc( recv_malloced_size );
osrc = (char*)malloc( send_malloced_size );
ptemp = malloc( chunk );
/* fill up the receiver with ZEROS */
{
for( size_t i = 0; i < send_malloced_size; i++ )
osrc[i] = i % 128 + 32;
}
memset( odst, 0, recv_malloced_size );
pdst = odst - recv_lb;
psrc = osrc - send_lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
size_t pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying different data-types using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
clean_and_return:
if( send_convertor != NULL ) {
OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
}
if( recv_convertor != NULL ) {
OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
}
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return OPAL_SUCCESS;
}
int oshmem_proc_refresh(void)
{
oshmem_proc_t *proc = NULL;
opal_list_item_t *item = NULL;
orte_vpid_t i = 0;
int hostname_length = 0;
OPAL_THREAD_LOCK(&oshmem_proc_lock);
for (item = opal_list_get_first(&oshmem_proc_list), i = 0;
item != opal_list_get_end(&oshmem_proc_list);
item = opal_list_get_next(item), ++i) {
proc = (oshmem_proc_t*) item;
/* Does not change: proc->proc_name.vpid */
proc->proc_name.jobid = ORTE_PROC_MY_NAME->jobid;
/* Make sure to clear the local flag before we set it below */
proc->proc_flags = 0;
proc->proc_arch = opal_local_arch;
oshmem_shmem_exchange_bcast(&proc->proc_arch,
sizeof(uint32_t),
i);
hostname_length = strlen(orte_process_info.nodename);
oshmem_shmem_exchange_bcast(&hostname_length,
sizeof(int),
i);
if (proc->proc_hostname)
free(proc->proc_hostname);
proc->proc_hostname = (
i == ORTE_PROC_MY_NAME->vpid ?
strdup(orte_process_info.nodename) :
(char *) malloc(hostname_length));
oshmem_shmem_exchange_bcast(proc->proc_hostname,
hostname_length,
i);
if (i == ORTE_PROC_MY_NAME->vpid) {
oshmem_proc_local_proc = proc;
} else {
/* if arch is different than mine, create a new convertor for this proc */
if (proc->proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(proc->proc_convertor);
proc->proc_convertor = opal_convertor_create(proc->proc_arch, 0);
#else
orte_show_help("help-shmem-runtime.txt",
"heterogeneous-support-unavailable",
true,
orte_process_info.nodename,
proc->proc_hostname == NULL ?
"<hostname unavailable>" :
proc->proc_hostname);
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return OSHMEM_ERR_NOT_SUPPORTED;
#endif
}
}
}
OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
return OSHMEM_SUCCESS;
}
static int local_copy_with_convertor( const opal_datatype_t const* pdt, int count, int chunk )
{
OPAL_PTRDIFF_TYPE extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data;
int32_t length = 0, done1 = 0, done2 = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
opal_datatype_type_extent( pdt, &extent );
pdst = malloc( extent * count );
psrc = malloc( extent * count );
ptemp = malloc( chunk );
{
int i = 0;
for( ; i < (count * extent); ((char*)psrc)[i] = i % 128 + 32, i++ );
}
memset( pdst, 0, count * extent );
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying same data-type using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
clean_and_return:
if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
if( NULL != pdst ) free( pdst );
if( NULL != psrc ) free( psrc );
if( NULL != ptemp ) free( ptemp );
return OPAL_SUCCESS;
}
int oshmem_proc_unpack(opal_buffer_t* buf,
int proclistsize,
oshmem_proc_t ***proclist,
int *newproclistsize,
oshmem_proc_t ***newproclist)
{
int i;
size_t newprocs_len = 0;
oshmem_proc_t **plist = NULL, **newprocs = NULL;
/* do not free plist *ever*, since it is used in the remote group
structure of a communicator */
plist = (oshmem_proc_t **) calloc(proclistsize, sizeof(oshmem_proc_t *));
if (NULL == plist) {
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
/* free this on the way out */
newprocs = (oshmem_proc_t **) calloc(proclistsize, sizeof(oshmem_proc_t *));
if (NULL == newprocs) {
free(plist);
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
/* cycle through the array of provided procs and unpack
* their info - as packed by oshmem_proc_pack
*/
for (i = 0; i < proclistsize; i++) {
orte_std_cntr_t count = 1;
orte_process_name_t new_name;
uint32_t new_arch;
char *new_hostname;
bool isnew = false;
int rc;
rc = opal_dss.unpack(buf, &new_name, &count, ORTE_NAME);
if (rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
rc = opal_dss.unpack(buf, &new_arch, &count, OPAL_UINT32);
if (rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING);
if (rc != ORTE_SUCCESS) {
ORTE_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
/* see if this proc is already on our oshmem_proc_list */
plist[i] = oshmem_proc_find_and_add(&new_name, &isnew);
if (isnew) {
/* if not, then it was added, so update the values
* in the proc_t struct with the info that was passed
* to us
*/
newprocs[newprocs_len++] = plist[i];
/* update all the values */
plist[i]->super.proc_arch = new_arch;
/* if arch is different than mine, create a new convertor for this proc */
if (plist[i]->super.proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(plist[i]->super.proc_convertor);
plist[i]->super.proc_convertor = opal_convertor_create(plist[i]->super.proc_arch, 0);
#else
orte_show_help("help-shmem-runtime.txt",
"heterogeneous-support-unavailable",
true,
orte_process_info.nodename,
new_hostname == NULL ? "<hostname unavailable>" :
new_hostname);
free(plist);
free(newprocs);
return OSHMEM_ERR_NOT_SUPPORTED;
#endif
}
if (0
== strcmp(oshmem_proc_local_proc->super.proc_hostname,
new_hostname)) {
plist[i]->super.proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU
| OPAL_PROC_ON_CLUSTER);
}
/* Save the hostname */
plist[i]->super.proc_hostname = new_hostname;
/* eventually, we will update the orte/mca/ess framework's data
* to contain the info for the new proc. For now, we ignore
* this step since the MPI layer already has all the info
* it requires
*/
}
}
if (NULL != newproclistsize)
*newproclistsize = newprocs_len;
if (NULL != newproclist) {
*newproclist = newprocs;
} else if (newprocs != NULL ) {
free(newprocs);
}
*proclist = plist;
return OSHMEM_SUCCESS;
}
static int
local_copy_with_convertor_2datatypes( ompi_datatype_t* send_type, int send_count,
ompi_datatype_t* recv_type, int recv_count,
int chunk )
{
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data;
int32_t length = 0, done1 = 0, done2 = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
size_t slength, rlength;
rlength = compute_buffer_length(recv_type, recv_count);
slength = compute_buffer_length(send_type, send_count);
pdst = malloc( rlength );
psrc = malloc( slength );
ptemp = malloc( chunk );
/* initialize the buffers to prevent valgrind from complaining */
for( int i = 0; i < slength; i++ )
((char*)psrc)[i] = i % 128 + 32;
memset(pdst, 0, rlength);
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, &(send_type->super), send_count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, &(recv_type->super), recv_count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying different data-types using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
clean_and_return:
if( send_convertor != NULL ) {
OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
}
if( recv_convertor != NULL ) {
OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
}
if( NULL != pdst ) free( pdst );
if( NULL != psrc ) free( psrc );
if( NULL != ptemp ) free( ptemp );
return OMPI_SUCCESS;
}
static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) {
int i, j, errors = 0;
struct iovec a;
unsigned int iov_count;
size_t max_data;
size_t pos;
opal_convertor_t * pConv;
for (j = 0; j < N; ++j) {
pbar[j].i[0] = 123+j;
pbar[j].i[1] = 789+j;
pbar[j].d[0] = 123.456+j;
pbar[j].d[1] = 789.123+j;
memset(&bar[j].i[0], 0xFF, sizeof(int));
memset(&bar[j].i[2], 0xFF, sizeof(int));
bar[j].i[1] = 0;
memset(&bar[j].d[0], 0xFF, sizeof(double));
memset(&bar[j].d[2], 0xFF, sizeof(double));
bar[j].d[1] = 0.0;
}
pConv = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(newtype->super), N, bar ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OMPI_ERROR;
}
for (i=0; arr[i][0] != 0; i++) {
/* add some garbage before and after the source data */
a.iov_base = malloc(arr[i][0]+2048);
if (NULL == a.iov_base) {
printf("cannot malloc iov_base\n");
return 1;
}
memset(a.iov_base, 0xAA, 1024);
memcpy((char*)a.iov_base+1024, (char *)pbar + arr[i][1], arr[i][0]);
memset((char*)a.iov_base+1024+arr[i][0], 0xAA, 1024);
a.iov_base = (char*)a.iov_base + 1024;
a.iov_len = arr[i][0];
iov_count = 1;
max_data = a.iov_len;
pos = arr[i][1];
opal_convertor_set_position(pConv, &pos);
assert(arr[i][1] == pos);
opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
a.iov_base = (char*)a.iov_base - 1024;
free(a.iov_base);
}
for (j = 0; j < N; ++j) {
if (bar[j].i[0] != pbar[j].i[0] ||
bar[j].i[1] != 0 ||
bar[j].i[2] != pbar[j].i[1] ||
bar[j].d[0] != pbar[j].d[0] ||
bar[j].d[1] != 0.0 ||
bar[j].d[2] != pbar[j].d[1]) {
if(0 == errors) {
fprintf(stderr, "ERROR ! count=%d, position=%d, ptr = %p"
" got (%d,%d,%d,%g,%g,%g) expected (%d,%d,%d,%g,%g,%g)\n",
N, j, (void*)&bar[j],
bar[j].i[0],
bar[j].i[1],
bar[j].i[2],
bar[j].d[0],
bar[j].d[1],
bar[j].d[2],
pbar[j].i[0],
0,
pbar[j].i[1],
pbar[j].d[0],
0.0,
pbar[j].d[1]);
print_bar_pbar(&bar[j], &pbar[j]);
}
errors++;
}
}
OBJ_RELEASE( pConv );
return errors;
}
int
ompio_io_ompio_file_open (ompi_communicator_t *comm,
const char *filename,
int amode,
ompi_info_t *info,
mca_io_ompio_file_t *ompio_fh, bool use_sharedfp)
{
int ret = OMPI_SUCCESS;
int remote_arch;
ompio_fh->f_iov_type = MPI_DATATYPE_NULL;
ompio_fh->f_comm = MPI_COMM_NULL;
if ( ((amode&MPI_MODE_RDONLY)?1:0) + ((amode&MPI_MODE_RDWR)?1:0) +
((amode&MPI_MODE_WRONLY)?1:0) != 1 ) {
return MPI_ERR_AMODE;
}
if ((amode & MPI_MODE_RDONLY) &&
((amode & MPI_MODE_CREATE) || (amode & MPI_MODE_EXCL))) {
return MPI_ERR_AMODE;
}
if ((amode & MPI_MODE_RDWR) && (amode & MPI_MODE_SEQUENTIAL)) {
return MPI_ERR_AMODE;
}
ompio_fh->f_rank = ompi_comm_rank (comm);
ompio_fh->f_size = ompi_comm_size (comm);
remote_arch = opal_local_arch;
ompio_fh->f_convertor = opal_convertor_create (remote_arch, 0);
if ( true == use_sharedfp ) {
ret = ompi_comm_dup (comm, &ompio_fh->f_comm);
if ( OMPI_SUCCESS != ret ) {
goto fn_fail;
}
}
else {
/* No need to duplicate the communicator if the file_open is called
from the sharedfp component, since the comm used as an input
is already a dup of the user level comm. */
ompio_fh->f_flags |= OMPIO_SHAREDFP_IS_SET;
ompio_fh->f_comm = comm;
}
ompio_fh->f_fstype = NONE;
ompio_fh->f_amode = amode;
ompio_fh->f_info = info;
ompio_fh->f_atomicity = 0;
ompi_io_ompio_set_file_defaults (ompio_fh);
ompio_fh->f_filename = filename;
ompio_fh->f_split_coll_req = NULL;
ompio_fh->f_split_coll_in_use = false;
/*Initialize the print_queues queues here!*/
coll_write_time = (mca_io_ompio_print_queue *) malloc (sizeof(mca_io_ompio_print_queue));
coll_read_time = (mca_io_ompio_print_queue *) malloc (sizeof(mca_io_ompio_print_queue));
ompi_io_ompio_initialize_print_queue(coll_write_time);
ompi_io_ompio_initialize_print_queue(coll_read_time);
/* set some function pointers required for fcoll, fbtls and sharedfp modules*/
ompio_fh->f_decode_datatype=ompi_io_ompio_decode_datatype;
ompio_fh->f_generate_current_file_view=ompi_io_ompio_generate_current_file_view;
ompio_fh->f_sort=ompi_io_ompio_sort;
ompio_fh->f_sort_iovec=ompi_io_ompio_sort_iovec;
ompio_fh->f_allgather_array=ompi_io_ompio_allgather_array;
ompio_fh->f_allgatherv_array=ompi_io_ompio_allgatherv_array;
ompio_fh->f_gather_array=ompi_io_ompio_gather_array;
ompio_fh->f_gatherv_array=ompi_io_ompio_gatherv_array;
ompio_fh->f_get_num_aggregators=mca_io_ompio_get_num_aggregators;
ompio_fh->f_get_bytes_per_agg=mca_io_ompio_get_bytes_per_agg;
ompio_fh->f_set_aggregator_props=ompi_io_ompio_set_aggregator_props;
ompio_fh->f_full_print_queue=ompi_io_ompio_full_print_queue;
ompio_fh->f_register_print_entry=ompi_io_ompio_register_print_entry;
/* This fix is needed for data seiving to work with
two-phase collective I/O */
if ((amode & MPI_MODE_WRONLY)){
amode -= MPI_MODE_WRONLY;
amode += MPI_MODE_RDWR;
}
/*--------------------------------------------------*/
if (OMPI_SUCCESS != (ret = mca_fs_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fs_base_file_select() failed\n");
goto fn_fail;
}
if (OMPI_SUCCESS != (ret = mca_fbtl_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fbtl_base_file_select() failed\n");
goto fn_fail;
}
if (OMPI_SUCCESS != (ret = mca_fcoll_base_file_select (ompio_fh,
NULL))) {
opal_output(1, "mca_fcoll_base_file_select() failed\n");
goto fn_fail;
}
ompio_fh->f_sharedfp_component = NULL; /*component*/
ompio_fh->f_sharedfp = NULL; /*module*/
ompio_fh->f_sharedfp_data = NULL; /*data*/
if ( true == use_sharedfp ) {
if (OMPI_SUCCESS != (ret = mca_sharedfp_base_file_select (ompio_fh, NULL))) {
opal_output ( ompi_io_base_framework.framework_output,
"mca_sharedfp_base_file_select() failed\n");
ompio_fh->f_sharedfp = NULL; /*module*/
/* Its ok to not have a shared file pointer module as long as the shared file
** pointer operations are not used. However, the first call to any file_read/write_shared
** function will return an error code.
*/
}
/* open the file once more for the shared file pointer if required.
** Per default, the shared file pointer specific actions are however
** only performed on first access of the shared file pointer, except
** for the addproc sharedfp component.
**
** Lazy open does not work for the addproc sharedfp
** component since it starts by spawning a process using MPI_Comm_spawn.
** For this, the first operation has to be collective which we can
** not guarantuee outside of the MPI_File_open operation.
*/
if ( NULL != ompio_fh->f_sharedfp &&
true == use_sharedfp &&
(!mca_io_ompio_sharedfp_lazy_open ||
!strcmp (ompio_fh->f_sharedfp_component->mca_component_name,
"addproc") )) {
ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm,
filename,
amode,
info,
ompio_fh);
if ( OMPI_SUCCESS != ret ) {
goto fn_fail;
}
}
}
/*Determine topology information if set*/
if (ompio_fh->f_comm->c_flags & OMPI_COMM_CART){
ret = mca_io_ompio_cart_based_grouping(ompio_fh);
if(OMPI_SUCCESS != ret ){
ret = MPI_ERR_FILE;
}
}
ret = ompio_fh->f_fs->fs_file_open (comm,
filename,
amode,
info,
ompio_fh);
if ( OMPI_SUCCESS != ret ) {
ret = MPI_ERR_FILE;
goto fn_fail;
}
/* If file has been opened in the append mode, move the internal
file pointer of OMPIO to the very end of the file. */
if ( ompio_fh->f_amode & MPI_MODE_APPEND ) {
OMPI_MPI_OFFSET_TYPE current_size;
ompio_fh->f_fs->fs_file_get_size( ompio_fh,
¤t_size);
ompi_io_ompio_set_explicit_offset (ompio_fh, current_size);
}
return OMPI_SUCCESS;
fn_fail:
/* no need to free resources here, since the destructor
* is calling mca_io_ompio_file_close, which actually gets
*rid of all allocated memory items */
return ret;
}
int ompi_proc_refresh(void) {
ompi_proc_t *proc = NULL;
opal_list_item_t *item = NULL;
ompi_vpid_t i = 0;
int ret=OMPI_SUCCESS;
OPAL_THREAD_LOCK(&ompi_proc_lock);
for( item = opal_list_get_first(&ompi_proc_list), i = 0;
item != opal_list_get_end(&ompi_proc_list);
item = opal_list_get_next(item), ++i ) {
proc = (ompi_proc_t*)item;
/* Does not change: proc->proc_name.vpid */
proc->proc_name.jobid = OMPI_PROC_MY_NAME->jobid;
/* Make sure to clear the local flag before we set it below */
proc->proc_flags = 0;
if (i == OMPI_PROC_MY_NAME->vpid) {
ompi_proc_local_proc = proc;
proc->proc_flags = OPAL_PROC_ALL_LOCAL;
proc->proc_hostname = ompi_process_info.nodename;
proc->proc_arch = opal_local_arch;
} else {
/* get the locality information */
ret = ompi_proc_set_locality(proc);
if (OMPI_SUCCESS != ret) {
break;
}
if (ompi_process_info.num_procs < ompi_hostname_cutoff) {
/* IF the number of procs falls below the specified cutoff,
* then we assume the job is small enough that retrieving
* the hostname (which will typically cause retrieval of
* ALL modex info for this proc) will have no appreciable
* impact on launch scaling
*/
ret = ompi_modex_recv_string_pointer(OMPI_DB_HOSTNAME, proc, (void**)&(proc->proc_hostname), OPAL_STRING);
if (OMPI_SUCCESS != ret) {
break;
}
} else {
/* just set the hostname to NULL for now - we'll fill it in
* as modex_recv's are called for procs we will talk to, thus
* avoiding retrieval of ALL modex info for this proc until
* required. Transports that delay calling modex_recv until
* first message will therefore scale better than those that
* call modex_recv on all procs during init.
*/
proc->proc_hostname = NULL;
}
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
{
/* get the remote architecture */
uint32_t* uiptr = &(proc->proc_arch);
ret = ompi_modex_recv_key_value("OMPI_ARCH", proc, (void**)&uiptr, OPAL_UINT32);
/* if arch is different than mine, create a new convertor for this proc */
if (proc->proc_arch != opal_local_arch) {
OBJ_RELEASE(proc->proc_convertor);
proc->proc_convertor = opal_convertor_create(proc->proc_arch, 0);
}
}
#else
/* must be same arch as my own */
proc->proc_arch = opal_local_arch;
#endif
}
}
OPAL_THREAD_UNLOCK(&ompi_proc_lock);
return ret;
}
static int test_upper( unsigned int length )
{
double *mat1, *mat2, *inbuf;
opal_datatype_t *pdt;
opal_convertor_t * pConv;
char *ptr;
int rc;
unsigned int i, j, iov_count, split_chunk, total_length;
size_t max_data;
struct iovec a;
TIMER_DATA_TYPE start, end;
long total_time;
printf( "test upper matrix\n" );
pdt = upper_matrix( length );
opal_datatype_dump( pdt );
mat1 = malloc( length * length * sizeof(double) );
init_random_upper_matrix( length, mat1 );
mat2 = calloc( length * length, sizeof(double) );
total_length = length * (length + 1) * ( sizeof(double) / 2);
inbuf = (double*)malloc( total_length );
ptr = (char*)inbuf;
/* copy upper matrix in the array simulating the input buffer */
for( i = 0; i < length; i++ ) {
uint32_t pos = i * length + i;
for( j = i; j < length; j++, pos++ ) {
*inbuf = mat1[pos];
inbuf++;
}
}
inbuf = (double*)ptr;
pConv = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, pdt, 1, mat2 ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OPAL_ERROR;
}
GET_TIME( start );
split_chunk = (length + 1) * sizeof(double);
/* split_chunk = (total_length + 1) * sizeof(double); */
for( i = total_length; i > 0; ) {
if( i <= split_chunk ) { /* equal test just to be able to set a breakpoint */
split_chunk = i;
}
a.iov_base = ptr;
a.iov_len = split_chunk;
iov_count = 1;
max_data = split_chunk;
opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
ptr += max_data;
i -= max_data;
if( mat2[0] != inbuf[0] ) assert(0);
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "complete unpacking in %ld microsec\n", total_time );
free( inbuf );
rc = check_diag_matrix( length, mat1, mat2 );
free( mat1 );
free( mat2 );
/* test the automatic destruction pf the data */
opal_datatype_destroy( &pdt );
assert( pdt == NULL );
OBJ_RELEASE( pConv );
return rc;
}
int
ompi_proc_unpack(opal_buffer_t* buf,
int proclistsize, ompi_proc_t ***proclist,
bool full_info,
int *newproclistsize, ompi_proc_t ***newproclist)
{
int i;
size_t newprocs_len = 0;
ompi_proc_t **plist=NULL, **newprocs = NULL;
/* do not free plist *ever*, since it is used in the remote group
structure of a communicator */
plist = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *));
if ( NULL == plist ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* free this on the way out */
newprocs = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *));
if (NULL == newprocs) {
free(plist);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* cycle through the array of provided procs and unpack
* their info - as packed by ompi_proc_pack
*/
for ( i=0; i<proclistsize; i++ ){
int32_t count=1;
ompi_process_name_t new_name;
uint32_t new_arch;
char *new_hostname;
bool isnew = false;
int rc;
rc = opal_dss.unpack(buf, &new_name, &count, OMPI_NAME);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
if (!full_info) {
rc = opal_dss.unpack(buf, &new_arch, &count, OPAL_UINT32);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
}
/* see if this proc is already on our ompi_proc_list */
plist[i] = ompi_proc_find_and_add(&new_name, &isnew);
if (isnew) {
/* if not, then it was added, so update the values
* in the proc_t struct with the info that was passed
* to us
*/
newprocs[newprocs_len++] = plist[i];
if (full_info) {
int32_t num_recvd_entries;
int32_t cnt;
int32_t j;
/* unpack the number of entries for this proc */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) {
OMPI_ERROR_LOG(rc);
break;
}
/*
* Extract the attribute names and values
*/
for (j = 0; j < num_recvd_entries; j++) {
opal_value_t *kv;
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) {
OMPI_ERROR_LOG(rc);
break;
}
/* if this is me, dump the data - we already have it in the db */
if (OPAL_EQUAL == ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
OMPI_PROC_MY_NAME, &new_name)) {
OBJ_RELEASE(kv);
} else {
/* store it in the database */
if (OPAL_SUCCESS != (rc = opal_db.store_pointer((opal_identifier_t*)&new_name, kv))) {
OMPI_ERROR_LOG(rc);
OBJ_RELEASE(kv);
}
/* do not release the kv - the db holds that pointer */
}
}
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
rc = opal_db.fetch((opal_identifier_t*)&new_name, "OMPI_ARCH",
(void**)&new_arch, OPAL_UINT32);
if( OPAL_SUCCESS == rc ) {
new_arch = opal_local_arch;
}
#else
new_arch = opal_local_arch;
#endif
if (ompi_process_info.num_procs < ompi_hostname_cutoff) {
/* retrieve the hostname */
rc = opal_db.fetch_pointer((opal_identifier_t*)&new_name, OMPI_DB_HOSTNAME,
(void**)&new_hostname, OPAL_STRING);
if( OPAL_SUCCESS != rc ) {
new_hostname = NULL;
}
} else {
/* just set the hostname to NULL for now - we'll fill it in
* as modex_recv's are called for procs we will talk to
*/
new_hostname = NULL;
}
}
/* update all the values */
plist[i]->proc_arch = new_arch;
/* if arch is different than mine, create a new convertor for this proc */
if (plist[i]->proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(plist[i]->proc_convertor);
plist[i]->proc_convertor = opal_convertor_create(plist[i]->proc_arch, 0);
#else
opal_show_help("help-mpi-runtime",
"heterogeneous-support-unavailable",
true, ompi_process_info.nodename,
new_hostname == NULL ? "<hostname unavailable>" :
new_hostname);
free(plist);
free(newprocs);
return OMPI_ERR_NOT_SUPPORTED;
#endif
}
if (0 == strcmp(ompi_proc_local_proc->proc_hostname, new_hostname)) {
plist[i]->proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
/* Save the hostname */
plist[i]->proc_hostname = new_hostname;
} else {
if (full_info) {
int32_t num_recvd_entries;
int32_t j, cnt;
/* discard all keys: they are already locally known */
cnt = 1;
if (OPAL_SUCCESS == (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) {
for (j = 0; j < num_recvd_entries; j++) {
opal_value_t *kv;
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) {
OMPI_ERROR_LOG(rc);
continue;
}
OBJ_RELEASE(kv);
}
} else {
OMPI_ERROR_LOG(rc);
}
}
}
}
if (NULL != newproclistsize) *newproclistsize = newprocs_len;
if (NULL != newproclist) {
*newproclist = newprocs;
} else if (newprocs != NULL) {
free(newprocs);
}
*proclist = plist;
return OMPI_SUCCESS;
}
