/** * Main function. Call several tests and print-out the results. It try to stress the convertor * using difficult data-type constructions as well as strange segment sizes for the conversion. * Usually, it is able to detect most of the data-type and convertor problems. Any modifications * on the data-type engine should first pass all the tests from this file, before going into other * tests. */ int main( int argc, char* argv[] ) { opal_datatype_init(); /** * By default simulate homogeneous architectures. */ remote_arch = opal_local_arch ^ OPAL_ARCH_ISBIGENDIAN; opal_convertor_t * pConv; int sbuf[2], rbuf[2]; size_t max_data; struct iovec a; uint32_t iov_count; sbuf[0] = 0x01000000; sbuf[1] = 0x02000000; printf( "\n\n#\n * TEST UNPACKING 1 int out of 1\n#\n\n" ); pConv = opal_convertor_create( remote_arch, 0 ); rbuf[0] = -1; rbuf[1] = -1; if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 1, rbuf ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OPAL_ERROR; } a.iov_base = sbuf; a.iov_len = 4; iov_count = 1; max_data = 4; opal_unpack_general( pConv, &a, &iov_count, &max_data ); assert(1 == rbuf[0]); assert(-1 == rbuf[1]); OBJ_RELEASE(pConv); printf( "\n\n#\n * TEST UNPACKING 1 int out of 2\n#\n\n" ); pConv = opal_convertor_create( remote_arch, 0 ); rbuf[0] = -1; rbuf[1] = -1; if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 2, rbuf ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OPAL_ERROR; } a.iov_base = sbuf; a.iov_len = 4; iov_count = 1; max_data = 4; opal_unpack_general( pConv, &a, &iov_count, &max_data ); assert(1 == rbuf[0]); assert(-1 == rbuf[1]); OBJ_RELEASE(pConv); /* clean-ups all data allocations */ opal_datatype_finalize(); opal_finalize(); return OPAL_SUCCESS; }
int32_t ompi_datatype_default_convertors_init( void ) { /* create the extern32 convertor */ ompi_mpi_external32_convertor = opal_convertor_create( ompi_datatype_external32_arch_id, 0 ); /* create the local convertor */ ompi_mpi_local_convertor = opal_convertor_create( opal_local_arch, 0 ); return OMPI_SUCCESS; }
int oshmem_proc_init(void) { orte_vpid_t i; OBJ_CONSTRUCT(&oshmem_proc_list, opal_list_t); OBJ_CONSTRUCT(&oshmem_proc_lock, opal_mutex_t); oshmem_shmem_local_convertor = opal_convertor_create(opal_local_arch, 0); size_t ompi_num_procs; ompi_proc_t **ompi_procs = ompi_proc_world(&ompi_num_procs); /* create proc structures and find self */ for (i = 0; i < orte_process_info.num_procs; i++) { oshmem_proc_t *proc = OBJ_NEW(oshmem_proc_t); opal_list_append(&oshmem_proc_list, (opal_list_item_t*)proc); proc->super.proc_name = ompi_procs[i]->super.proc_name; proc->super.proc_arch = ompi_procs[i]->super.proc_arch; proc->super.proc_flags = ompi_procs[i]->super.proc_flags; proc->super.proc_hostname = ompi_procs[i]->super.proc_hostname; if (i == ORTE_PROC_MY_NAME->vpid) { oshmem_proc_local_proc = proc; } } if (ompi_procs) free(ompi_procs); return OSHMEM_SUCCESS; }
static int create_segments( ompi_datatype_t* datatype, int count, size_t segment_length, ddt_segment_t** segments, int* seg_count ) { size_t data_size, total_length, position; opal_convertor_t* convertor; int i; ddt_segment_t* segment; ompi_datatype_type_size( datatype, &data_size ); data_size *= count; *seg_count = data_size / segment_length; if( ((*seg_count) * segment_length) != data_size ) *seg_count += 1; allocate_segments: *segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) ); convertor = opal_convertor_create( opal_local_arch, 0 ); opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL ); position = 0; total_length = 0; for( i = 0; i < (*seg_count); i++ ) { segment = &((*segments)[i]); segment->buffer = malloc(segment_length); segment->position = position; /* Find the end of the segment */ position += segment_length; opal_convertor_set_position( convertor, &position ); segment->size = position - segment->position; total_length += segment->size; } OBJ_RELEASE(convertor); if( total_length != data_size ) { for( i = 0; i < (*seg_count); i++ ) { segment = &((*segments)[i]); free(segment->buffer); } free( *segments ); (*seg_count) += 1; goto allocate_segments; } return 0; }
static int test_upper( unsigned int length ) { ompi_datatype_t *pdt; opal_convertor_t * pConv; int rc = OMPI_SUCCESS; unsigned int i, iov_count, split_chunk, total_length; size_t max_data; struct iovec iov[5]; TIMER_DATA_TYPE start, end; long total_time; printf( "test upper matrix\n" ); pdt = upper_matrix( length ); /*dt_dump( pdt );*/ total_length = length * (length + 1) * ( sizeof(double) / 2); pConv = opal_convertor_create( remote_arch, 0 ); if( OMPI_SUCCESS != opal_convertor_prepare_for_send( pConv, &(pdt->super), 1, NULL ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OMPI_ERROR; } GET_TIME( start ); split_chunk = (length + 1) * sizeof(double); /* split_chunk = (total_length + 1) * sizeof(double); */ for( i = total_length; i > 0; ) { iov_count = 5; max_data = 0; opal_convertor_raw( pConv, iov, &iov_count, &max_data ); i -= max_data; } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "complete raw in %ld microsec\n", total_time ); /* test the automatic destruction pf the data */ ompi_datatype_destroy( &pdt ); assert( pdt == NULL ); OBJ_RELEASE( pConv ); return rc; }
/** * Conversion function. They deal with data-types in 3 ways, always making local copies. * In order to allow performance testings, there are 3 functions: * - one copying directly from one memory location to another one using the * data-type copy function. * - one which use a 2 convertors created with the same data-type * - and one using 2 convertors created from different data-types. * */ static int local_copy_ddt_raw( ompi_datatype_t* pdt, int count, int iov_num ) { struct iovec* iov; opal_convertor_t* convertor; TIMER_DATA_TYPE start, end; long total_time; uint32_t iov_count = iov_num; size_t max_data = 0, remaining_length; iov = (struct iovec*)malloc(iov_num * sizeof(struct iovec)); convertor = opal_convertor_create( remote_arch, 0 ); if( OMPI_SUCCESS != opal_convertor_prepare_for_send( convertor, &(pdt->super), count, NULL ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OMPI_ERROR; } remaining_length = count * pdt->super.size; GET_TIME( start ); while( 0 == opal_convertor_raw(convertor, iov, &iov_count, &max_data) ) { #if 0 printf( "New raw extraction (iov_count = %d, max_data = %zu)\n", iov_count, max_data ); for( i = 0; i < iov_count; i++ ) { printf( "\t{%p, %d}\n", iov[i].iov_base, iov[i].iov_len ); } #endif remaining_length -= max_data; iov_count = iov_num; } remaining_length -= max_data; GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "raw extraction in %ld microsec\n", total_time ); OBJ_RELEASE( convertor ); if( remaining_length != 0 ) { printf( "Not all raw description was been extracted (%lu bytes missing)\n", (unsigned long) remaining_length ); } free(iov); return OMPI_SUCCESS; }
/* in some cases, all PE procs are required to do a modex so they * can (at the least) exchange their architecture. Since we cannot * know in advance if this was required, we provide a separate function * to set the arch (instead of doing it inside of oshmem_proc_init) that * can be called after the modex completes in oshmem_shmem_init. Thus, we * know that - regardless of how the arch is known, whether via modex * or dropped in from a local daemon - the arch can be set correctly * at this time */ int oshmem_proc_set_arch(void) { oshmem_proc_t *proc = NULL; opal_list_item_t *item = NULL; int ret = OSHMEM_SUCCESS; OPAL_THREAD_LOCK(&oshmem_proc_lock); for (item = opal_list_get_first(&oshmem_proc_list); item != opal_list_get_end(&oshmem_proc_list); item = opal_list_get_next(item)) { proc = (oshmem_proc_t*) item; if (OSHMEM_PROC_VPID(proc) != ORTE_PROC_MY_NAME->vpid) { /* if arch is different than mine, create a new convertor for this proc */ if (proc->super.proc_arch != opal_local_arch) { #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT OBJ_RELEASE(proc->super.proc_convertor); proc->super.proc_convertor = opal_convertor_create(proc->super.proc_arch, 0); #else orte_show_help("help-shmem-runtime.txt", "heterogeneous-support-unavailable", true, orte_process_info.nodename, proc->super.proc_hostname == NULL ? "<hostname unavailable>" : proc->super.proc_hostname); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return OSHMEM_ERR_NOT_SUPPORTED; #endif } } } /* Set predefined groups */ ret = oshmem_proc_group_init(); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return ret; }
static int unpack_segments( ompi_datatype_t* datatype, int count, size_t segment_size, ddt_segment_t* segments, int seg_count, void* buffer ) { opal_convertor_t* convertor; size_t max_size, position; int i; uint32_t iov_count; struct iovec iov; convertor = opal_convertor_create( opal_local_arch, 0 ); opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer ); for( i = 0; i < seg_count; i++ ) { iov.iov_len = segments[i].size; iov.iov_base = segments[i].buffer; max_size = iov.iov_len; position = segments[i].position; opal_convertor_set_position( convertor, &position ); if( position != segments[i].position ) { opal_output( 0, "Setting position failed (%lu != %lu)\n", (unsigned long)segments[i].position, (unsigned long)position ); break; } iov_count = 1; opal_convertor_unpack( convertor, &iov, &iov_count, &max_size ); if( max_size != segments[i].size ) { opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n", (unsigned long)max_size, (unsigned long)segments[i].size ); opal_output( 0, "Segment %d position %lu size %lu\n", i, (unsigned long)segments[i].position, segments[i].size ); } } OBJ_RELEASE(convertor); return 0; }
int ompio_io_ompio_file_open (ompi_communicator_t *comm, char *filename, int amode, ompi_info_t *info, mca_io_ompio_file_t *ompio_fh, bool use_sharedfp) { int ret = OMPI_SUCCESS; int remote_arch; if ( ((amode&MPI_MODE_RDONLY)?1:0) + ((amode&MPI_MODE_RDWR)?1:0) + ((amode&MPI_MODE_WRONLY)?1:0) != 1 ) { return MPI_ERR_AMODE; } if ((amode & MPI_MODE_RDONLY) && ((amode & MPI_MODE_CREATE) || (amode & MPI_MODE_EXCL))) { return MPI_ERR_AMODE; } if ((amode & MPI_MODE_RDWR) && (amode & MPI_MODE_SEQUENTIAL)) { return MPI_ERR_AMODE; } ompio_fh->f_iov_type = MPI_DATATYPE_NULL; ompio_fh->f_rank = ompi_comm_rank (comm); ompio_fh->f_size = ompi_comm_size (comm); remote_arch = opal_local_arch; ompio_fh->f_convertor = opal_convertor_create (remote_arch, 0); ret = ompi_comm_dup (comm, &ompio_fh->f_comm); if ( ret != OMPI_SUCCESS ) { goto fn_fail; } ompio_fh->f_fstype = NONE; ompio_fh->f_amode = amode; ompio_fh->f_info = info; ompio_fh->f_atomicity = 0; ompi_io_ompio_set_file_defaults (ompio_fh); ompio_fh->f_filename = filename; /*Initialize the print_queues queues here!*/ coll_write_time = (print_queue *) malloc (sizeof(print_queue)); coll_read_time = (print_queue *) malloc (sizeof(print_queue)); ompi_io_ompio_initialize_print_queue(coll_write_time); ompi_io_ompio_initialize_print_queue(coll_read_time); /* if (MPI_INFO_NULL != info) { ret = ompi_info_dup (info, &ompio_fh->f_info); if (OMPI_SUCCESS != ret) { goto fn_fail; } } */ /* This fix is needed for data seiving to work with two-phase collective I/O */ if ((amode & MPI_MODE_WRONLY)) { amode -= MPI_MODE_WRONLY; amode += MPI_MODE_RDWR; } /*--------------------------------------------------*/ if (OMPI_SUCCESS != (ret = mca_fs_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fs_base_file_select() failed\n"); goto fn_fail; } if (OMPI_SUCCESS != (ret = mca_fbtl_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fbtl_base_file_select() failed\n"); goto fn_fail; } if (OMPI_SUCCESS != (ret = mca_fcoll_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fcoll_base_file_select() failed\n"); goto fn_fail; } ompio_fh->f_sharedfp_component = NULL; /*component*/ ompio_fh->f_sharedfp = NULL; /*module*/ ompio_fh->f_sharedfp_data = NULL; /*data*/ if (OMPI_SUCCESS != (ret = mca_sharedfp_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_sharedfp_base_file_select() failed\n"); goto fn_fail; } /*Determine topology information if set*/ if (ompio_fh->f_comm->c_flags & OMPI_COMM_CART) { ret = mca_io_ompio_cart_based_grouping(ompio_fh); if(OMPI_SUCCESS != ret ) { ret = MPI_ERR_FILE; } } ret = ompio_fh->f_fs->fs_file_open (comm, filename, amode, info, ompio_fh); if ( OMPI_SUCCESS != ret ) { ret = MPI_ERR_FILE; goto fn_fail; } /* open the file once more for the shared file pointer if required. ** Per default, the shared file pointer specific actions are however ** only performed on first access of the shared file pointer, except ** for the addproc sharedfp component. ** ** Lazy open does not work for the addproc sharedfp ** component since it starts by spawning a process using MPI_Comm_spawn. ** For this, the first operation has to be collective which we can ** not guarantuee outside of the MPI_File_open operation. */ if ( true == use_sharedfp && (!mca_io_ompio_sharedfp_lazy_open || !strcmp (ompio_fh->f_sharedfp_component->mca_component_name, "addproc") )) { ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm, filename, amode, info, ompio_fh); if ( OMPI_SUCCESS != ret ) { goto fn_fail; } } /* If file has been opened in the append mode, move the internal file pointer of OMPIO to the very end of the file. */ if ( ompio_fh->f_amode & MPI_MODE_APPEND ) { OMPI_MPI_OFFSET_TYPE current_size; ompio_fh->f_fs->fs_file_get_size( ompio_fh, ¤t_size); ompi_io_ompio_set_explicit_offset (ompio_fh, current_size); } return OMPI_SUCCESS; fn_fail: /* no need to free resources here, since the destructor * is calling mca_io_ompio_file_close, which actually gets *rid of all allocated memory items */ return ret; }
static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk ) { OPAL_PTRDIFF_TYPE lb, extent; void *pdst = NULL, *psrc = NULL, *ptemp = NULL; char *odst, *osrc; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data, length = 0, malloced_size; int32_t done1 = 0, done2 = 0, errors = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; malloced_size = compute_memory_size(pdt, count); opal_datatype_get_extent( pdt, &lb, &extent ); odst = (char*)malloc( malloced_size ); osrc = (char*)malloc( malloced_size ); ptemp = malloc( chunk ); { for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ ); memcpy(odst, osrc, malloced_size); } pdst = odst - lb; psrc = osrc - lb; send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; if( outputFlags & RESET_CONVERTORS ) { struct dt_stack_t stack[1+send_convertor->stack_pos]; int i, stack_pos = send_convertor->stack_pos; size_t pos; if( 0 == done1 ) { memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t)); pos = 0; opal_convertor_set_position(send_convertor, &pos); pos = length; opal_convertor_set_position(send_convertor, &pos); assert(pos == length); for(i = 0; i <= stack_pos; i++ ) { if( stack[i].index != send_convertor->pStack[i].index ) {errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n", i, stack[i].index, send_convertor->pStack[i].index, length, pdt->size * count);} if( stack[i].count != send_convertor->pStack[i].count ) { if( stack[i].type == send_convertor->pStack[i].type ) { {errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n", i, stack[i].count, send_convertor->pStack[i].count, length, pdt->size * count);} } else { if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) ) {errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n", i, (int)stack[i].type, (int)send_convertor->pStack[i].type, length, pdt->size * count);} else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) != (send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) ) {errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n", i, (int)stack[i].type, stack[i].count, (int)send_convertor->pStack[i].type, send_convertor->pStack[i].count, length, pdt->size * count);} } } if( stack[i].disp != send_convertor->pStack[i].disp ) {errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n", i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp, length, pdt->size * count);} if(0 != errors) {assert(0); exit(-1);} } } if( 0 == done2 ) { memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t)); pos = 0; opal_convertor_set_position(recv_convertor, &pos); pos = length; opal_convertor_set_position(recv_convertor, &pos); assert(pos == length); for(i = 0; i <= stack_pos; i++ ) { if( stack[i].index != recv_convertor->pStack[i].index ) {errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n", i, stack[i].index, recv_convertor->pStack[i].index, length, pdt->size * count);} if( stack[i].count != recv_convertor->pStack[i].count ) { if( stack[i].type == recv_convertor->pStack[i].type ) { {errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n", i, stack[i].count, recv_convertor->pStack[i].count, length, pdt->size * count);} } else { if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) ) {errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n", i, (int)stack[i].type, (int)recv_convertor->pStack[i].type, length, pdt->size * count);} else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) != (recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) ) {errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n", i, (int)stack[i].type, stack[i].count, (int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count, length, pdt->size * count);} } } if( stack[i].disp != recv_convertor->pStack[i].disp ) {errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n", i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp, length, pdt->size * count);} if(0 != errors) {assert(0); exit(-1);} } } } } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying same data-type using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); if(outputFlags & VALIDATE_DATA) { for( size_t i = errors = 0; i < malloced_size; i++ ) { if( odst[i] != osrc[i] ) { printf("error at position %lu (%d != %d)\n", (unsigned long)i, (int)(odst[i]), (int)(osrc[i])); errors++; if(outputFlags & QUIT_ON_FIRST_ERROR) { opal_datatype_dump(pdt); assert(0); exit(-1); } } } if( 0 == errors ) { printf("Validation check succesfully passed\n"); } else { printf("Found %d errors. Giving up!\n", errors); exit(-1); } } clean_and_return: if( NULL != send_convertor ) OBJ_RELEASE( send_convertor ); if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor ); if( NULL != odst ) free( odst ); if( NULL != osrc ) free( osrc ); if( NULL != ptemp ) free( ptemp ); return (0 == errors ? OPAL_SUCCESS : errors); }
static int local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count, opal_datatype_t const * const recv_type, int recv_count, int chunk ) { OPAL_PTRDIFF_TYPE send_lb, send_extent, recv_lb, recv_extent; void *pdst = NULL, *psrc = NULL, *ptemp = NULL; char *odst, *osrc; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data, length = 0, send_malloced_size, recv_malloced_size;; int32_t done1 = 0, done2 = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; send_malloced_size = compute_memory_size(send_type, send_count); recv_malloced_size = compute_memory_size(recv_type, recv_count); opal_datatype_get_extent( send_type, &send_lb, &send_extent ); opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent ); odst = (char*)malloc( recv_malloced_size ); osrc = (char*)malloc( send_malloced_size ); ptemp = malloc( chunk ); /* fill up the receiver with ZEROS */ { for( size_t i = 0; i < send_malloced_size; i++ ) osrc[i] = i % 128 + 32; } memset( odst, 0, recv_malloced_size ); pdst = odst - recv_lb; psrc = osrc - send_lb; send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; if( outputFlags & RESET_CONVERTORS ) { size_t pos = 0; opal_convertor_set_position(send_convertor, &pos); pos = length; opal_convertor_set_position(send_convertor, &pos); assert(pos == length); pos = 0; opal_convertor_set_position(recv_convertor, &pos); pos = length; opal_convertor_set_position(recv_convertor, &pos); assert(pos == length); } } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying different data-types using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); clean_and_return: if( send_convertor != NULL ) { OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL ); } if( recv_convertor != NULL ) { OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL ); } if( NULL != odst ) free( odst ); if( NULL != osrc ) free( osrc ); if( NULL != ptemp ) free( ptemp ); return OPAL_SUCCESS; }
int oshmem_proc_refresh(void) { oshmem_proc_t *proc = NULL; opal_list_item_t *item = NULL; orte_vpid_t i = 0; int hostname_length = 0; OPAL_THREAD_LOCK(&oshmem_proc_lock); for (item = opal_list_get_first(&oshmem_proc_list), i = 0; item != opal_list_get_end(&oshmem_proc_list); item = opal_list_get_next(item), ++i) { proc = (oshmem_proc_t*) item; /* Does not change: proc->proc_name.vpid */ proc->proc_name.jobid = ORTE_PROC_MY_NAME->jobid; /* Make sure to clear the local flag before we set it below */ proc->proc_flags = 0; proc->proc_arch = opal_local_arch; oshmem_shmem_exchange_bcast(&proc->proc_arch, sizeof(uint32_t), i); hostname_length = strlen(orte_process_info.nodename); oshmem_shmem_exchange_bcast(&hostname_length, sizeof(int), i); if (proc->proc_hostname) free(proc->proc_hostname); proc->proc_hostname = ( i == ORTE_PROC_MY_NAME->vpid ? strdup(orte_process_info.nodename) : (char *) malloc(hostname_length)); oshmem_shmem_exchange_bcast(proc->proc_hostname, hostname_length, i); if (i == ORTE_PROC_MY_NAME->vpid) { oshmem_proc_local_proc = proc; } else { /* if arch is different than mine, create a new convertor for this proc */ if (proc->proc_arch != opal_local_arch) { #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT OBJ_RELEASE(proc->proc_convertor); proc->proc_convertor = opal_convertor_create(proc->proc_arch, 0); #else orte_show_help("help-shmem-runtime.txt", "heterogeneous-support-unavailable", true, orte_process_info.nodename, proc->proc_hostname == NULL ? "<hostname unavailable>" : proc->proc_hostname); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return OSHMEM_ERR_NOT_SUPPORTED; #endif } } } OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return OSHMEM_SUCCESS; }
static int local_copy_with_convertor( const opal_datatype_t const* pdt, int count, int chunk ) { OPAL_PTRDIFF_TYPE extent; void *pdst = NULL, *psrc = NULL, *ptemp = NULL; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data; int32_t length = 0, done1 = 0, done2 = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; opal_datatype_type_extent( pdt, &extent ); pdst = malloc( extent * count ); psrc = malloc( extent * count ); ptemp = malloc( chunk ); { int i = 0; for( ; i < (count * extent); ((char*)psrc)[i] = i % 128 + 32, i++ ); } memset( pdst, 0, count * extent ); send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying same data-type using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); clean_and_return: if( NULL != send_convertor ) OBJ_RELEASE( send_convertor ); if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor ); if( NULL != pdst ) free( pdst ); if( NULL != psrc ) free( psrc ); if( NULL != ptemp ) free( ptemp ); return OPAL_SUCCESS; }
int oshmem_proc_unpack(opal_buffer_t* buf, int proclistsize, oshmem_proc_t ***proclist, int *newproclistsize, oshmem_proc_t ***newproclist) { int i; size_t newprocs_len = 0; oshmem_proc_t **plist = NULL, **newprocs = NULL; /* do not free plist *ever*, since it is used in the remote group structure of a communicator */ plist = (oshmem_proc_t **) calloc(proclistsize, sizeof(oshmem_proc_t *)); if (NULL == plist) { return OSHMEM_ERR_OUT_OF_RESOURCE; } /* free this on the way out */ newprocs = (oshmem_proc_t **) calloc(proclistsize, sizeof(oshmem_proc_t *)); if (NULL == newprocs) { free(plist); return OSHMEM_ERR_OUT_OF_RESOURCE; } /* cycle through the array of provided procs and unpack * their info - as packed by oshmem_proc_pack */ for (i = 0; i < proclistsize; i++) { orte_std_cntr_t count = 1; orte_process_name_t new_name; uint32_t new_arch; char *new_hostname; bool isnew = false; int rc; rc = opal_dss.unpack(buf, &new_name, &count, ORTE_NAME); if (rc != ORTE_SUCCESS) { ORTE_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } rc = opal_dss.unpack(buf, &new_arch, &count, OPAL_UINT32); if (rc != ORTE_SUCCESS) { ORTE_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING); if (rc != ORTE_SUCCESS) { ORTE_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } /* see if this proc is already on our oshmem_proc_list */ plist[i] = oshmem_proc_find_and_add(&new_name, &isnew); if (isnew) { /* if not, then it was added, so update the values * in the proc_t struct with the info that was passed * to us */ newprocs[newprocs_len++] = plist[i]; /* update all the values */ plist[i]->super.proc_arch = new_arch; /* if arch is different than mine, create a new convertor for this proc */ if (plist[i]->super.proc_arch != opal_local_arch) { #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT OBJ_RELEASE(plist[i]->super.proc_convertor); plist[i]->super.proc_convertor = opal_convertor_create(plist[i]->super.proc_arch, 0); #else orte_show_help("help-shmem-runtime.txt", "heterogeneous-support-unavailable", true, orte_process_info.nodename, new_hostname == NULL ? "<hostname unavailable>" : new_hostname); free(plist); free(newprocs); return OSHMEM_ERR_NOT_SUPPORTED; #endif } if (0 == strcmp(oshmem_proc_local_proc->super.proc_hostname, new_hostname)) { plist[i]->super.proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER); } /* Save the hostname */ plist[i]->super.proc_hostname = new_hostname; /* eventually, we will update the orte/mca/ess framework's data * to contain the info for the new proc. For now, we ignore * this step since the MPI layer already has all the info * it requires */ } } if (NULL != newproclistsize) *newproclistsize = newprocs_len; if (NULL != newproclist) { *newproclist = newprocs; } else if (newprocs != NULL ) { free(newprocs); } *proclist = plist; return OSHMEM_SUCCESS; }
static int local_copy_with_convertor_2datatypes( ompi_datatype_t* send_type, int send_count, ompi_datatype_t* recv_type, int recv_count, int chunk ) { void *pdst = NULL, *psrc = NULL, *ptemp = NULL; opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL; struct iovec iov; uint32_t iov_count; size_t max_data; int32_t length = 0, done1 = 0, done2 = 0; TIMER_DATA_TYPE start, end, unpack_start, unpack_end; long total_time, unpack_time = 0; size_t slength, rlength; rlength = compute_buffer_length(recv_type, recv_count); slength = compute_buffer_length(send_type, send_count); pdst = malloc( rlength ); psrc = malloc( slength ); ptemp = malloc( chunk ); /* initialize the buffers to prevent valgrind from complaining */ for( int i = 0; i < slength; i++ ) ((char*)psrc)[i] = i % 128 + 32; memset(pdst, 0, rlength); send_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, &(send_type->super), send_count, psrc ) ) { printf( "Unable to create the send convertor. Is the datatype committed ?\n" ); goto clean_and_return; } recv_convertor = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, &(recv_type->super), recv_count, pdst ) ) { printf( "Unable to create the recv convertor. Is the datatype committed ?\n" ); goto clean_and_return; } cache_trash(); /* make sure the cache is useless */ GET_TIME( start ); while( (done1 & done2) != 1 ) { /* They are supposed to finish in exactly the same time. */ if( done1 | done2 ) { printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n", (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish") ); } max_data = chunk; iov_count = 1; iov.iov_base = ptemp; iov.iov_len = chunk; if( done1 == 0 ) { done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data ); } if( done2 == 0 ) { GET_TIME( unpack_start ); done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data ); GET_TIME( unpack_end ); unpack_time += ELAPSED_TIME( unpack_start, unpack_end ); } length += max_data; } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "copying different data-types using convertors in %ld microsec\n", total_time ); printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time, total_time - unpack_time ); clean_and_return: if( send_convertor != NULL ) { OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL ); } if( recv_convertor != NULL ) { OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL ); } if( NULL != pdst ) free( pdst ); if( NULL != psrc ) free( psrc ); if( NULL != ptemp ) free( ptemp ); return OMPI_SUCCESS; }
static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) { int i, j, errors = 0; struct iovec a; unsigned int iov_count; size_t max_data; size_t pos; opal_convertor_t * pConv; for (j = 0; j < N; ++j) { pbar[j].i[0] = 123+j; pbar[j].i[1] = 789+j; pbar[j].d[0] = 123.456+j; pbar[j].d[1] = 789.123+j; memset(&bar[j].i[0], 0xFF, sizeof(int)); memset(&bar[j].i[2], 0xFF, sizeof(int)); bar[j].i[1] = 0; memset(&bar[j].d[0], 0xFF, sizeof(double)); memset(&bar[j].d[2], 0xFF, sizeof(double)); bar[j].d[1] = 0.0; } pConv = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(newtype->super), N, bar ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OMPI_ERROR; } for (i=0; arr[i][0] != 0; i++) { /* add some garbage before and after the source data */ a.iov_base = malloc(arr[i][0]+2048); if (NULL == a.iov_base) { printf("cannot malloc iov_base\n"); return 1; } memset(a.iov_base, 0xAA, 1024); memcpy((char*)a.iov_base+1024, (char *)pbar + arr[i][1], arr[i][0]); memset((char*)a.iov_base+1024+arr[i][0], 0xAA, 1024); a.iov_base = (char*)a.iov_base + 1024; a.iov_len = arr[i][0]; iov_count = 1; max_data = a.iov_len; pos = arr[i][1]; opal_convertor_set_position(pConv, &pos); assert(arr[i][1] == pos); opal_convertor_unpack( pConv, &a, &iov_count, &max_data ); a.iov_base = (char*)a.iov_base - 1024; free(a.iov_base); } for (j = 0; j < N; ++j) { if (bar[j].i[0] != pbar[j].i[0] || bar[j].i[1] != 0 || bar[j].i[2] != pbar[j].i[1] || bar[j].d[0] != pbar[j].d[0] || bar[j].d[1] != 0.0 || bar[j].d[2] != pbar[j].d[1]) { if(0 == errors) { fprintf(stderr, "ERROR ! count=%d, position=%d, ptr = %p" " got (%d,%d,%d,%g,%g,%g) expected (%d,%d,%d,%g,%g,%g)\n", N, j, (void*)&bar[j], bar[j].i[0], bar[j].i[1], bar[j].i[2], bar[j].d[0], bar[j].d[1], bar[j].d[2], pbar[j].i[0], 0, pbar[j].i[1], pbar[j].d[0], 0.0, pbar[j].d[1]); print_bar_pbar(&bar[j], &pbar[j]); } errors++; } } OBJ_RELEASE( pConv ); return errors; }
int ompio_io_ompio_file_open (ompi_communicator_t *comm, const char *filename, int amode, ompi_info_t *info, mca_io_ompio_file_t *ompio_fh, bool use_sharedfp) { int ret = OMPI_SUCCESS; int remote_arch; ompio_fh->f_iov_type = MPI_DATATYPE_NULL; ompio_fh->f_comm = MPI_COMM_NULL; if ( ((amode&MPI_MODE_RDONLY)?1:0) + ((amode&MPI_MODE_RDWR)?1:0) + ((amode&MPI_MODE_WRONLY)?1:0) != 1 ) { return MPI_ERR_AMODE; } if ((amode & MPI_MODE_RDONLY) && ((amode & MPI_MODE_CREATE) || (amode & MPI_MODE_EXCL))) { return MPI_ERR_AMODE; } if ((amode & MPI_MODE_RDWR) && (amode & MPI_MODE_SEQUENTIAL)) { return MPI_ERR_AMODE; } ompio_fh->f_rank = ompi_comm_rank (comm); ompio_fh->f_size = ompi_comm_size (comm); remote_arch = opal_local_arch; ompio_fh->f_convertor = opal_convertor_create (remote_arch, 0); if ( true == use_sharedfp ) { ret = ompi_comm_dup (comm, &ompio_fh->f_comm); if ( OMPI_SUCCESS != ret ) { goto fn_fail; } } else { /* No need to duplicate the communicator if the file_open is called from the sharedfp component, since the comm used as an input is already a dup of the user level comm. */ ompio_fh->f_flags |= OMPIO_SHAREDFP_IS_SET; ompio_fh->f_comm = comm; } ompio_fh->f_fstype = NONE; ompio_fh->f_amode = amode; ompio_fh->f_info = info; ompio_fh->f_atomicity = 0; ompi_io_ompio_set_file_defaults (ompio_fh); ompio_fh->f_filename = filename; ompio_fh->f_split_coll_req = NULL; ompio_fh->f_split_coll_in_use = false; /*Initialize the print_queues queues here!*/ coll_write_time = (mca_io_ompio_print_queue *) malloc (sizeof(mca_io_ompio_print_queue)); coll_read_time = (mca_io_ompio_print_queue *) malloc (sizeof(mca_io_ompio_print_queue)); ompi_io_ompio_initialize_print_queue(coll_write_time); ompi_io_ompio_initialize_print_queue(coll_read_time); /* set some function pointers required for fcoll, fbtls and sharedfp modules*/ ompio_fh->f_decode_datatype=ompi_io_ompio_decode_datatype; ompio_fh->f_generate_current_file_view=ompi_io_ompio_generate_current_file_view; ompio_fh->f_sort=ompi_io_ompio_sort; ompio_fh->f_sort_iovec=ompi_io_ompio_sort_iovec; ompio_fh->f_allgather_array=ompi_io_ompio_allgather_array; ompio_fh->f_allgatherv_array=ompi_io_ompio_allgatherv_array; ompio_fh->f_gather_array=ompi_io_ompio_gather_array; ompio_fh->f_gatherv_array=ompi_io_ompio_gatherv_array; ompio_fh->f_get_num_aggregators=mca_io_ompio_get_num_aggregators; ompio_fh->f_get_bytes_per_agg=mca_io_ompio_get_bytes_per_agg; ompio_fh->f_set_aggregator_props=ompi_io_ompio_set_aggregator_props; ompio_fh->f_full_print_queue=ompi_io_ompio_full_print_queue; ompio_fh->f_register_print_entry=ompi_io_ompio_register_print_entry; /* This fix is needed for data seiving to work with two-phase collective I/O */ if ((amode & MPI_MODE_WRONLY)){ amode -= MPI_MODE_WRONLY; amode += MPI_MODE_RDWR; } /*--------------------------------------------------*/ if (OMPI_SUCCESS != (ret = mca_fs_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fs_base_file_select() failed\n"); goto fn_fail; } if (OMPI_SUCCESS != (ret = mca_fbtl_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fbtl_base_file_select() failed\n"); goto fn_fail; } if (OMPI_SUCCESS != (ret = mca_fcoll_base_file_select (ompio_fh, NULL))) { opal_output(1, "mca_fcoll_base_file_select() failed\n"); goto fn_fail; } ompio_fh->f_sharedfp_component = NULL; /*component*/ ompio_fh->f_sharedfp = NULL; /*module*/ ompio_fh->f_sharedfp_data = NULL; /*data*/ if ( true == use_sharedfp ) { if (OMPI_SUCCESS != (ret = mca_sharedfp_base_file_select (ompio_fh, NULL))) { opal_output ( ompi_io_base_framework.framework_output, "mca_sharedfp_base_file_select() failed\n"); ompio_fh->f_sharedfp = NULL; /*module*/ /* Its ok to not have a shared file pointer module as long as the shared file ** pointer operations are not used. However, the first call to any file_read/write_shared ** function will return an error code. */ } /* open the file once more for the shared file pointer if required. ** Per default, the shared file pointer specific actions are however ** only performed on first access of the shared file pointer, except ** for the addproc sharedfp component. ** ** Lazy open does not work for the addproc sharedfp ** component since it starts by spawning a process using MPI_Comm_spawn. ** For this, the first operation has to be collective which we can ** not guarantuee outside of the MPI_File_open operation. */ if ( NULL != ompio_fh->f_sharedfp && true == use_sharedfp && (!mca_io_ompio_sharedfp_lazy_open || !strcmp (ompio_fh->f_sharedfp_component->mca_component_name, "addproc") )) { ret = ompio_fh->f_sharedfp->sharedfp_file_open(comm, filename, amode, info, ompio_fh); if ( OMPI_SUCCESS != ret ) { goto fn_fail; } } } /*Determine topology information if set*/ if (ompio_fh->f_comm->c_flags & OMPI_COMM_CART){ ret = mca_io_ompio_cart_based_grouping(ompio_fh); if(OMPI_SUCCESS != ret ){ ret = MPI_ERR_FILE; } } ret = ompio_fh->f_fs->fs_file_open (comm, filename, amode, info, ompio_fh); if ( OMPI_SUCCESS != ret ) { ret = MPI_ERR_FILE; goto fn_fail; } /* If file has been opened in the append mode, move the internal file pointer of OMPIO to the very end of the file. */ if ( ompio_fh->f_amode & MPI_MODE_APPEND ) { OMPI_MPI_OFFSET_TYPE current_size; ompio_fh->f_fs->fs_file_get_size( ompio_fh, ¤t_size); ompi_io_ompio_set_explicit_offset (ompio_fh, current_size); } return OMPI_SUCCESS; fn_fail: /* no need to free resources here, since the destructor * is calling mca_io_ompio_file_close, which actually gets *rid of all allocated memory items */ return ret; }
int ompi_proc_refresh(void) { ompi_proc_t *proc = NULL; opal_list_item_t *item = NULL; ompi_vpid_t i = 0; int ret=OMPI_SUCCESS; OPAL_THREAD_LOCK(&ompi_proc_lock); for( item = opal_list_get_first(&ompi_proc_list), i = 0; item != opal_list_get_end(&ompi_proc_list); item = opal_list_get_next(item), ++i ) { proc = (ompi_proc_t*)item; /* Does not change: proc->proc_name.vpid */ proc->proc_name.jobid = OMPI_PROC_MY_NAME->jobid; /* Make sure to clear the local flag before we set it below */ proc->proc_flags = 0; if (i == OMPI_PROC_MY_NAME->vpid) { ompi_proc_local_proc = proc; proc->proc_flags = OPAL_PROC_ALL_LOCAL; proc->proc_hostname = ompi_process_info.nodename; proc->proc_arch = opal_local_arch; } else { /* get the locality information */ ret = ompi_proc_set_locality(proc); if (OMPI_SUCCESS != ret) { break; } if (ompi_process_info.num_procs < ompi_hostname_cutoff) { /* IF the number of procs falls below the specified cutoff, * then we assume the job is small enough that retrieving * the hostname (which will typically cause retrieval of * ALL modex info for this proc) will have no appreciable * impact on launch scaling */ ret = ompi_modex_recv_string_pointer(OMPI_DB_HOSTNAME, proc, (void**)&(proc->proc_hostname), OPAL_STRING); if (OMPI_SUCCESS != ret) { break; } } else { /* just set the hostname to NULL for now - we'll fill it in * as modex_recv's are called for procs we will talk to, thus * avoiding retrieval of ALL modex info for this proc until * required. Transports that delay calling modex_recv until * first message will therefore scale better than those that * call modex_recv on all procs during init. */ proc->proc_hostname = NULL; } #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT { /* get the remote architecture */ uint32_t* uiptr = &(proc->proc_arch); ret = ompi_modex_recv_key_value("OMPI_ARCH", proc, (void**)&uiptr, OPAL_UINT32); /* if arch is different than mine, create a new convertor for this proc */ if (proc->proc_arch != opal_local_arch) { OBJ_RELEASE(proc->proc_convertor); proc->proc_convertor = opal_convertor_create(proc->proc_arch, 0); } } #else /* must be same arch as my own */ proc->proc_arch = opal_local_arch; #endif } } OPAL_THREAD_UNLOCK(&ompi_proc_lock); return ret; }
static int test_upper( unsigned int length ) { double *mat1, *mat2, *inbuf; opal_datatype_t *pdt; opal_convertor_t * pConv; char *ptr; int rc; unsigned int i, j, iov_count, split_chunk, total_length; size_t max_data; struct iovec a; TIMER_DATA_TYPE start, end; long total_time; printf( "test upper matrix\n" ); pdt = upper_matrix( length ); opal_datatype_dump( pdt ); mat1 = malloc( length * length * sizeof(double) ); init_random_upper_matrix( length, mat1 ); mat2 = calloc( length * length, sizeof(double) ); total_length = length * (length + 1) * ( sizeof(double) / 2); inbuf = (double*)malloc( total_length ); ptr = (char*)inbuf; /* copy upper matrix in the array simulating the input buffer */ for( i = 0; i < length; i++ ) { uint32_t pos = i * length + i; for( j = i; j < length; j++, pos++ ) { *inbuf = mat1[pos]; inbuf++; } } inbuf = (double*)ptr; pConv = opal_convertor_create( remote_arch, 0 ); if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, pdt, 1, mat2 ) ) { printf( "Cannot attach the datatype to a convertor\n" ); return OPAL_ERROR; } GET_TIME( start ); split_chunk = (length + 1) * sizeof(double); /* split_chunk = (total_length + 1) * sizeof(double); */ for( i = total_length; i > 0; ) { if( i <= split_chunk ) { /* equal test just to be able to set a breakpoint */ split_chunk = i; } a.iov_base = ptr; a.iov_len = split_chunk; iov_count = 1; max_data = split_chunk; opal_convertor_unpack( pConv, &a, &iov_count, &max_data ); ptr += max_data; i -= max_data; if( mat2[0] != inbuf[0] ) assert(0); } GET_TIME( end ); total_time = ELAPSED_TIME( start, end ); printf( "complete unpacking in %ld microsec\n", total_time ); free( inbuf ); rc = check_diag_matrix( length, mat1, mat2 ); free( mat1 ); free( mat2 ); /* test the automatic destruction pf the data */ opal_datatype_destroy( &pdt ); assert( pdt == NULL ); OBJ_RELEASE( pConv ); return rc; }
int ompi_proc_unpack(opal_buffer_t* buf, int proclistsize, ompi_proc_t ***proclist, bool full_info, int *newproclistsize, ompi_proc_t ***newproclist) { int i; size_t newprocs_len = 0; ompi_proc_t **plist=NULL, **newprocs = NULL; /* do not free plist *ever*, since it is used in the remote group structure of a communicator */ plist = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *)); if ( NULL == plist ) { return OMPI_ERR_OUT_OF_RESOURCE; } /* free this on the way out */ newprocs = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *)); if (NULL == newprocs) { free(plist); return OMPI_ERR_OUT_OF_RESOURCE; } /* cycle through the array of provided procs and unpack * their info - as packed by ompi_proc_pack */ for ( i=0; i<proclistsize; i++ ){ int32_t count=1; ompi_process_name_t new_name; uint32_t new_arch; char *new_hostname; bool isnew = false; int rc; rc = opal_dss.unpack(buf, &new_name, &count, OMPI_NAME); if (rc != OPAL_SUCCESS) { OMPI_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } if (!full_info) { rc = opal_dss.unpack(buf, &new_arch, &count, OPAL_UINT32); if (rc != OPAL_SUCCESS) { OMPI_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING); if (rc != OPAL_SUCCESS) { OMPI_ERROR_LOG(rc); free(plist); free(newprocs); return rc; } } /* see if this proc is already on our ompi_proc_list */ plist[i] = ompi_proc_find_and_add(&new_name, &isnew); if (isnew) { /* if not, then it was added, so update the values * in the proc_t struct with the info that was passed * to us */ newprocs[newprocs_len++] = plist[i]; if (full_info) { int32_t num_recvd_entries; int32_t cnt; int32_t j; /* unpack the number of entries for this proc */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) { OMPI_ERROR_LOG(rc); break; } /* * Extract the attribute names and values */ for (j = 0; j < num_recvd_entries; j++) { opal_value_t *kv; cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) { OMPI_ERROR_LOG(rc); break; } /* if this is me, dump the data - we already have it in the db */ if (OPAL_EQUAL == ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, OMPI_PROC_MY_NAME, &new_name)) { OBJ_RELEASE(kv); } else { /* store it in the database */ if (OPAL_SUCCESS != (rc = opal_db.store_pointer((opal_identifier_t*)&new_name, kv))) { OMPI_ERROR_LOG(rc); OBJ_RELEASE(kv); } /* do not release the kv - the db holds that pointer */ } } #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT rc = opal_db.fetch((opal_identifier_t*)&new_name, "OMPI_ARCH", (void**)&new_arch, OPAL_UINT32); if( OPAL_SUCCESS == rc ) { new_arch = opal_local_arch; } #else new_arch = opal_local_arch; #endif if (ompi_process_info.num_procs < ompi_hostname_cutoff) { /* retrieve the hostname */ rc = opal_db.fetch_pointer((opal_identifier_t*)&new_name, OMPI_DB_HOSTNAME, (void**)&new_hostname, OPAL_STRING); if( OPAL_SUCCESS != rc ) { new_hostname = NULL; } } else { /* just set the hostname to NULL for now - we'll fill it in * as modex_recv's are called for procs we will talk to */ new_hostname = NULL; } } /* update all the values */ plist[i]->proc_arch = new_arch; /* if arch is different than mine, create a new convertor for this proc */ if (plist[i]->proc_arch != opal_local_arch) { #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT OBJ_RELEASE(plist[i]->proc_convertor); plist[i]->proc_convertor = opal_convertor_create(plist[i]->proc_arch, 0); #else opal_show_help("help-mpi-runtime", "heterogeneous-support-unavailable", true, ompi_process_info.nodename, new_hostname == NULL ? "<hostname unavailable>" : new_hostname); free(plist); free(newprocs); return OMPI_ERR_NOT_SUPPORTED; #endif } if (0 == strcmp(ompi_proc_local_proc->proc_hostname, new_hostname)) { plist[i]->proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER); } /* Save the hostname */ plist[i]->proc_hostname = new_hostname; } else { if (full_info) { int32_t num_recvd_entries; int32_t j, cnt; /* discard all keys: they are already locally known */ cnt = 1; if (OPAL_SUCCESS == (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) { for (j = 0; j < num_recvd_entries; j++) { opal_value_t *kv; cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) { OMPI_ERROR_LOG(rc); continue; } OBJ_RELEASE(kv); } } else { OMPI_ERROR_LOG(rc); } } } } if (NULL != newproclistsize) *newproclistsize = newprocs_len; if (NULL != newproclist) { *newproclist = newprocs; } else if (newprocs != NULL) { free(newprocs); } *proclist = plist; return OMPI_SUCCESS; }