int main( int argc, char **argv )
{
// initialize MPI
MPI_Init( &argc, &argv );
// get MPI info
int rank;
int comm_size;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
// declarations
printf( "Entering proc rank %i\n", rank );
int arrsize = -1;
int *rec_buff = NULL;
int rec_buff_size = -1;
if( rank == 0 ) // parent
{
FILE *hfile = fopen( argv[ 1 ], "rb" );
if( hfile == NULL ) // bad file
{
int placeholder = BAD_SIZE;
MPI_Bcast( &placeholder, 1, MPI_INT, 0, MPI_COMM_WORLD );
MPI_Finalize();
return printf( "no such file %s\n", argv[1] );
}
// read the file for element count and cap the per-proc buffer size
char buffer[ TINY_SIZE ] = { 0 };
fgets( buffer, sizeof( buffer ), hfile );
arrsize = atoi( buffer );
rec_buff_size = ( arrsize / comm_size ) + ( ( arrsize % comm_size ) ? 1 : 0 );
// map the buffer with the integers
int *buff = malloc( rec_buff_size * comm_size * sizeof( int ) );
for( int it = 0; it < arrsize; ++it )
{
memset( buffer, '\0', sizeof( buffer ) );
char c;
while( isspace( ( c = fgetc( hfile ) ) ) ); // skip the spaces
for( int it2 = 0; it2 < sizeof( buffer ) && !isspace( c ); c = fgetc( hfile ) )
buffer[ it2++ ] = c;
buff[ it ] = atoi( buffer );
}
// fill the rest with INT_MAX to be truncated later
for( int it = arrsize; it < rec_buff_size * comm_size; ++it )
buff[ it ] = INT_MAX;
// scatter the array across other processes
rec_buff = malloc( rec_buff_size * sizeof( int ) );
MPI_Bcast( &rec_buff_size, 1, MPI_INT, 0, MPI_COMM_WORLD ); // broadcast the data size
MPI_Scatter( buff, rec_buff_size, MPI_INT, rec_buff, rec_buff_size, MPI_INT, 0, MPI_COMM_WORLD );
printf( "parent broadcast sent\n" );
printf( "child rank %i recieved the buffer size. It is %i\n", rank, rec_buff_size );
// clean up
free( buff );
fclose( hfile );
}
else // child
{
// recieve the data size
MPI_Bcast( &rec_buff_size, 1, MPI_INT, 0, MPI_COMM_WORLD );
if( rec_buff_size == BAD_SIZE ) // invalid input filename
{
MPI_Finalize();
return printf( "child rank %i recieved the terminate message\n", rank );
}
printf( "child rank %i recieved the buffer size. It is %i\n", rank, rec_buff_size );
rec_buff = malloc( rec_buff_size * sizeof( int ) );
// recieve the data
MPI_Scatter( NULL, 0, MPI_DATATYPE_NULL, rec_buff, rec_buff_size, MPI_INT, 0, MPI_COMM_WORLD );
printf( "child rank %i recieved the data\n", rank );
}
// finally, sort the data
mergesort( rec_buff, rec_buff_size );
// begin merging the data
for( int step = 1; step < comm_size; step *= 2 )
{
if( rank % (2 * step) )
{
MPI_Send( &rec_buff_size, 1, MPI_INT, rank - step, 0, MPI_COMM_WORLD ); // send data size
MPI_Send( rec_buff, rec_buff_size, MPI_INT, rank - step, 0, MPI_COMM_WORLD ); // send data for merging
break; // we are now finished for this proc
}
else
{
if( rank + step < comm_size )
{
// allocate a second buffer to recieve data, and double the buffer size
printf( "rank %i is merging from rank %i\n", rank, rank + step );
// recieve the incoming buffer size and allocate enough memory for merging
int sec_buff_size;
MPI_Recv( &sec_buff_size, 1, MPI_INT, rank + step, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
int *second_buff = malloc( sec_buff_size * sizeof( int ) );
int *new_rec_buff = malloc( (rec_buff_size + sec_buff_size) * sizeof( int ) );
// recieve the buffer
MPI_Recv( second_buff, sec_buff_size, MPI_INT, rank + step, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
// merge
quick_merge( new_rec_buff, rec_buff, rec_buff_size, second_buff, sec_buff_size );
// cleanup
free( second_buff );
free( rec_buff );
rec_buff = new_rec_buff;
rec_buff_size *= 2;
}
}
}
// create and write the output file
if( rank == 0 )
{
char *outfilename = argc > 2 ? argv[ 2 ] : "outfile";
write_outfile( outfilename, rec_buff, arrsize );
}
// clean up
free( rec_buff );
MPI_Finalize();
return 0;
}
int start()
{
/* This is the "starter process" for statistical profiling.
*
* Create output file for profiling data. Create named pipe to
* synchronize with stopper process. Fork so the parent can exit.
* Allocate memory for profiling data. Start profiling in kernel.
* Complete detachment from terminal. Write known string to named
* pipe, which blocks until read by stopper process. Redirect
* stdout/stderr to the named pipe. Write profiling data to file.
* Clean up.
*/
int log_fd;
if (init_outfile() || create_named_pipe()) return 1;
printf("Starting statistical profiling.\n");
if (fork() != 0) exit(0);
if (alloc_mem()) return 1;
if (sprofile(PROF_START, mem_size, freq, intr_type, &sprof_info, mem_ptr)) {
perror("sprofile");
fprintf(stderr, "Error starting profiling.\n");
return 1;
}
detach();
/* Temporarily redirect to system log to catch errors. */
log_fd = open(DEV_LOG, O_WRONLY);
dup2(log_fd, 1);
dup2(log_fd, 2);
if ((npipe_fd = open(NPIPE, O_WRONLY)) < 0) {
fprintf(stderr, "Unable to open named pipe %s.\n", NPIPE);
return 1;
} else
/* Synchronize with stopper process. */
write(npipe_fd, SYNCING, strlen(SYNCING));
/* Now redirect to named pipe. */
dup2(npipe_fd, 1);
dup2(npipe_fd, 2);
mem_used = sprof_info.mem_used;
if (mem_used == -1) {
fprintf(stderr, "WARNING: Profiling was stopped prematurely due to ");
fprintf(stderr, "insufficient memory.\n");
fprintf(stderr, "Try increasing available memory using the -m switch.\n");
}
if (write_outfile()) return 1;
close(log_fd);
close(npipe_fd);
unlink(NPIPE);
close(outfile_fd);
free(mem_ptr);
return 0;
}
int main(int argc, char **argv) {
int fn,argnum,retval;
size_t tsdata_size;
char *iotrace_data_dir, *current_work_dir;
/* tsdumps for the source and destination sides */
xdd_ts_header_t *src = NULL;
xdd_ts_header_t *dst = NULL;
xdd_ts_header_t *tsdata = NULL;
/* timestamp entries sorted by ending time */
xdd_ts_tte_t **read_op = NULL;
xdd_ts_tte_t **send_op = NULL;
xdd_ts_tte_t **recv_op = NULL;
xdd_ts_tte_t **write_op = NULL;
/* get command line options */
argnum = getoptions(argc, argv);
fn = MAX(argnum,1);
/* get the src or dst data structs */
retval = xdd_readfile(argv[fn],&tsdata,&tsdata_size);
if (retval == 0) {
fprintf(stderr,"xdd_readfile() failed...reading 1st xdd timestamp file: %s ...exiting.\n",argv[fn]);
exit(1);
}
/* determine whether this is the source or destination file */
/* if only 1 file given, then source==read, destination==write */
if (READ_OP(tsdata->tsh_tte[0].tte_op_type)) {
src = tsdata;
srcfilename = argv[fn];
}
else
if (WRITE_OP(tsdata->tsh_tte[0].tte_op_type)){
dst = tsdata;
dstfilename = argv[fn];
}
else {
fprintf(stderr,"Timestamp dump had invalid operation in tte[0]: %s\n",argv[fn]);
exit(1);
}
/* if 2nd file specified, this must be an e2e pair (src,dst) */
if ( argv[fn+1] != NULL ) {
/* get the src or dst data structs */
retval = xdd_readfile(argv[fn+1],&tsdata,&tsdata_size);
if (retval == 0) {
fprintf(stderr,"xdd_readfile() failed...reading 2nd xdd timestamp file: %s ...exiting.\n",argv[fn+1]);
exit(1);
}
/* determine whether this is the source or destination file */
if (READ_OP(tsdata->tsh_tte[0].tte_op_type)) {
if (src != NULL) {
fprintf(stderr,"You passed 2 source dump files.\n");
fprintf(stderr,"Please pass matching source and destination dumps or a single dump .\n");
exit(1);
}
src = tsdata;
srcfilename = argv[fn+1];
}
else
if (WRITE_OP(tsdata->tsh_tte[0].tte_op_type)) {
if (dst != NULL) {
fprintf(stderr,"You passed 2 destination dump files.\n");
fprintf(stderr,"Please pass matching source and destination dumps or a single dump .\n");
exit(1);
}
dst = tsdata;
dstfilename = argv[fn+1];
}
else {
fprintf(stderr,"Timestamp dump had invalid operation in tte[0]: %s\n",argv[fn+1]);
exit(1);
}
/* If these files are from the same transfer, they should be the same size. */
if (src->tsh_tt_bytes != dst->tsh_tt_bytes) {
fprintf(stderr,"Source and destination files are not the same size.\n");
fprintf(stderr,"Please pass matching source and destination dumps.\n");
exit(1);
}
}
/* get total threads */
if (src != NULL) xdd_getthreads(src,&total_threads_src, thread_id_src, &op_mix);
if (dst != NULL) xdd_getthreads(dst,&total_threads_dst, thread_id_dst, &op_mix);
/* If e2e, better have same number of threads. */
if (total_threads_src > 0 && total_threads_dst > 0 && total_threads_src != total_threads_dst) {
fprintf(stderr,"Warning: source (%d) and destination (%d) files don't have the same number of I/O threads.\n",
total_threads_src, total_threads_dst);
}
/* match-add kernel events with xdd events if specfied */
/* at this point, src & dst contain all xdd events */
/* note that iotrace_data_dir is independent of $PWD */
if (kernel_trace)
{
if ((iotrace_data_dir=getenv("TRACE_LOG_LOC")) == NULL)
iotrace_data_dir=getenv("HOME");
current_work_dir=getenv("PWD");
if (src != NULL) {
sprintf(kernfilename,"decode %s/dictionary* %s/iotrace_data.%d.out",
iotrace_data_dir, iotrace_data_dir, src->tsh_target_thread_id);
if ( system(kernfilename) == -1 ) {
fprintf(stderr,"in main: shell command failed: %s\n",kernfilename);
exit(1);
}
if (strcmp(iotrace_data_dir,current_work_dir) != 0) {
sprintf(kernfilename,"mv %s/iotrace_data.%d.out.ascii %s",
iotrace_data_dir, src->tsh_target_thread_id,getenv("PWD"));
if ( system(kernfilename) == -1 ) {
fprintf(stderr,"in main: shell command failed: %s\n",kernfilename);
exit(1);
}
}
sprintf(kernfilename,"%s/iotrace_data.%d.out.ascii",
current_work_dir, src->tsh_target_thread_id);
matchadd_kernel_events(1,total_threads_src,thread_id_src,kernfilename,src);
if (op_mix > 0.0)
matchadd_kernel_events(0,total_threads_src,thread_id_src,kernfilename,src);
/* write out the src and dst data structs that now contain kernel data */
sprintf(kernfilename,"%s/%sk", current_work_dir, srcfilename);
retval = xdd_writefile(kernfilename,src,tsdata_size);
if (retval == 0) {
fprintf(stderr,"xdd_writefile() failed to write %s ...exiting.\n",kernfilename);
exit(1);
}
}
if (dst != NULL) {
sprintf(kernfilename,"decode %s/dictionary* %s/iotrace_data.%d.out",
iotrace_data_dir, iotrace_data_dir, dst->tsh_target_thread_id);
if ( system(kernfilename) == -1 ) {
fprintf(stderr,"in main: shell command failed: %s\n",kernfilename);
exit(1);
}
if (strcmp(iotrace_data_dir,current_work_dir) != 0) {
sprintf(kernfilename,"mv %s/iotrace_data.%d.out.ascii %s",
iotrace_data_dir, dst->tsh_target_thread_id,getenv("PWD"));
if ( system(kernfilename) == -1 ) {
fprintf(stderr,"in main: shell command failed: %s\n",kernfilename);
exit(1);
}
}
sprintf(kernfilename,"%s/iotrace_data.%d.out.ascii",
current_work_dir, dst->tsh_target_thread_id);
matchadd_kernel_events(0,total_threads_dst,thread_id_dst,kernfilename,dst);
if (op_mix > 0.0)
matchadd_kernel_events(1,total_threads_dst,thread_id_dst,kernfilename,dst);
sprintf(kernfilename,"%s/%sk", current_work_dir, dstfilename);
retval = xdd_writefile(kernfilename,dst,tsdata_size);
if (retval == 0) {
fprintf(stderr,"xdd_writefile() failed to write %s ...exiting.\n",kernfilename);
exit(1);
}
}
}
/* get the MIN timestamp normalize the times */
if (src!=NULL) normalize_time(src,&src_start_norm);
if (dst!=NULL) normalize_time(dst,&dst_start_norm);
/* sort timestamp entries for each operation (read/send/recv/write) */
/* kernel events are co-located with matching xdd op */
if (src!=NULL) {
sort_by_time(src, DISK_END, &read_op);
if (dst!=NULL) {
sort_by_time(src, NET_END, &send_op);
}
}
if (dst!=NULL) {
sort_by_time(dst, DISK_END, &write_op);
if (src!=NULL) {
sort_by_time(dst, NET_END, &recv_op);
}
}
/* write the outfile(s) */
write_outfile (src,dst,read_op,send_op,recv_op,write_op);
/* free memory */
free(read_op);
free(send_op);
free(recv_op);
free(write_op);
free(src);
free(dst);
return 0;
}
