static int PyMPELog_NewState(int commID,
const char name[],
const char color[],
const char format[],
int stateID[2])
{
int ierr = 0;
#if HAVE_MPE
MPI_Comm comm = PyMPELog_GetComm(commID);
if (comm == MPI_COMM_NULL) return 0;
#if MPE_VERSION==2
ierr = MPE_Log_get_state_eventIDs(&stateID[0], &stateID[1]);
if (ierr == -99999) { ierr = 0; stateID[0] = stateID[1] = -99999; }
if (ierr != 0) return ierr;
ierr = MPE_Describe_comm_state(comm, stateID[0], stateID[1],
name, color, format);
#else
stateID[0] = MPE_Log_get_event_number();
stateID[1] = MPE_Log_get_event_number();
ierr = MPE_Describe_state(stateID[0], stateID[1],
(char *)name, (char *)color);
#endif
#endif /* HAVE_MPE */
return ierr;
}
/**
Sets up user_state events
This does nothing if MPE is not enabled
*/
static void
setup_mpe_events(int num_types, int* types)
{
#ifdef ENABLE_MPE
PMPI_Comm_rank(MPI_COMM_WORLD,&my_log_rank);
user_state_start = malloc(num_types * sizeof(int));
user_state_end = malloc(num_types * sizeof(int));
for (int i = 0; i < num_types; i++)
{
MPE_Log_get_state_eventIDs(&user_state_start[i],
&user_state_end[i]);
if ( my_log_rank == 0 )
{
sprintf(user_state_description,"user_state_%d", types[i]);
MPE_Describe_state(user_state_start[i], user_state_end[i],
user_state_description, "MPE_CHOOSE_COLOR");
}
}
#endif
}
int main(int argc, char *argv[]) {
int nproc; // mpi groupsize
// init
MPI_Init(&argc, &argv);
#ifdef MPE
MPE_Log_get_state_eventIDs(&compute_begin, &compute_end);
MPE_Describe_state(compute_begin, compute_end, "Compute", "red");
#endif
GetArgs(argc, argv);
// deleted TP 10/12/12
// #ifdef USE_BIL
// BIL_Init(MPI_COMM_WORLD);
// #endif
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
Init();
// run
MPI_Barrier(MPI_COMM_WORLD);
TotTime = MPI_Wtime();
Run(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
TotTime = MPI_Wtime() - TotTime;
// print the performance stats
//parflow->PrintPerf(TotTime, TotInTime, TotOutTime,
// TotCompCommTime, TotParticles, size);
//#ifdef GRAPHICS
#if 0
if (rank == 0) {
VECTOR3 min, max;
min = VECTOR3(0.0f, 0.0f, 0.0f);
max = VECTOR3((float)(size[0] - 1), (float)(size[1] - 1),
(float)(size[2] - 1));
DrawInit(pt, npt, tot_ntrace, argc, argv, min, max, 0);
}
#endif
// Gather traces
{
// synchronize prior to gathering
MPI_Barrier(MPI_COMM_WORLD);
int *ntrace = NULL; // number of traces for each proc
int n; // total number of my points
// gather number of points in each trace at the root => ntrace
int all_gather = 0; // 0: only root collects the data
n = parflow->GatherNumPts(ntrace, all_gather, nblocks);
// gather the actual points in each trace at the root
parflow->GatherPts(ntrace, n, nblocks);
MPI_Barrier(MPI_COMM_WORLD);
if (ntrace)
delete[] ntrace;
}
#if 1
if (rank==0) {
PathlineLoader trace("field_lines.out");
trace.connectTraces();
trace.dump();
}
#else
// The direct printout does not have fixed order
if (rank==0)
{
printf("Traces=%d\n", tot_ntrace);
int i,j, c=0;
for (i=0; i<tot_ntrace; i++) {
for (j=0; j<npt[i]; j++)
{
VECTOR4 &v = pt[c++];
printf("%f %f %f %f, ", v[0], v[1], v[2], v[3]);
}
printf("\n");
}
}
#endif
//printf("cleaning up\n");
Cleanup();
MPI_Barrier(MPI_COMM_WORLD);
// edited TP 10/12/12
// #ifdef USE_BIL
// BIL_Finalize();
// #endif
//printf("DIY_Finalize\n");
DIY_Finalize();
// end TP
MPI_Finalize();
}
void ADIO_Init(int *argc, char ***argv, int *error_code)
{
#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
char *c;
#endif
MPL_UNREFERENCED_ARG(argc);
MPL_UNREFERENCED_ARG(argv);
#ifdef ROMIO_INSIDE_MPICH
MPIR_Ext_init();
#endif
#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
c = getenv("MPIO_DIRECT_READ");
if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE")))
ADIOI_Direct_read = 1;
else
ADIOI_Direct_read = 0;
c = getenv("MPIO_DIRECT_WRITE");
if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE")))
ADIOI_Direct_write = 1;
else
ADIOI_Direct_write = 0;
#endif
#ifdef ADIOI_MPE_LOGGING
{
MPE_Log_get_state_eventIDs(&ADIOI_MPE_open_a, &ADIOI_MPE_open_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_read_a, &ADIOI_MPE_read_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_write_a, &ADIOI_MPE_write_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_lseek_a, &ADIOI_MPE_lseek_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_close_a, &ADIOI_MPE_close_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_writelock_a, &ADIOI_MPE_writelock_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_readlock_a, &ADIOI_MPE_readlock_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_unlock_a, &ADIOI_MPE_unlock_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_postwrite_a, &ADIOI_MPE_postwrite_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_openinternal_a, &ADIOI_MPE_openinternal_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_stat_a, &ADIOI_MPE_stat_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_iread_a, &ADIOI_MPE_iread_b);
MPE_Log_get_state_eventIDs(&ADIOI_MPE_iwrite_a, &ADIOI_MPE_iwrite_b);
int comm_world_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
if (comm_world_rank == 0) {
MPE_Describe_state(ADIOI_MPE_open_a, ADIOI_MPE_open_b, "open", "orange");
MPE_Describe_state(ADIOI_MPE_read_a, ADIOI_MPE_read_b, "read", "green");
MPE_Describe_state(ADIOI_MPE_write_a, ADIOI_MPE_write_b, "write", "blue");
MPE_Describe_state(ADIOI_MPE_lseek_a, ADIOI_MPE_lseek_b, "lseek", "red");
MPE_Describe_state(ADIOI_MPE_close_a, ADIOI_MPE_close_b, "close", "grey");
MPE_Describe_state(ADIOI_MPE_writelock_a, ADIOI_MPE_writelock_b, "writelock", "plum");
MPE_Describe_state(ADIOI_MPE_readlock_a, ADIOI_MPE_readlock_b, "readlock", "magenta");
MPE_Describe_state(ADIOI_MPE_unlock_a, ADIOI_MPE_unlock_b, "unlock", "purple");
MPE_Describe_state(ADIOI_MPE_postwrite_a, ADIOI_MPE_postwrite_b, "postwrite", "ivory");
MPE_Describe_state(ADIOI_MPE_openinternal_a, ADIOI_MPE_openinternal_b, "open system",
"blue");
MPE_Describe_state(ADIOI_MPE_stat_a, ADIOI_MPE_stat_b, "stat", "purple");
MPE_Describe_state(ADIOI_MPE_iread_a, ADIOI_MPE_iread_b, "iread", "purple");
MPE_Describe_state(ADIOI_MPE_iwrite_a, ADIOI_MPE_iwrite_b, "iwrite", "purple");
}
}
#endif
*error_code = MPI_SUCCESS;
MPI_Op_create(my_consensus, 1, &ADIO_same_amode);
}
void ADIO_Init(int *argc, char ***argv, int *error_code)
{
#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
char *c;
#endif
ADIOI_UNREFERENCED_ARG(argc);
ADIOI_UNREFERENCED_ARG(argv);
/* initialize the linked list containing flattened datatypes */
ADIOI_Flatlist = (ADIOI_Flatlist_node *) ADIOI_Malloc(sizeof(ADIOI_Flatlist_node));
ADIOI_Flatlist->type = MPI_DATATYPE_NULL;
ADIOI_Flatlist->next = NULL;
ADIOI_Flatlist->blocklens = NULL;
ADIOI_Flatlist->indices = NULL;
#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
c = getenv("MPIO_DIRECT_READ");
if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE")))
ADIOI_Direct_read = 1;
else ADIOI_Direct_read = 0;
c = getenv("MPIO_DIRECT_WRITE");
if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE")))
ADIOI_Direct_write = 1;
else ADIOI_Direct_write = 0;
#endif
/* Assume system-wide hints won't change between runs: move hint processing
* from ADIO_Open to here */
/* FIXME should be checking error code from MPI_Info_create here */
MPI_Info_create(&ADIOI_syshints);
ADIOI_process_system_hints(ADIOI_syshints);
#ifdef ADIOI_MPE_LOGGING
{
MPE_Log_get_state_eventIDs( &ADIOI_MPE_open_a, &ADIOI_MPE_open_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_read_a, &ADIOI_MPE_read_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_write_a, &ADIOI_MPE_write_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_lseek_a, &ADIOI_MPE_lseek_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_close_a, &ADIOI_MPE_close_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_writelock_a,
&ADIOI_MPE_writelock_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_readlock_a,
&ADIOI_MPE_readlock_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_unlock_a, &ADIOI_MPE_unlock_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_postwrite_a,
&ADIOI_MPE_postwrite_b );
MPE_Log_get_state_eventIDs( &ADIOI_MPE_openinternal_a,
&ADIOI_MPE_openinternal_b);
MPE_Log_get_state_eventIDs( &ADIOI_MPE_stat_a, &ADIOI_MPE_stat_b);
MPE_Log_get_state_eventIDs( &ADIOI_MPE_iread_a, &ADIOI_MPE_iread_b);
MPE_Log_get_state_eventIDs( &ADIOI_MPE_iwrite_a, &ADIOI_MPE_iwrite_b);
int comm_world_rank;
MPI_Comm_rank( MPI_COMM_WORLD, &comm_world_rank );
if ( comm_world_rank == 0 ) {
MPE_Describe_state( ADIOI_MPE_open_a, ADIOI_MPE_open_b,
"open", "orange" );
MPE_Describe_state( ADIOI_MPE_read_a, ADIOI_MPE_read_b,
"read", "green" );
MPE_Describe_state( ADIOI_MPE_write_a, ADIOI_MPE_write_b,
"write", "blue" );
MPE_Describe_state( ADIOI_MPE_lseek_a, ADIOI_MPE_lseek_b,
"lseek", "red" );
MPE_Describe_state( ADIOI_MPE_close_a, ADIOI_MPE_close_b,
"close", "grey" );
MPE_Describe_state( ADIOI_MPE_writelock_a, ADIOI_MPE_writelock_b,
"writelock", "plum" );
MPE_Describe_state( ADIOI_MPE_readlock_a, ADIOI_MPE_readlock_b,
"readlock", "magenta" );
MPE_Describe_state( ADIOI_MPE_unlock_a, ADIOI_MPE_unlock_b,
"unlock", "purple" );
MPE_Describe_state( ADIOI_MPE_postwrite_a, ADIOI_MPE_postwrite_b,
"postwrite", "ivory" );
MPE_Describe_state( ADIOI_MPE_openinternal_a, ADIOI_MPE_openinternal_b, "open system", "blue");
MPE_Describe_state( ADIOI_MPE_stat_a, ADIOI_MPE_stat_b, "stat", "purple");
MPE_Describe_state( ADIOI_MPE_iread_a, ADIOI_MPE_iread_b, "iread", "purple");
MPE_Describe_state( ADIOI_MPE_iwrite_a, ADIOI_MPE_iwrite_b, "iwrite", "purple");
}
}
#endif
*error_code = MPI_SUCCESS;
MPI_Op_create(my_consensus, 1, &ADIO_same_amode);
}
int main( int argc, char *argv[] )
{
int n, myid, numprocs, ii, jj;
double PI25DT = 3.141592653589793238462643;
double mypi, pi, h, sum, x;
double startwtime = 0.0, endwtime;
int namelen;
int event1a, event1b, event2a, event2b,
event3a, event3b, event4a, event4b;
int event1, event2, event3;
char processor_name[ MPI_MAX_PROCESSOR_NAME ];
MPI_Init( &argc, &argv );
MPI_Pcontrol( 0 );
MPI_Comm_size( MPI_COMM_WORLD, &numprocs );
MPI_Comm_rank( MPI_COMM_WORLD, &myid );
MPI_Get_processor_name( processor_name, &namelen );
fprintf( stderr, "Process %d running on %s\n", myid, processor_name );
/*
MPE_Init_log() & MPE_Finish_log() are NOT needed when
liblmpe.a is linked with this program. In that case,
MPI_Init() would have called MPE_Init_log() already.
*/
#if defined( NO_MPI_LOGGING )
MPE_Init_log();
#endif
/*
user should NOT assign eventIDs directly in MPE_Describe_state()
Get the eventIDs for user-defined STATES(rectangles) from
MPE_Log_get_state_eventIDs() instead of the deprecated function
MPE_Log_get_event_number().
*/
MPE_Log_get_state_eventIDs( &event1a, &event1b );
MPE_Log_get_state_eventIDs( &event2a, &event2b );
MPE_Log_get_state_eventIDs( &event3a, &event3b );
MPE_Log_get_state_eventIDs( &event4a, &event4b );
if ( myid == 0 ) {
MPE_Describe_state( event1a, event1b, "Broadcast", "red" );
MPE_Describe_state( event2a, event2b, "Sync", "orange" );
MPE_Describe_state( event3a, event3b, "Compute", "blue" );
MPE_Describe_state( event4a, event4b, "Reduce", "green" );
}
/* Get event ID for Solo-Event(single timestamp object) from MPE */
MPE_Log_get_solo_eventID( &event1 );
MPE_Log_get_solo_eventID( &event2 );
MPE_Log_get_solo_eventID( &event3 );
if ( myid == 0 ) {
MPE_Describe_event( event1, "Broadcast Post", "white" );
MPE_Describe_event( event2, "Compute Start", "purple" );
MPE_Describe_event( event3, "Compute End", "navy" );
}
if ( myid == 0 ) {
n = 1000000;
startwtime = MPI_Wtime();
}
MPI_Barrier( MPI_COMM_WORLD );
MPI_Pcontrol( 1 );
/*
MPE_Start_log();
*/
for ( jj = 0; jj < 5; jj++ ) {
MPE_Log_event( event1a, 0, NULL );
MPI_Bcast( &n, 1, MPI_INT, 0, MPI_COMM_WORLD );
MPE_Log_event( event1b, 0, NULL );
MPE_Log_event( event1, 0, NULL );
MPE_Log_event( event2a, 0, NULL );
MPI_Barrier( MPI_COMM_WORLD );
MPE_Log_event( event2b, 0, NULL );
MPE_Log_event( event2, 0, NULL );
MPE_Log_event( event3a, 0, NULL );
h = 1.0 / (double) n;
sum = 0.0;
for ( ii = myid + 1; ii <= n; ii += numprocs ) {
x = h * ((double)ii - 0.5);
sum += f(x);
}
mypi = h * sum;
MPE_Log_event( event3b, 0, NULL );
MPE_Log_event( event3, 0, NULL );
pi = 0.0;
MPE_Log_event( event4a, 0, NULL );
MPI_Reduce( &mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD );
MPE_Log_event( event4b, 0, NULL );
MPE_Log_sync_clocks();
}
#if defined( NO_MPI_LOGGING )
if ( argv != NULL )
MPE_Finish_log( argv[0] );
else
MPE_Finish_log( "cpilog" );
#endif
if ( myid == 0 ) {
endwtime = MPI_Wtime();
printf( "pi is approximately %.16f, Error is %.16f\n",
pi, fabs(pi - PI25DT) );
printf( "wall clock time = %f\n", endwtime-startwtime );
}
MPI_Finalize();
return( 0 );
}
