/*
* Control of counters (Reading/Writing/Starting/Stopping/Setup)
* functions
*/
int
IOUNIT_init_control_state( hwd_control_state_t * ptr )
{
#ifdef DEBUG_BGQ
printf( "IOUNIT_init_control_state\n" );
#endif
IOUNIT_control_state_t * this_state = ( IOUNIT_control_state_t * ) ptr;
this_state->EventGroup = Bgpm_CreateEventSet();
CHECK_BGPM_ERROR( this_state->EventGroup, "Bgpm_CreateEventSet" );
return PAPI_OK;
}
/*
* Control of counters (Reading/Writing/Starting/Stopping/Setup)
* functions
*/
int
NWUNIT_init_control_state( hwd_control_state_t * ptr )
{
#ifdef DEBUG_BGQ
printf( "NWUNIT_init_control_state\n" );
#endif
int retval;
NWUNIT_control_state_t * this_state = ( NWUNIT_control_state_t * ) ptr;
this_state->EventGroup = Bgpm_CreateEventSet();
retval = _check_BGPM_error( this_state->EventGroup, "Bgpm_CreateEventSet" );
if ( retval < 0 ) return retval;
return PAPI_OK;
}
void multiplex(period,normalize)
{
// initialize sw thread to use BGPM in sw thread distributed mode
// Allows up to 12 simultaineous punit counter per thread when using only
// 2 threads/core
Bgpm_Init(BGPM_MODE_SWDISTRIB);
// Create event set and add events needed to calculate CPI,
int hEvtSet = Bgpm_CreateEventSet();
// Multiplexing must be enabled prior to adding events to a Punit event set.
// With Multiplexing active on Punit event set, we can have as many Punit events as we like,
// regardless of Max Events. Muxing will create as many groups as needed to hold all the events,
// but by default only assigns up to 5 events per group (And reserves the 6th counter to track the number of
// cycles a group has been active).
// However, a few of the core-wide attributes used by events must still remain consistent
// (like the l1p mode associated with the l1p events)
//
// Parameters:
// period-> number of cycles between multiplex switches. A value of 0 disables multiplexing for this thread,
// which means you must explicitly call Bgpm_SwitchMux to switch between groups.
// normalize->Pass in BGPM_NORMAL or BGPM_NOTNORMAL to choose whether event counts
// reported by Bgpm_ReadEvent() or Bgpm_ReadEventList() will be scaled to the to the maximum number of cycles spent
// by any mux group.
// With BGPM_NOTNORMAL, Bgpm_ReadEvent() and Bgpm_ReadEventList() report the raw numbers and you may use
// Bgpm_GetMultiplex() and Bgpm_GetMuxEventElapsedCycles() to do your own normalization as desired.
Bgpm_SetMultiplex2(hEvtSet, period, normalize, BGPMMuxMinEvts);
Bgpm_AddEventList(hEvtSet, evtList, sizeof(evtList)/sizeof(unsigned) );
Bgpm_Apply(hEvtSet);
Bgpm_Start(hEvtSet);
const int loops = 100;
int i;
for (i=0; i<loops; i++) {
CreateEvents();
if(!period){Bgpm_SwitchMux(hEvtSet);} // Switching mutiplex is needed only in case of period=0.
}
Bgpm_Stop(hEvtSet);
PrintCounts(hEvtSet);
Bgpm_Disable();
}
/*
* Control of counters (Reading/Writing/Starting/Stopping/Setup)
* functions
*/
int
L2UNIT_init_control_state( hwd_control_state_t * ptr )
{
#ifdef DEBUG_BGQ
printf( "L2UNIT_init_control_state\n" );
#endif
L2UNIT_control_state_t * this_state = ( L2UNIT_control_state_t * ) ptr;
this_state->EventGroup = Bgpm_CreateEventSet();
CHECK_BGPM_ERROR( this_state->EventGroup, "Bgpm_CreateEventSet" );
// initialize overflow flag to OFF (0)
this_state->overflow = 0;
this_state->overflow_count = 0;
// initialized BGPM eventGroup flag to NOT applied yet (0)
this_state->bgpm_eventset_applied = 0;
return PAPI_OK;
}
/*
* Control of counters (Reading/Writing/Starting/Stopping/Setup)
* functions
*/
int
IOUNIT_init_control_state( hwd_control_state_t * ptr )
{
#ifdef DEBUG_BGQ
printf( "IOUNIT_init_control_state\n" );
#endif
int retval;
IOUNIT_control_state_t * this_state = ( IOUNIT_control_state_t * ) ptr;
this_state->EventGroup = Bgpm_CreateEventSet();
retval = _check_BGPM_error( this_state->EventGroup, "Bgpm_CreateEventSet" );
if ( retval < 0 ) return retval;
// initialize overflow flag to OFF (0)
this_state->overflow = 0;
this_state->overflow_count = 0;
return PAPI_OK;
}
/*
* Delete BGPM eventGroup and create an new empty one
*/
int
_common_deleteRecreate( int *EventGroup_ptr )
{
#ifdef DEBUG_BGQ
printf( _AT_ " _common_deleteRecreate: *EventGroup_ptr=%d\n", *EventGroup_ptr);
#endif
int retval;
// delete previous bgpm eventset
retval = Bgpm_DeleteEventSet( *EventGroup_ptr );
retval = _check_BGPM_error( retval, "Bgpm_DeleteEventSet" );
if ( retval < 0 ) return retval;
// create a new empty bgpm eventset
*EventGroup_ptr = Bgpm_CreateEventSet();
retval = _check_BGPM_error( *EventGroup_ptr, "Bgpm_CreateEventSet" );
if ( retval < 0 ) return retval;
#ifdef DEBUG_BGQ
printf( _AT_ " _common_deleteRecreate: *EventGroup_ptr=%d\n", *EventGroup_ptr);
#endif
return PAPI_OK;
}
void tw_run(void) {
tw_pe *me;
late_sanity_check();
me = setup_pes();
#ifdef USE_BGPM
Bgpm_Init(BGPM_MODE_SWDISTRIB);
// Create event set and add events needed to calculate CPI,
int hEvtSet = Bgpm_CreateEventSet();
// Change the type here to get the other performance monitor sets -- defaults to instList set
// Sets need to be different because there are conflicts when mixing certain events.
Bgpm_AddEventList(hEvtSet, instList, sizeof(instList)/sizeof(unsigned) );
if( me->id == 0) {
printf("***************************************************************************************** \n");
printf("* NOTICE: Build configured with Blue Gene/Q specific, BGPM performance monitoring code!!* \n");
printf("***************************************************************************************** \n");
}
Bgpm_Apply(hEvtSet);
Bgpm_Start(hEvtSet);
#endif
tw_sched_init(me);
switch(g_tw_synchronization_protocol) {
case SEQUENTIAL: // case 1
tw_scheduler_sequential(me);
break;
case CONSERVATIVE: // case 2
tw_scheduler_conservative(me);
break;
case OPTIMISTIC: // case 3
tw_scheduler_optimistic(me);
break;
case OPTIMISTIC_DEBUG: // case 4
tw_scheduler_optimistic_debug(me);
break;
default:
tw_error(TW_LOC, "No Synchronization Protocol Specified! \n");
}
#ifdef USE_BGPM
Bgpm_Stop(hEvtSet);
if( me->id == 0 ) { // Have only RANK 0 output stats
int i;
uint64_t cnt;
int numEvts = Bgpm_NumEvents(hEvtSet);
printf("\n \n ================================= \n");
printf( "Performance Counter Results:\n");
printf("--------------------------------- \n");
for (i=0; i<numEvts; i++) {
Bgpm_ReadEvent(hEvtSet, i, &cnt);
printf(" %40s = %20llu\n", Bgpm_GetEventLabel(hEvtSet, i), cnt);
}
printf("================================= \n");
}
#endif
}
INLINE void PROFILER_INIT()
{
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
MPI_Comm_size(MPI_COMM_WORLD,&numranks);
int coords[6], tmasterRank;
MPIX_Rank2torus(myrank, coords);
/* choose the MPI rank on (0, 0, 0, 0, 0) [0] as the master rank */
if(coords[0]+coords[1]+coords[2]+coords[3]+coords[4]+coords[5] == 0) {
isMaster = 1;
printf("Init intercepted by bgqcounter unit\n");
} else {
isMaster = 0;
}
char *filename = getenv("BGQ_COUNTER_FILE");
if(isMaster) {
if(filename != NULL)
dataFile = fopen(filename,"w");
else
dataFile = stdout;
}
#if BGQ_DEBUG
if(isMaster) {
printf("File opened, Initializing BGPM\n");
}
#endif
Bgpm_Init(BGPM_MODE_SWDISTRIB);
#if BGQ_DEBUG
if(isMaster) {
printf("Initialized BGPM, Splitting communicator\n");
}
#endif
/* split communicator based on the T dimension */
isZero = (coords[5] == 0) ? 1 : 0;
MPI_Comm_split(MPI_COMM_WORLD, isZero, myrank, &profile_comm);
#if BGQ_DEBUG
if(isMaster) {
printf("Communicator split done, find master\n");
}
#endif
/* Every process needs to know the master rank in profile_comm to know
the root of the broadcast */
coords[0] = coords[1] = coords[2] = coords[3] = coords[4] = coords[5] = 0;
MPIX_Torus2rank(coords, &tmasterRank);
#if BGQ_DEBUG
if(isMaster) {
printf("Found master, informing master\n");
}
#endif
if(isMaster) {
MPI_Comm_rank(profile_comm, &masterRank);
}
/* Broadcast the rank of the master in profile_comm */
MPI_Bcast(&masterRank, 1, MPI_INT, tmasterRank, MPI_COMM_WORLD);
#if BGQ_DEBUG
if(isMaster) {
printf("Informed master, attaching counters\n");
}
#endif
if(isZero) {
hNWSet = Bgpm_CreateEventSet();
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_DYN_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_ESC_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_SUBC_COL_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_RECV);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_RECV_FIFO);
numevents = 6;
if (Bgpm_Attach(hNWSet, UPC_NW_ALL_TORUS_LINKS, 0) != 0) {
printf("Error: something went wrong in attaching link counters\n");
}
curset = maxset = 0;
for(unsigned int i = 0; i < NUM_REGIONS; i++) {
for(unsigned int j = 0; j < NUM_TORUS_LINKS * numevents; j++) {
values[i].counters[j] = 0;
}
values[i].time = 0;
}
}
if(isMaster) {
printf("Init intercept complete\n");
}
}
