void monitor_mpi_pre_init(void)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
tls->in_mpi_pre_init = true;
CBTF_in_mpi_startup = true;
init_mpi_comm_rank = false;
cbtf_offline_notify_event(CBTF_Monitor_MPI_pre_init_event);
if (tls->sampling_status == CBTF_Monitor_Started) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_pre_init PAUSE SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_MPI_pre_init_event,CBTF_Monitor_Paused);
} else {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_pre_init IS PAUSED\n",
thePid,monitor_get_thread_num());
}
}
}
/*
* callbacks for handling of THREADS
*/
void monitor_fini_thread(void *ptr)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if (!tls->collector_started) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_thread collector never started.\n",
thePid,monitor_get_thread_num());
}
return;
}
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_thread STOP SAMPLING\n",
thePid,monitor_get_thread_num());
}
cbtf_offline_stop_sampling(NULL,true);
tls->sampling_status = CBTF_Monitor_Finished;
tls->thread_is_terminating = 1;
// The following call will handle any mrnet specific activity.
cbtf_offline_notify_event(CBTF_Monitor_fini_thread_event);
}
void*
monitor_thread_pre_create(void)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
cbtf_offline_notify_event(CBTF_Monitor_thread_pre_create_event);
/* Stop sampling prior to real thread_create. */
if (tls->sampling_status != CBTF_Monitor_Paused) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_thread_pre_create PAUSE SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_thread_pre_create_event,CBTF_Monitor_Paused);
} else if (tls->sampling_status == CBTF_Monitor_Paused) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_thread_pre_create ALREADY PAUSED\n",
thePid,monitor_get_thread_num());
}
}
return (NULL);
}
void
monitor_thread_post_create(void* data)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] Entered cbtf monitor_thread_post_create callback sampling_status:%d\n",
thePid,monitor_get_thread_num(),tls->sampling_status);
}
if (tls->sampling_status != CBTF_Monitor_Resumed) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_thread_post_create RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_thread_post_create_event,CBTF_Monitor_Resumed);
} else if (tls->sampling_status == CBTF_Monitor_Paused) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_thread_pre_create ALREADY RESUMED\n",
thePid,monitor_get_thread_num());
}
}
}
/**
* The OpenMP runtime invokes this callback when a thread starts to idle
* outside a parallel region. The callback executes in the environment
* of the idling thread. ompt_scope_begin.
* The OpenMP runtime invokes this callback when a thread finishes idling
* outside a parallel region. The callback executes in the environment of
* the thread that is about to resume useful work. ompt_scope_end.
*/
void CBTF_ompt_callback_idle(ompt_scope_endpoint_t endpoint)
{
switch(endpoint)
{
case ompt_scope_begin:
{
TLS_update_ompt_idle_begin(CBTF_GetTime());
#ifndef NDEBUG
if (cbtf_ompt_debug_blame) {
fprintf(stderr,"[%d,%d] CBTF_ompt_callback_idle ompt_scope_begin context:%lx\n",
getpid(),monitor_get_thread_num(), current_region_context);
}
#endif
}
break;
case ompt_scope_end:
{
#ifndef NDEBUG
if (cbtf_ompt_debug_blame) {
fprintf(stderr,"[%d,%d] CBTF_ompt_callback_idle ompt_scope_end context:%lx\n",
getpid(),monitor_get_thread_num(), current_region_context);
}
#endif
TLS_update_ompt_idle_totals(CBTF_GetTime());
}
break;
}
}
/*
* callbacks for handling of PROCESS
*/
void monitor_fini_process(int how, void *data)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
/* An assert here can cause libmonitor to hang with mpt.
* We may have forked a process which then did not call
* monitor_init_process and allocate TLS.
*/
if (tls == NULL) {
fprintf(stderr,"Warning. monitor_fini_process called with no TLS.\n");
return;
}
if (tls->sampling_status == CBTF_Monitor_Finished) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_process ALREADY FINISHED\n",thePid,monitor_get_thread_num());
}
return;
}
if (tls->sampling_status == CBTF_Monitor_Not_Started) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_process SAMPLING NOT STARTED\n",thePid,monitor_get_thread_num());
}
return;
}
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_process FINISHED SAMPLING\n",
thePid,monitor_get_thread_num());
}
static int f = 0;
if (f > 0)
raise(SIGSEGV);
f++;
/* collector stop_sampling does not use the arguments param */
tls->sampling_status = CBTF_Monitor_Finished;
if(how == MONITOR_EXIT_EXEC) {
tls->process_is_terminating = 1;
cbtf_offline_stop_sampling(NULL, true);
} else {
tls->process_is_terminating = 1;
cbtf_offline_stop_sampling(NULL, true);
}
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_fini_process process_is_terminating %d\n",
thePid,monitor_get_thread_num(),tls->process_is_terminating);
}
// The following call handles mrnet specific activity.
cbtf_offline_notify_event(CBTF_Monitor_fini_process_event);
}
/**
* The OpenMP runtime system invokes this callback, after an implicit task
* is fully initialized but before the task executes its work.
* This callback executes in the context of the new implicit task.
* We overide the callstack's first frame with the context provided
* by the region this task is doing work in.
* Without this context, call stack is at __kmp_invoke_task_func.
*/
void
CBTF_ompt_callback_implicit_task(
ompt_scope_endpoint_t endpoint,
ompt_data_t *parallel_data,
ompt_data_t *task_data,
unsigned int team_size,
unsigned int thread_num)
{
switch(endpoint) {
case ompt_scope_begin:
{
if(task_data->ptr) {
fprintf(stderr, "%s\n", "0: task_data initially not null");
}
#ifndef NDEBUG
if (cbtf_ompt_debug_blame) {
fprintf(stderr,"[%d,%d] CBTF_ompt_implicit_task ompt_scope_begin thread_num:%u\n",
getpid(),monitor_get_thread_num(), thread_num);
}
#endif
task_data->value = ompt_get_unique_id();
#ifndef NDEBUG
if (cbtf_ompt_debug_trace) {
fprintf(stderr,"%" PRIu64 ": ompt_event_implicit_task_begin: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32 "\n", ompt_get_thread_data()->value, parallel_data->value, task_data->value, team_size, thread_num);
}
#endif
TLS_update_ompt_task_begin(CBTF_GetTime());
// Find the parallel region this task is running under.
// Aquire it's parallel context and use it here.
//CBTF_omptp_region r = CBTF_omptp_region_with_id(parallel_data->value);
}
break;
case ompt_scope_end:
{
#ifndef NDEBUG
if (cbtf_ompt_debug_blame) {
fprintf(stderr,"[%d,%d] CBTF_ompt_implicit_task ompt_scope_end context:%u\n",
getpid(),monitor_get_thread_num(), thread_num);
}
#endif
#ifndef NDEBUG
if (cbtf_ompt_debug_trace) {
fprintf(stderr,"%" PRIu64 ": ompt_event_implicit_task_end: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32 "\n",
ompt_get_thread_data()->value, (parallel_data)?parallel_data->value:0, task_data->value, team_size, thread_num);
}
#endif
TLS_update_ompt_task_totals(CBTF_GetTime());
}
break;
}
}
/*
* callbacks for handling of DLOPEN/DLCLOSE.
*/
void monitor_dlopen(const char *library, int flags, void *handle)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
// FIXME: sampling_status is an enum.
if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) {
return;
}
if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlopen returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return;
}
if (library == NULL) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlopen ignores null library name\n",thePid,monitor_get_thread_num());
}
return;
}
/* TODO:
* if CBTF_GetDLInfo does not handle errors do so here.
*/
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlopen called with %s\n",
thePid,monitor_get_thread_num(),library);
}
tls->cbtf_dllist_curr = (cbtf_dlinfoList*)calloc(1,sizeof(cbtf_dlinfoList));
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time = CBTF_GetTime();
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time = CBTF_GetTime() + 1;
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name = strdup(library);
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.handle = handle;
tls->cbtf_dllist_curr->cbtf_dlinfo_next = tls->cbtf_dllist_head;
tls->cbtf_dllist_head = tls->cbtf_dllist_curr;
if ((tls->sampling_status == CBTF_Monitor_Paused) && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlopen RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_dlopen_event,CBTF_Monitor_Resumed);
}
}
void
monitor_post_dlclose(void *handle, int ret)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
// FIXME: sampling_status is an enum.
if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) {
return;
}
if (tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_post_dlclose returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return;
}
if (!tls->thread_is_terminating || !tls->process_is_terminating) {
if (tls->sampling_status == CBTF_Monitor_Paused && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_post_dlclose RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_post_dlclose_event,CBTF_Monitor_Resumed);
}
}
}
static int
ignore_this_thread()
{
if (ignore_thread == THREAD_DOINIT) {
ignore_thread = THREAD_SAMPLING;
char *string = getenv("HPCRUN_IGNORE_THREAD");
if (string) {
// eliminate special cases by adding comma delimiters at front and back
char all_str[1024];
sprintf(all_str, ",%s,", string);
int myid = monitor_get_thread_num();
char myid_str[20];
sprintf(myid_str, ",%d,", myid);
if (strstr(all_str, myid_str)) {
ignore_thread = THREAD_NOSAMPLING;
TMSG(IGNORE, "Thread %d ignore sampling", myid);
}
}
}
return ignore_thread == THREAD_NOSAMPLING;
}
void monitor_post_fork(pid_t child, void *data)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_post_fork returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return;
}
/* Resume/start sampling forked process. */
if (tls->sampling_status == CBTF_Monitor_Paused ||
tls->sampling_status == CBTF_Monitor_Finished) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_post_fork RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->CBTF_monitor_type = CBTF_Monitor_Proc;
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_post_fork_event,CBTF_Monitor_Resumed);
}
}
void CBTF_MRNet_Send(const int tag, const xdrproc_t xdrproc, const void* data)
{
unsigned size=0,dm_size=0;
char* dm_contents = NULL;
for(size = 1024; dm_contents == NULL; size *= 2) {
char* buffer = alloca(size);
XDR xdrs;
xdrmem_create(&xdrs, buffer, size, XDR_ENCODE);
if ((*xdrproc)(&xdrs, (void*)data) == TRUE) {
dm_size = xdr_getpos(&xdrs);
dm_contents = buffer;
}
xdr_destroy(&xdrs);
}
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr,"[%d,%d] CBTF_MRNet_Send: sends message tag:%d size: %d\n",
getpid(),monitor_get_thread_num(),tag ,dm_size);
}
#endif
CBTF_MRNet_LW_sendToFrontend(tag ,dm_size , (void *) dm_contents);
}
void ompt_finalize(ompt_data_t *tool_data)
{
#ifndef NDEBUG
if (cbtf_ompt_debug) {
fprintf(stderr,"[%d,%d] ompt_event_runtime_shutdown\n",getpid(),monitor_get_thread_num());
}
#endif
}
// ompt_event_MAY_ALWAYS
// shut down of ompt ...
void CBTF_ompt_callback_runtime_shutdown()
{
#ifndef NDEBUG
if (cbtf_ompt_debug_details) {
fprintf(stderr, "[%d,%d] CBTF_ompt_callback_runtime_shutdown:\n",getpid(),monitor_get_thread_num());
}
#endif
}
// ompt_event_MAY_ALWAYS
void CBTF_ompt_callback_control(uint64_t command, uint64_t modifier)
{
#ifndef NDEBUG
if (cbtf_ompt_debug_details) {
fprintf(stderr,"[%d,%d] CBTF_ompt_callback_control: %lu, %lu\n",getpid(),monitor_get_thread_num(), command, modifier);
}
#endif
}
void
monitor_pre_dlopen(const char *path, int flags)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
// FIXME: sampling_status is an enum.
if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) {
return;
}
if (tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_dlopen returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return;
}
if (path == NULL) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_dlopen ignores null path\n",thePid,monitor_get_thread_num());
}
return;
}
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_dlopen %s\n",thePid,monitor_get_thread_num(),path);
}
if ((tls->sampling_status == CBTF_Monitor_Started ||
tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_dlopen PAUSE SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_pre_dlopen_event,CBTF_Monitor_Paused);
}
}
// ompt_event_MAY_ALWAYS
// monitor notifies us of this as well...
void CBTF_ompt_callback_thread_end()
{
#ifndef NDEBUG
if (cbtf_ompt_debug) {
fprintf(stderr, "[%d,%d] ompt_callback_thread_end\n",getpid(),monitor_get_thread_num());
}
#endif
set_ompt_thread_finished(true);
cbtf_timer_service_stop_sampling(NULL);
set_ompt_flag(false);
}
// ompt_event_MAY_ALWAYS
// monitor notifies us of this...
// pass omp thread num to collector.
void CBTF_ompt_callback_thread_begin()
{
#ifndef NDEBUG
if (cbtf_ompt_debug) {
fprintf(stderr, "[%d,%d] CBTF_ompt_callback_thread_begin:%ld\n",getpid(),monitor_get_thread_num(),ompt_get_thread_data()->value);
}
#endif
cbtf_collector_set_openmp_threadid(omp_get_thread_num());
set_ompt_thread_finished(false);
set_ompt_flag(true);
}
// ompt_event_MAY_ALWAYS
void CBTF_ompt_callback_parallel_end(
ompt_data_t *parallel_data,
ompt_data_t *task_data,
ompt_invoker_t invoker,
const void *codeptr_ra)
{
#ifndef NDEBUG
if (cbtf_ompt_debug) {
fprintf(stderr,
"[Overview %d,%d] CBTF_ompt_callback_parallel_end parallelID:%lu parent_taskID:%lu invoker:%d context:%lx\n",
getpid(),monitor_get_thread_num(), parallel_data->value, task_data->value, invoker, current_region_context);
}
#endif
current_region_context = (uint64_t)NULL;
CBTF_omptp_region_clear(parallel_data->value);
}
void monitor_mpi_post_fini(void)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_fini CALLED\n",
thePid,monitor_get_thread_num());
}
cbtf_offline_notify_event(CBTF_Monitor_MPI_post_fini_event);
/* No special pause resume here. */
}
void CBTF_Timer(uint64_t interval, const CBTF_TimerEventHandler handler)
{
/* Create and/or access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
if(tls == NULL) {
tls = malloc(sizeof(TLS));
Assert(tls != NULL);
CBTF_SetTLS(TLSKey, tls);
}
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if(!init_timer_signal) {
CBTF_SetTimerSignal();
}
#ifdef HAVE_POSIX_TIMERS
/* The whole timer API was predicated on itmer and sigprof
* which was process wide timer shared by threads. With posix timers
* we have an extra step of initializing the thread specific timers.
* But the long standing API used by CBTF_Timer was to simply
* use interval 0 and handler NULL to imply removal of a timer.
* Initialize posix_timer_initialized to false when CBTF_Timer is called
* with a non-zero interval and a non-NULL handler.
* The assumption being that all users of this API are starting a new
* timer when called with a non-zero interval and a non-NULL handler.
* Therefore the need to reinit posix_timer_initialized to false to statisfy
* the previous API usage. This assumes that a caller has called this function
* with an interval of 0 and a NULL handler to force deleting any
* previous timer.
*/
#if 0
fprintf(stderr,"[%d,%d] CBTF_Timer enterd interval:%ld posix_timer_initialized:%d handler:%p\n",
getpid(),monitor_get_thread_num(),interval,tls->posix_timer_initialized, handler);
#endif
if (interval > 0 && handler != NULL) {
tls->posix_timer_initialized = false;
}
#endif
__CBTF_Timer(interval, handler);
}
// Writers always wait until they acquire the lock. Now allow writers
// to lock against themselves, but only in the same thread.
static void
hpcrun_dlopen_write_lock(void)
{
int tid = monitor_get_thread_num();
int acquire = 0;
do {
spinlock_lock(&dlopen_lock);
if (dlopen_num_writers == 0 || tid == dlopen_writer_tid) {
dlopen_num_writers++;
dlopen_writer_tid = tid;
acquire = 1;
}
spinlock_unlock(&dlopen_lock);
} while (! acquire);
// Wait for any readers to finish.
if (! ENABLED(DLOPEN_RISKY)) {
while (dlopen_num_readers > 0) ;
}
}
void CBTF_MRNet_LW_sendToFrontend(const int tag, const int size, void *data)
{
const char* fmt_str = "%auc";
/*
* No need to repeatedly check mrnet_connected within a while loop as is
* typical with a condition variable because it is only ever set once by
* CBTF_MRNet_LW_connect() above.
*/
pthread_mutex_lock(&mrnet_connected_mutex);
if (!mrnet_connected)
{
pthread_cond_wait(&mrnet_connected_cond, &mrnet_connected_mutex);
}
pthread_mutex_unlock(&mrnet_connected_mutex);
#if defined(ENABLE_CBTF_MRNET_PLAYBACK)
if (playback_intercept(tag, (uint32_t)size, data))
{
return; /* Squelch the actual sending of this message. */
}
#endif
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr,"[%d,%d] CBTF_MRNet_LW_sendToFrontend: sends message with tag %d\n",
getpid(),monitor_get_thread_num(),tag);
}
#endif
Stream_t* CBTF_MRNet_stream = Network_get_Stream(CBTF_MRNet_netPtr,stream_id);
if ( (Stream_send(CBTF_MRNet_stream, tag, fmt_str, data, size) == -1) ||
Stream_flush(CBTF_MRNet_stream) == -1 ) {
fprintf(stderr, "BE: stream::send() failure\n");
}
fflush(stdout);
fflush(stderr);
}
/*
* callbacks for handling of FORK.
* NOTE that this callback can return a void pointer if desired.
*/
void * monitor_pre_fork(void)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
/* The sgi MPT mpi startup apparently can fork
* a process such that monitor does not go
* through its normal path and our tls is not
* allocated as we expect.
*/
TLS* tls = CBTF_GetTLS(TLSKey);
if (tls == NULL) {
tls = malloc(sizeof(TLS));
}
Assert(tls != NULL);
CBTF_SetTLS(TLSKey, tls);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_fork returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return (NULL);
}
/* Stop sampling prior to real fork. */
if (tls->sampling_status == CBTF_Monitor_Paused ||
tls->sampling_status == CBTF_Monitor_Started) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_pre_fork PAUSE SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_pre_fork_event,CBTF_Monitor_Paused);
}
return (NULL);
}
// ompt_event_MAY_ALWAYS
// record start time and get address or callstack of this context
// This seems to only be active in the master thread.
// increment is parallel flag. This flag indicates that a
// parallel region is active to other callbacks that may be
// interested.
void CBTF_ompt_callback_parallel_begin(
ompt_data_t *parent_task_data,
const ompt_frame_t *parent_taskframe,
ompt_data_t* parallel_data,
uint32_t requested_team_size,
ompt_invoker_t invoker,
const void *parallel_function)
{
/* Regions are only tracked on the master thread. We will create a
* callstack context here that is used by all the worker threads.
* The implicit_task callbasks will record the original context
* from the master.
*
* Handle nested regions via regionMap.
*/
current_region_context = (uint64_t)parallel_function;
uint64_t stacktrace[MaxFramesPerStackTrace];
unsigned stacktrace_size = 0;
//CBTF_overview_omptp_event event;
//omptp_start_event(&event,(uint64_t)parallel_function, stacktrace, &stacktrace_size);
//CBTF_overview_omptp_event event;
//TLS_start_ompt_event(&event);
//event.type = CBTF_OMPT_REGION;
//event.time = 0;
parallel_data->value = ompt_get_unique_id();
int nest_level = CBTF_omptp_region_add(parallel_data->value, stacktrace, stacktrace_size);
#ifndef NDEBUG
if (cbtf_ompt_debug) {
fprintf(stderr,
"[%d,%d] CBTF_ompt_callback_parallel_begin parallelID:%lu parent_taskID:%lu req_team_size:%u invoker:%d context:%p nest_level=%d\n",
getpid(),monitor_get_thread_num(), parallel_data->value, parent_task_data->value, requested_team_size, invoker, parallel_function, nest_level);
}
#endif
}
// This callback should is only needed during mpi startup and mrnet connection.
// Once connected, this should just return as early as possible.
// Since we do not really want mrnet specific info here, we rely on the fact
// that we only use this callback the first time mpi_comm_rank is called
// so that the underlying common collector code can aquire the rank which
// is need for mrnet connections. Therefore we should never do anything
// except in the first call.
void monitor_mpi_post_comm_rank(void)
{
/* We do not even want to aquire the tls if mpi_comm_rank has
* been called already. */
if (init_mpi_comm_rank) {
return;
}
init_mpi_comm_rank = true;
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
#ifndef NDEBUG
char* statusstr = "UNKNOWNSTATUS";
if (IsMonitorDebugEnabled) {
switch(tls->sampling_status) {
case CBTF_Monitor_Resumed:
statusstr = "RESUME";
break;
case CBTF_Monitor_Paused:
statusstr = "PAUSE";
break;
case CBTF_Monitor_Started:
statusstr = "STARTED";
break;
case CBTF_Monitor_Not_Started:
statusstr = "NOTSTARTED";
break;
case CBTF_Monitor_Finished:
statusstr = "FINISHED";
break;
}
}
#endif
#ifndef NDEBUG
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank sampling_status:%s\n",
thePid,monitor_get_thread_num(),statusstr);
}
#endif
// NOTE: For post_comm_rank, we always need to resume since
// that is where mrnet connection is made. Therefore that
// resume should not resume if we are not start enabled.
bool resume_sampling = false;
// FIXME: commented out if test remove?
//if (tls->sampling_status == CBTF_Monitor_Paused) {
if (tls->mpi_pcontrol && tls->start_enabled) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING pcontrol start enabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
resume_sampling = true;
tls->sampling_status = CBTF_Monitor_Resumed;
} else if(tls->mpi_pcontrol && !tls->start_enabled) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING pcontrol start disabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
// FORCE THIS HERE FOR NOW.
resume_sampling = true;
// tell collector service that start was deferred
//cbtf_offline_service_start_deferred();
} else {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING enabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
resume_sampling = true;
tls->sampling_status = CBTF_Monitor_Resumed;
}
//}
if (resume_sampling) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
//tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_MPI_post_comm_rank_event,CBTF_Monitor_Resumed);
}
}
void
monitor_init_mpi(int *argc, char ***argv)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
bool resume_sampling = false;
if (tls->sampling_status == CBTF_Monitor_Paused) {
if (tls->mpi_pcontrol && tls->start_enabled) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING pcontrol start enabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
resume_sampling = true;
} else if(tls->mpi_pcontrol && !tls->start_enabled) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING pcontrol start disabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
} else {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING enabled rank:%d\n",
thePid,monitor_get_thread_num(), monitor_mpi_comm_rank());
}
resume_sampling = true;
}
}
cbtf_offline_notify_event(CBTF_Monitor_MPI_init_event);
if (resume_sampling) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_mpi RESUME SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_MPI_init_event,CBTF_Monitor_Resumed);
} else {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_mpi is SAMPLING\n",
thePid,monitor_get_thread_num());
}
// this is the case where start collection was disabled util at
// the first (if any) mpi_pcontrol(1) is encountered.
// We need to make this available to any code downstream that
// may temporarily disable the collector via cbtf_offline_service_stop_timer
// and later reenable the collector via cbtf_offline_service_start_timer.
// Both of those live in services/collector/collector.c.
// the disable and reenable is used internaly in collector.c to
// protect one CBTF_MRNet_Send of attached threads from being
// sampled. The other use is in the services/collector/monitor.c
// code where we need to disable,reenable for similar reasons.
// Could check tls->sampling_status for this from both
// cbtf_offline_service_stop_timer and cbtf_offline_service_start_timer.
// Provide this as util function that returns this value.
}
tls->in_mpi_pre_init = false;
CBTF_in_mpi_startup = false;
}
/* monitor_mpi_pcontrol is reponsible for starting and stopping data
* collection based on the user coding:
*
* MPI_Pcontrol(0) to disable collection
* and
* MPI_Pcontrol(1) to reenable data collection
*
*/
void monitor_mpi_pcontrol(int level)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
if ( tls->mpi_pcontrol ) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_mpi_pcontrol CALLED level:%d\n", thePid,monitor_get_thread_num(),level);
}
if (level == 0) {
if ((tls->sampling_status == CBTF_Monitor_Started ||
tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) {
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_mpi_pcontrol_event,CBTF_Monitor_Paused);
if (IsMonitorDebugEnabled) {
fprintf(stderr,"monitor_mpi_pcontrol level 0 collector started, PAUSE SAMPLING %d,%d\n", thePid,monitor_get_thread_num());
}
}
} else if (level == 1) {
// this should not happen now.
if (tls->sampling_status == CBTF_Monitor_Not_Started ) {
// should handle mrnet connection for cbtf-mrnet case.
cbtf_offline_notify_event(CBTF_Monitor_mpi_pcontrol_event);
tls->CBTF_monitor_type = CBTF_Monitor_Proc;
tls->sampling_status = CBTF_Monitor_Started;
cbtf_offline_start_sampling(NULL);
} else if (tls->sampling_status == CBTF_Monitor_Paused && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"monitor_mpi_pcontrol level 1 collector started RESUME SAMPLING %d,%d\n", thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Resumed;
cbtf_offline_sampling_status(CBTF_Monitor_mpi_pcontrol_event,CBTF_Monitor_Resumed);
} else if (tls->sampling_status == CBTF_Monitor_Resumed && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"monitor_mpi_pcontrol level 1 collector ALREADY RESUMED %d,%d\n", thePid,monitor_get_thread_num());
}
}
} else {
fprintf(stderr,"monitor_mpi_pcontrol CALLED with unsupported level=%d\n", level);
}
} else {
/* early return - do not honor mpi_pcontrol */
if (IsMonitorDebugEnabled) {
fprintf(stderr,"monitor_mpi_pcontrol ENABLE_MPI_PCONTROL **NOT** SET IGNORING MPI_PCONTROL CALL %d,%d\n", thePid,monitor_get_thread_num());
}
return;
}
}
void *monitor_init_thread(int tid, void *data)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = malloc(sizeof(TLS));
Assert(tls != NULL);
CBTF_SetTLS(TLSKey, tls);
#else
TLS* tls = &the_tls;
#endif
Assert(tls != NULL);
thePid = getpid();
#if 0
if (monitor_is_threaded()) {
monitor_get_thread_num() = monitor_get_thread_num();
} else {
monitor_get_thread_num() = 0;
}
#endif
short debug_mpi_pcontrol=0;
if ( (getenv("CBTF_DEBUG_MPI_PCONTROL") != NULL)) {
debug_mpi_pcontrol=1;
}
tls->collector_started = false;
tls->cbtf_dllist_head = NULL;
tls->cbtf_dllist_curr = NULL;
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_thread BEGIN SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->in_mpi_pre_init = false;
tls->CBTF_monitor_type = CBTF_Monitor_Thread;
tls->mpi_pcontrol = 0, tls->start_enabled = 0;
if (getenv("OPENSS_ENABLE_MPI_PCONTROL") != NULL) tls->mpi_pcontrol = 1;
if (getenv("OPENSS_START_ENABLED") != NULL) tls->start_enabled = 1;
if (getenv("CBTF_ENABLE_MPI_PCONTROL") != NULL) tls->mpi_pcontrol = 1;
if (getenv("CBTF_START_ENABLED") != NULL) tls->start_enabled = 1;
/*
* Always start the collector via cbtf_offline_start_sampling.
* In the presence of active mpi_pcontrol and start with the
* collector disabled, we issue a cbtf_offline_sampling_status paused call.
*/
#if 0
if (CBTF_in_mpi_startup) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_thread IN MPI STARTUP - DEFER SAMPLING\n",
thePid,monitor_get_thread_num());
}
return (data);
}
#endif
tls->collector_started = true;
tls->sampling_status = CBTF_Monitor_Started;
cbtf_offline_start_sampling(NULL);
cbtf_offline_notify_event(CBTF_Monitor_init_thread_event);
if ( tls->mpi_pcontrol && !tls->start_enabled) {
if (debug_mpi_pcontrol) {
fprintf(stderr,"[%d,%d] monitor_init_thread CBTF_START_ENABLED NOT SET. Defer sampling at start-up time.\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_init_thread_event,CBTF_Monitor_Paused);
} else if ( tls->mpi_pcontrol && tls->start_enabled) {
if (debug_mpi_pcontrol) {
fprintf(stderr,"[%d,%d] monitor_init_thread CBTF_START_ENABLED SET. START SAMPLING\n",
thePid,monitor_get_thread_num());
}
} else {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_init_thread START SAMPLING\n",
thePid,monitor_get_thread_num());
}
}
return(data);
}
void
monitor_dlclose(void *handle)
{
/* Access our thread-local storage */
#ifdef USE_EXPLICIT_TLS
TLS* tls = CBTF_GetTLS(TLSKey);
#else
TLS* tls = &the_tls;
#endif
// FIXME: sampling_status is an enum.
if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) {
return;
}
if (tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlclose returns early due to in mpi init\n",thePid,monitor_get_thread_num());
}
return;
}
while (tls->cbtf_dllist_curr) {
if (tls->cbtf_dllist_curr->cbtf_dlinfo_entry.handle == handle) {
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time = CBTF_GetTime();
if (IsMonitorDebugEnabled) {
fprintf(stderr,"FOUND %p %s\n",handle, tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name);
fprintf(stderr,"loaded at %ld, unloaded at %ld\n",
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time,
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time);
}
/* On some systems (NASA) it appears that dlopen can be called
* before monitor_init_process (or even monitor_early_init).
* So we need to use getpid() directly here.
*/
/* FIXME: Handle return value? */
int retval = CBTF_GetDLInfo(getpid(),
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name,
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time,
tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time
);
if (retval) {
}
break;
}
tls->cbtf_dllist_curr = tls->cbtf_dllist_curr->cbtf_dlinfo_next;
}
if (!tls->thread_is_terminating || !tls->process_is_terminating) {
if ((tls->sampling_status == CBTF_Monitor_Started ||
tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) {
if (IsMonitorDebugEnabled) {
fprintf(stderr,"[%d,%d] monitor_dlclose PAUSE SAMPLING\n",
thePid,monitor_get_thread_num());
}
tls->sampling_status = CBTF_Monitor_Paused;
cbtf_offline_sampling_status(CBTF_Monitor_dlclose_event,CBTF_Monitor_Paused);
}
}
}
