void monitor_mpi_pre_init(void) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); tls->in_mpi_pre_init = true; CBTF_in_mpi_startup = true; init_mpi_comm_rank = false; cbtf_offline_notify_event(CBTF_Monitor_MPI_pre_init_event); if (tls->sampling_status == CBTF_Monitor_Started) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_pre_init PAUSE SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_MPI_pre_init_event,CBTF_Monitor_Paused); } else { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_pre_init IS PAUSED\n", thePid,monitor_get_thread_num()); } } }
/* * callbacks for handling of THREADS */ void monitor_fini_thread(void *ptr) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if (!tls->collector_started) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_thread collector never started.\n", thePid,monitor_get_thread_num()); } return; } if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_thread STOP SAMPLING\n", thePid,monitor_get_thread_num()); } cbtf_offline_stop_sampling(NULL,true); tls->sampling_status = CBTF_Monitor_Finished; tls->thread_is_terminating = 1; // The following call will handle any mrnet specific activity. cbtf_offline_notify_event(CBTF_Monitor_fini_thread_event); }
void* monitor_thread_pre_create(void) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); cbtf_offline_notify_event(CBTF_Monitor_thread_pre_create_event); /* Stop sampling prior to real thread_create. */ if (tls->sampling_status != CBTF_Monitor_Paused) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_thread_pre_create PAUSE SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_thread_pre_create_event,CBTF_Monitor_Paused); } else if (tls->sampling_status == CBTF_Monitor_Paused) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_thread_pre_create ALREADY PAUSED\n", thePid,monitor_get_thread_num()); } } return (NULL); }
void monitor_thread_post_create(void* data) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] Entered cbtf monitor_thread_post_create callback sampling_status:%d\n", thePid,monitor_get_thread_num(),tls->sampling_status); } if (tls->sampling_status != CBTF_Monitor_Resumed) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_thread_post_create RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_thread_post_create_event,CBTF_Monitor_Resumed); } else if (tls->sampling_status == CBTF_Monitor_Paused) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_thread_pre_create ALREADY RESUMED\n", thePid,monitor_get_thread_num()); } } }
/** * The OpenMP runtime invokes this callback when a thread starts to idle * outside a parallel region. The callback executes in the environment * of the idling thread. ompt_scope_begin. * The OpenMP runtime invokes this callback when a thread finishes idling * outside a parallel region. The callback executes in the environment of * the thread that is about to resume useful work. ompt_scope_end. */ void CBTF_ompt_callback_idle(ompt_scope_endpoint_t endpoint) { switch(endpoint) { case ompt_scope_begin: { TLS_update_ompt_idle_begin(CBTF_GetTime()); #ifndef NDEBUG if (cbtf_ompt_debug_blame) { fprintf(stderr,"[%d,%d] CBTF_ompt_callback_idle ompt_scope_begin context:%lx\n", getpid(),monitor_get_thread_num(), current_region_context); } #endif } break; case ompt_scope_end: { #ifndef NDEBUG if (cbtf_ompt_debug_blame) { fprintf(stderr,"[%d,%d] CBTF_ompt_callback_idle ompt_scope_end context:%lx\n", getpid(),monitor_get_thread_num(), current_region_context); } #endif TLS_update_ompt_idle_totals(CBTF_GetTime()); } break; } }
/* * callbacks for handling of PROCESS */ void monitor_fini_process(int how, void *data) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif /* An assert here can cause libmonitor to hang with mpt. * We may have forked a process which then did not call * monitor_init_process and allocate TLS. */ if (tls == NULL) { fprintf(stderr,"Warning. monitor_fini_process called with no TLS.\n"); return; } if (tls->sampling_status == CBTF_Monitor_Finished) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_process ALREADY FINISHED\n",thePid,monitor_get_thread_num()); } return; } if (tls->sampling_status == CBTF_Monitor_Not_Started) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_process SAMPLING NOT STARTED\n",thePid,monitor_get_thread_num()); } return; } if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_process FINISHED SAMPLING\n", thePid,monitor_get_thread_num()); } static int f = 0; if (f > 0) raise(SIGSEGV); f++; /* collector stop_sampling does not use the arguments param */ tls->sampling_status = CBTF_Monitor_Finished; if(how == MONITOR_EXIT_EXEC) { tls->process_is_terminating = 1; cbtf_offline_stop_sampling(NULL, true); } else { tls->process_is_terminating = 1; cbtf_offline_stop_sampling(NULL, true); } if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_fini_process process_is_terminating %d\n", thePid,monitor_get_thread_num(),tls->process_is_terminating); } // The following call handles mrnet specific activity. cbtf_offline_notify_event(CBTF_Monitor_fini_process_event); }
/** * The OpenMP runtime system invokes this callback, after an implicit task * is fully initialized but before the task executes its work. * This callback executes in the context of the new implicit task. * We overide the callstack's first frame with the context provided * by the region this task is doing work in. * Without this context, call stack is at __kmp_invoke_task_func. */ void CBTF_ompt_callback_implicit_task( ompt_scope_endpoint_t endpoint, ompt_data_t *parallel_data, ompt_data_t *task_data, unsigned int team_size, unsigned int thread_num) { switch(endpoint) { case ompt_scope_begin: { if(task_data->ptr) { fprintf(stderr, "%s\n", "0: task_data initially not null"); } #ifndef NDEBUG if (cbtf_ompt_debug_blame) { fprintf(stderr,"[%d,%d] CBTF_ompt_implicit_task ompt_scope_begin thread_num:%u\n", getpid(),monitor_get_thread_num(), thread_num); } #endif task_data->value = ompt_get_unique_id(); #ifndef NDEBUG if (cbtf_ompt_debug_trace) { fprintf(stderr,"%" PRIu64 ": ompt_event_implicit_task_begin: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32 "\n", ompt_get_thread_data()->value, parallel_data->value, task_data->value, team_size, thread_num); } #endif TLS_update_ompt_task_begin(CBTF_GetTime()); // Find the parallel region this task is running under. // Aquire it's parallel context and use it here. //CBTF_omptp_region r = CBTF_omptp_region_with_id(parallel_data->value); } break; case ompt_scope_end: { #ifndef NDEBUG if (cbtf_ompt_debug_blame) { fprintf(stderr,"[%d,%d] CBTF_ompt_implicit_task ompt_scope_end context:%u\n", getpid(),monitor_get_thread_num(), thread_num); } #endif #ifndef NDEBUG if (cbtf_ompt_debug_trace) { fprintf(stderr,"%" PRIu64 ": ompt_event_implicit_task_end: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32 "\n", ompt_get_thread_data()->value, (parallel_data)?parallel_data->value:0, task_data->value, team_size, thread_num); } #endif TLS_update_ompt_task_totals(CBTF_GetTime()); } break; } }
/* * callbacks for handling of DLOPEN/DLCLOSE. */ void monitor_dlopen(const char *library, int flags, void *handle) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif // FIXME: sampling_status is an enum. if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) { return; } if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlopen returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return; } if (library == NULL) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlopen ignores null library name\n",thePid,monitor_get_thread_num()); } return; } /* TODO: * if CBTF_GetDLInfo does not handle errors do so here. */ if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlopen called with %s\n", thePid,monitor_get_thread_num(),library); } tls->cbtf_dllist_curr = (cbtf_dlinfoList*)calloc(1,sizeof(cbtf_dlinfoList)); tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time = CBTF_GetTime(); tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time = CBTF_GetTime() + 1; tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name = strdup(library); tls->cbtf_dllist_curr->cbtf_dlinfo_entry.handle = handle; tls->cbtf_dllist_curr->cbtf_dlinfo_next = tls->cbtf_dllist_head; tls->cbtf_dllist_head = tls->cbtf_dllist_curr; if ((tls->sampling_status == CBTF_Monitor_Paused) && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlopen RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_dlopen_event,CBTF_Monitor_Resumed); } }
void monitor_post_dlclose(void *handle, int ret) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif // FIXME: sampling_status is an enum. if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) { return; } if (tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_post_dlclose returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return; } if (!tls->thread_is_terminating || !tls->process_is_terminating) { if (tls->sampling_status == CBTF_Monitor_Paused && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_post_dlclose RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_post_dlclose_event,CBTF_Monitor_Resumed); } } }
static int ignore_this_thread() { if (ignore_thread == THREAD_DOINIT) { ignore_thread = THREAD_SAMPLING; char *string = getenv("HPCRUN_IGNORE_THREAD"); if (string) { // eliminate special cases by adding comma delimiters at front and back char all_str[1024]; sprintf(all_str, ",%s,", string); int myid = monitor_get_thread_num(); char myid_str[20]; sprintf(myid_str, ",%d,", myid); if (strstr(all_str, myid_str)) { ignore_thread = THREAD_NOSAMPLING; TMSG(IGNORE, "Thread %d ignore sampling", myid); } } } return ignore_thread == THREAD_NOSAMPLING; }
void monitor_post_fork(pid_t child, void *data) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_post_fork returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return; } /* Resume/start sampling forked process. */ if (tls->sampling_status == CBTF_Monitor_Paused || tls->sampling_status == CBTF_Monitor_Finished) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_post_fork RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } tls->CBTF_monitor_type = CBTF_Monitor_Proc; tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_post_fork_event,CBTF_Monitor_Resumed); } }
void CBTF_MRNet_Send(const int tag, const xdrproc_t xdrproc, const void* data) { unsigned size=0,dm_size=0; char* dm_contents = NULL; for(size = 1024; dm_contents == NULL; size *= 2) { char* buffer = alloca(size); XDR xdrs; xdrmem_create(&xdrs, buffer, size, XDR_ENCODE); if ((*xdrproc)(&xdrs, (void*)data) == TRUE) { dm_size = xdr_getpos(&xdrs); dm_contents = buffer; } xdr_destroy(&xdrs); } #ifndef NDEBUG if (IsMRNetDebugEnabled) { fprintf(stderr,"[%d,%d] CBTF_MRNet_Send: sends message tag:%d size: %d\n", getpid(),monitor_get_thread_num(),tag ,dm_size); } #endif CBTF_MRNet_LW_sendToFrontend(tag ,dm_size , (void *) dm_contents); }
void ompt_finalize(ompt_data_t *tool_data) { #ifndef NDEBUG if (cbtf_ompt_debug) { fprintf(stderr,"[%d,%d] ompt_event_runtime_shutdown\n",getpid(),monitor_get_thread_num()); } #endif }
// ompt_event_MAY_ALWAYS // shut down of ompt ... void CBTF_ompt_callback_runtime_shutdown() { #ifndef NDEBUG if (cbtf_ompt_debug_details) { fprintf(stderr, "[%d,%d] CBTF_ompt_callback_runtime_shutdown:\n",getpid(),monitor_get_thread_num()); } #endif }
// ompt_event_MAY_ALWAYS void CBTF_ompt_callback_control(uint64_t command, uint64_t modifier) { #ifndef NDEBUG if (cbtf_ompt_debug_details) { fprintf(stderr,"[%d,%d] CBTF_ompt_callback_control: %lu, %lu\n",getpid(),monitor_get_thread_num(), command, modifier); } #endif }
void monitor_pre_dlopen(const char *path, int flags) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif // FIXME: sampling_status is an enum. if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) { return; } if (tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_dlopen returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return; } if (path == NULL) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_dlopen ignores null path\n",thePid,monitor_get_thread_num()); } return; } if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_dlopen %s\n",thePid,monitor_get_thread_num(),path); } if ((tls->sampling_status == CBTF_Monitor_Started || tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_dlopen PAUSE SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_pre_dlopen_event,CBTF_Monitor_Paused); } }
// ompt_event_MAY_ALWAYS // monitor notifies us of this as well... void CBTF_ompt_callback_thread_end() { #ifndef NDEBUG if (cbtf_ompt_debug) { fprintf(stderr, "[%d,%d] ompt_callback_thread_end\n",getpid(),monitor_get_thread_num()); } #endif set_ompt_thread_finished(true); cbtf_timer_service_stop_sampling(NULL); set_ompt_flag(false); }
// ompt_event_MAY_ALWAYS // monitor notifies us of this... // pass omp thread num to collector. void CBTF_ompt_callback_thread_begin() { #ifndef NDEBUG if (cbtf_ompt_debug) { fprintf(stderr, "[%d,%d] CBTF_ompt_callback_thread_begin:%ld\n",getpid(),monitor_get_thread_num(),ompt_get_thread_data()->value); } #endif cbtf_collector_set_openmp_threadid(omp_get_thread_num()); set_ompt_thread_finished(false); set_ompt_flag(true); }
// ompt_event_MAY_ALWAYS void CBTF_ompt_callback_parallel_end( ompt_data_t *parallel_data, ompt_data_t *task_data, ompt_invoker_t invoker, const void *codeptr_ra) { #ifndef NDEBUG if (cbtf_ompt_debug) { fprintf(stderr, "[Overview %d,%d] CBTF_ompt_callback_parallel_end parallelID:%lu parent_taskID:%lu invoker:%d context:%lx\n", getpid(),monitor_get_thread_num(), parallel_data->value, task_data->value, invoker, current_region_context); } #endif current_region_context = (uint64_t)NULL; CBTF_omptp_region_clear(parallel_data->value); }
void monitor_mpi_post_fini(void) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_fini CALLED\n", thePid,monitor_get_thread_num()); } cbtf_offline_notify_event(CBTF_Monitor_MPI_post_fini_event); /* No special pause resume here. */ }
void CBTF_Timer(uint64_t interval, const CBTF_TimerEventHandler handler) { /* Create and/or access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); if(tls == NULL) { tls = malloc(sizeof(TLS)); Assert(tls != NULL); CBTF_SetTLS(TLSKey, tls); } #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if(!init_timer_signal) { CBTF_SetTimerSignal(); } #ifdef HAVE_POSIX_TIMERS /* The whole timer API was predicated on itmer and sigprof * which was process wide timer shared by threads. With posix timers * we have an extra step of initializing the thread specific timers. * But the long standing API used by CBTF_Timer was to simply * use interval 0 and handler NULL to imply removal of a timer. * Initialize posix_timer_initialized to false when CBTF_Timer is called * with a non-zero interval and a non-NULL handler. * The assumption being that all users of this API are starting a new * timer when called with a non-zero interval and a non-NULL handler. * Therefore the need to reinit posix_timer_initialized to false to statisfy * the previous API usage. This assumes that a caller has called this function * with an interval of 0 and a NULL handler to force deleting any * previous timer. */ #if 0 fprintf(stderr,"[%d,%d] CBTF_Timer enterd interval:%ld posix_timer_initialized:%d handler:%p\n", getpid(),monitor_get_thread_num(),interval,tls->posix_timer_initialized, handler); #endif if (interval > 0 && handler != NULL) { tls->posix_timer_initialized = false; } #endif __CBTF_Timer(interval, handler); }
// Writers always wait until they acquire the lock. Now allow writers // to lock against themselves, but only in the same thread. static void hpcrun_dlopen_write_lock(void) { int tid = monitor_get_thread_num(); int acquire = 0; do { spinlock_lock(&dlopen_lock); if (dlopen_num_writers == 0 || tid == dlopen_writer_tid) { dlopen_num_writers++; dlopen_writer_tid = tid; acquire = 1; } spinlock_unlock(&dlopen_lock); } while (! acquire); // Wait for any readers to finish. if (! ENABLED(DLOPEN_RISKY)) { while (dlopen_num_readers > 0) ; } }
void CBTF_MRNet_LW_sendToFrontend(const int tag, const int size, void *data) { const char* fmt_str = "%auc"; /* * No need to repeatedly check mrnet_connected within a while loop as is * typical with a condition variable because it is only ever set once by * CBTF_MRNet_LW_connect() above. */ pthread_mutex_lock(&mrnet_connected_mutex); if (!mrnet_connected) { pthread_cond_wait(&mrnet_connected_cond, &mrnet_connected_mutex); } pthread_mutex_unlock(&mrnet_connected_mutex); #if defined(ENABLE_CBTF_MRNET_PLAYBACK) if (playback_intercept(tag, (uint32_t)size, data)) { return; /* Squelch the actual sending of this message. */ } #endif #ifndef NDEBUG if (IsMRNetDebugEnabled) { fprintf(stderr,"[%d,%d] CBTF_MRNet_LW_sendToFrontend: sends message with tag %d\n", getpid(),monitor_get_thread_num(),tag); } #endif Stream_t* CBTF_MRNet_stream = Network_get_Stream(CBTF_MRNet_netPtr,stream_id); if ( (Stream_send(CBTF_MRNet_stream, tag, fmt_str, data, size) == -1) || Stream_flush(CBTF_MRNet_stream) == -1 ) { fprintf(stderr, "BE: stream::send() failure\n"); } fflush(stdout); fflush(stderr); }
/* * callbacks for handling of FORK. * NOTE that this callback can return a void pointer if desired. */ void * monitor_pre_fork(void) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS /* The sgi MPT mpi startup apparently can fork * a process such that monitor does not go * through its normal path and our tls is not * allocated as we expect. */ TLS* tls = CBTF_GetTLS(TLSKey); if (tls == NULL) { tls = malloc(sizeof(TLS)); } Assert(tls != NULL); CBTF_SetTLS(TLSKey, tls); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if (CBTF_in_mpi_startup || tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_fork returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return (NULL); } /* Stop sampling prior to real fork. */ if (tls->sampling_status == CBTF_Monitor_Paused || tls->sampling_status == CBTF_Monitor_Started) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_pre_fork PAUSE SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_pre_fork_event,CBTF_Monitor_Paused); } return (NULL); }
// ompt_event_MAY_ALWAYS // record start time and get address or callstack of this context // This seems to only be active in the master thread. // increment is parallel flag. This flag indicates that a // parallel region is active to other callbacks that may be // interested. void CBTF_ompt_callback_parallel_begin( ompt_data_t *parent_task_data, const ompt_frame_t *parent_taskframe, ompt_data_t* parallel_data, uint32_t requested_team_size, ompt_invoker_t invoker, const void *parallel_function) { /* Regions are only tracked on the master thread. We will create a * callstack context here that is used by all the worker threads. * The implicit_task callbasks will record the original context * from the master. * * Handle nested regions via regionMap. */ current_region_context = (uint64_t)parallel_function; uint64_t stacktrace[MaxFramesPerStackTrace]; unsigned stacktrace_size = 0; //CBTF_overview_omptp_event event; //omptp_start_event(&event,(uint64_t)parallel_function, stacktrace, &stacktrace_size); //CBTF_overview_omptp_event event; //TLS_start_ompt_event(&event); //event.type = CBTF_OMPT_REGION; //event.time = 0; parallel_data->value = ompt_get_unique_id(); int nest_level = CBTF_omptp_region_add(parallel_data->value, stacktrace, stacktrace_size); #ifndef NDEBUG if (cbtf_ompt_debug) { fprintf(stderr, "[%d,%d] CBTF_ompt_callback_parallel_begin parallelID:%lu parent_taskID:%lu req_team_size:%u invoker:%d context:%p nest_level=%d\n", getpid(),monitor_get_thread_num(), parallel_data->value, parent_task_data->value, requested_team_size, invoker, parallel_function, nest_level); } #endif }
// This callback should is only needed during mpi startup and mrnet connection. // Once connected, this should just return as early as possible. // Since we do not really want mrnet specific info here, we rely on the fact // that we only use this callback the first time mpi_comm_rank is called // so that the underlying common collector code can aquire the rank which // is need for mrnet connections. Therefore we should never do anything // except in the first call. void monitor_mpi_post_comm_rank(void) { /* We do not even want to aquire the tls if mpi_comm_rank has * been called already. */ if (init_mpi_comm_rank) { return; } init_mpi_comm_rank = true; /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); #ifndef NDEBUG char* statusstr = "UNKNOWNSTATUS"; if (IsMonitorDebugEnabled) { switch(tls->sampling_status) { case CBTF_Monitor_Resumed: statusstr = "RESUME"; break; case CBTF_Monitor_Paused: statusstr = "PAUSE"; break; case CBTF_Monitor_Started: statusstr = "STARTED"; break; case CBTF_Monitor_Not_Started: statusstr = "NOTSTARTED"; break; case CBTF_Monitor_Finished: statusstr = "FINISHED"; break; } } #endif #ifndef NDEBUG if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank sampling_status:%s\n", thePid,monitor_get_thread_num(),statusstr); } #endif // NOTE: For post_comm_rank, we always need to resume since // that is where mrnet connection is made. Therefore that // resume should not resume if we are not start enabled. bool resume_sampling = false; // FIXME: commented out if test remove? //if (tls->sampling_status == CBTF_Monitor_Paused) { if (tls->mpi_pcontrol && tls->start_enabled) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING pcontrol start enabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } resume_sampling = true; tls->sampling_status = CBTF_Monitor_Resumed; } else if(tls->mpi_pcontrol && !tls->start_enabled) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING pcontrol start disabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } // FORCE THIS HERE FOR NOW. resume_sampling = true; // tell collector service that start was deferred //cbtf_offline_service_start_deferred(); } else { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank SAMPLING enabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } resume_sampling = true; tls->sampling_status = CBTF_Monitor_Resumed; } //} if (resume_sampling) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_post_comm_rank RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } //tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_MPI_post_comm_rank_event,CBTF_Monitor_Resumed); } }
void monitor_init_mpi(int *argc, char ***argv) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); bool resume_sampling = false; if (tls->sampling_status == CBTF_Monitor_Paused) { if (tls->mpi_pcontrol && tls->start_enabled) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING pcontrol start enabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } resume_sampling = true; } else if(tls->mpi_pcontrol && !tls->start_enabled) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING pcontrol start disabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } } else { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_mpi SAMPLING enabled rank:%d\n", thePid,monitor_get_thread_num(), monitor_mpi_comm_rank()); } resume_sampling = true; } } cbtf_offline_notify_event(CBTF_Monitor_MPI_init_event); if (resume_sampling) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_mpi RESUME SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_MPI_init_event,CBTF_Monitor_Resumed); } else { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_mpi is SAMPLING\n", thePid,monitor_get_thread_num()); } // this is the case where start collection was disabled util at // the first (if any) mpi_pcontrol(1) is encountered. // We need to make this available to any code downstream that // may temporarily disable the collector via cbtf_offline_service_stop_timer // and later reenable the collector via cbtf_offline_service_start_timer. // Both of those live in services/collector/collector.c. // the disable and reenable is used internaly in collector.c to // protect one CBTF_MRNet_Send of attached threads from being // sampled. The other use is in the services/collector/monitor.c // code where we need to disable,reenable for similar reasons. // Could check tls->sampling_status for this from both // cbtf_offline_service_stop_timer and cbtf_offline_service_start_timer. // Provide this as util function that returns this value. } tls->in_mpi_pre_init = false; CBTF_in_mpi_startup = false; }
/* monitor_mpi_pcontrol is reponsible for starting and stopping data * collection based on the user coding: * * MPI_Pcontrol(0) to disable collection * and * MPI_Pcontrol(1) to reenable data collection * */ void monitor_mpi_pcontrol(int level) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); if ( tls->mpi_pcontrol ) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_mpi_pcontrol CALLED level:%d\n", thePid,monitor_get_thread_num(),level); } if (level == 0) { if ((tls->sampling_status == CBTF_Monitor_Started || tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) { tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_mpi_pcontrol_event,CBTF_Monitor_Paused); if (IsMonitorDebugEnabled) { fprintf(stderr,"monitor_mpi_pcontrol level 0 collector started, PAUSE SAMPLING %d,%d\n", thePid,monitor_get_thread_num()); } } } else if (level == 1) { // this should not happen now. if (tls->sampling_status == CBTF_Monitor_Not_Started ) { // should handle mrnet connection for cbtf-mrnet case. cbtf_offline_notify_event(CBTF_Monitor_mpi_pcontrol_event); tls->CBTF_monitor_type = CBTF_Monitor_Proc; tls->sampling_status = CBTF_Monitor_Started; cbtf_offline_start_sampling(NULL); } else if (tls->sampling_status == CBTF_Monitor_Paused && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"monitor_mpi_pcontrol level 1 collector started RESUME SAMPLING %d,%d\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Resumed; cbtf_offline_sampling_status(CBTF_Monitor_mpi_pcontrol_event,CBTF_Monitor_Resumed); } else if (tls->sampling_status == CBTF_Monitor_Resumed && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"monitor_mpi_pcontrol level 1 collector ALREADY RESUMED %d,%d\n", thePid,monitor_get_thread_num()); } } } else { fprintf(stderr,"monitor_mpi_pcontrol CALLED with unsupported level=%d\n", level); } } else { /* early return - do not honor mpi_pcontrol */ if (IsMonitorDebugEnabled) { fprintf(stderr,"monitor_mpi_pcontrol ENABLE_MPI_PCONTROL **NOT** SET IGNORING MPI_PCONTROL CALL %d,%d\n", thePid,monitor_get_thread_num()); } return; } }
void *monitor_init_thread(int tid, void *data) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = malloc(sizeof(TLS)); Assert(tls != NULL); CBTF_SetTLS(TLSKey, tls); #else TLS* tls = &the_tls; #endif Assert(tls != NULL); thePid = getpid(); #if 0 if (monitor_is_threaded()) { monitor_get_thread_num() = monitor_get_thread_num(); } else { monitor_get_thread_num() = 0; } #endif short debug_mpi_pcontrol=0; if ( (getenv("CBTF_DEBUG_MPI_PCONTROL") != NULL)) { debug_mpi_pcontrol=1; } tls->collector_started = false; tls->cbtf_dllist_head = NULL; tls->cbtf_dllist_curr = NULL; if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_thread BEGIN SAMPLING\n", thePid,monitor_get_thread_num()); } tls->in_mpi_pre_init = false; tls->CBTF_monitor_type = CBTF_Monitor_Thread; tls->mpi_pcontrol = 0, tls->start_enabled = 0; if (getenv("OPENSS_ENABLE_MPI_PCONTROL") != NULL) tls->mpi_pcontrol = 1; if (getenv("OPENSS_START_ENABLED") != NULL) tls->start_enabled = 1; if (getenv("CBTF_ENABLE_MPI_PCONTROL") != NULL) tls->mpi_pcontrol = 1; if (getenv("CBTF_START_ENABLED") != NULL) tls->start_enabled = 1; /* * Always start the collector via cbtf_offline_start_sampling. * In the presence of active mpi_pcontrol and start with the * collector disabled, we issue a cbtf_offline_sampling_status paused call. */ #if 0 if (CBTF_in_mpi_startup) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_thread IN MPI STARTUP - DEFER SAMPLING\n", thePid,monitor_get_thread_num()); } return (data); } #endif tls->collector_started = true; tls->sampling_status = CBTF_Monitor_Started; cbtf_offline_start_sampling(NULL); cbtf_offline_notify_event(CBTF_Monitor_init_thread_event); if ( tls->mpi_pcontrol && !tls->start_enabled) { if (debug_mpi_pcontrol) { fprintf(stderr,"[%d,%d] monitor_init_thread CBTF_START_ENABLED NOT SET. Defer sampling at start-up time.\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_init_thread_event,CBTF_Monitor_Paused); } else if ( tls->mpi_pcontrol && tls->start_enabled) { if (debug_mpi_pcontrol) { fprintf(stderr,"[%d,%d] monitor_init_thread CBTF_START_ENABLED SET. START SAMPLING\n", thePid,monitor_get_thread_num()); } } else { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_init_thread START SAMPLING\n", thePid,monitor_get_thread_num()); } } return(data); }
void monitor_dlclose(void *handle) { /* Access our thread-local storage */ #ifdef USE_EXPLICIT_TLS TLS* tls = CBTF_GetTLS(TLSKey); #else TLS* tls = &the_tls; #endif // FIXME: sampling_status is an enum. if (tls == NULL || (tls && tls->sampling_status == CBTF_Monitor_Finished) ) { return; } if (tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlclose returns early due to in mpi init\n",thePid,monitor_get_thread_num()); } return; } while (tls->cbtf_dllist_curr) { if (tls->cbtf_dllist_curr->cbtf_dlinfo_entry.handle == handle) { tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time = CBTF_GetTime(); if (IsMonitorDebugEnabled) { fprintf(stderr,"FOUND %p %s\n",handle, tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name); fprintf(stderr,"loaded at %ld, unloaded at %ld\n", tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time, tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time); } /* On some systems (NASA) it appears that dlopen can be called * before monitor_init_process (or even monitor_early_init). * So we need to use getpid() directly here. */ /* FIXME: Handle return value? */ int retval = CBTF_GetDLInfo(getpid(), tls->cbtf_dllist_curr->cbtf_dlinfo_entry.name, tls->cbtf_dllist_curr->cbtf_dlinfo_entry.load_time, tls->cbtf_dllist_curr->cbtf_dlinfo_entry.unload_time ); if (retval) { } break; } tls->cbtf_dllist_curr = tls->cbtf_dllist_curr->cbtf_dlinfo_next; } if (!tls->thread_is_terminating || !tls->process_is_terminating) { if ((tls->sampling_status == CBTF_Monitor_Started || tls->sampling_status == CBTF_Monitor_Resumed) && !tls->in_mpi_pre_init) { if (IsMonitorDebugEnabled) { fprintf(stderr,"[%d,%d] monitor_dlclose PAUSE SAMPLING\n", thePid,monitor_get_thread_num()); } tls->sampling_status = CBTF_Monitor_Paused; cbtf_offline_sampling_status(CBTF_Monitor_dlclose_event,CBTF_Monitor_Paused); } } }