unwind_interval * process_unconditional_branch(xed_decoded_inst_t *xptr, bool irdebug, interval_arg_t *iarg, mem_alloc m_alloc) { unwind_interval *next = iarg->current; if ((iarg->highwatermark).state == HW_UNINITIALIZED) { (iarg->highwatermark).uwi = iarg->current; (iarg->highwatermark).state = HW_INITIALIZED; } reset_to_canonical_interval(xptr, &next, irdebug, iarg, m_alloc); TMSG(TAIL_CALL,"checking for tail call via unconditional branch @ %p",iarg->ins); void *possible = x86_get_branch_target(iarg->ins, xptr); if (possible == NULL) { TMSG(TAIL_CALL,"indirect unconditional branch ==> possible tail call"); UWI_RECIPE(next)->has_tail_calls = true; } else if ((possible >= iarg->end) || ((uintptr_t)possible < UWI_START_ADDR(iarg->first))) { TMSG(TAIL_CALL,"unconditional branch to address %p outside of current routine (%p to %p)", possible, UWI_START_ADDR(iarg->first), iarg->end); UWI_RECIPE(next)->has_tail_calls = true; } return next; }
// // Returns: address of freeable region at the high end, // else NULL on failure. // void * hpcrun_malloc_freeable(size_t size) { return hpcrun_malloc(size); // For now, don't bother with freeable memory. #if 0 hpcrun_meminfo_t *mi = &TD_GET(memstore); void *addr, *ans; size = round_up(size); addr = mi->mi_low + size; // Recoverable out of memory. if (addr >= mi->mi_high) { TD_GET(mem_low) = 1; TMSG(MALLOC, "%s: size = %ld, failure: temporary out of memory", __func__, size); num_failures++; return NULL; } // Low on memory. if (addr + low_memsize > mi->mi_high) { TD_GET(mem_low) = 1; TMSG(MALLOC, "%s: low on memory, setting epoch flush flag", __func__); } ans = mi->mi_low; mi->mi_low = addr; total_freeable += size; TMSG(MALLOC, "%s: size = %ld, addr = %p", __func__, size, addr); return ans; #endif }
// See usage in header. cct_node_t* hpcrun_cct_insert_backtrace(cct_node_t* treenode, frame_t* path_beg, frame_t* path_end) { TMSG(FENCE, "insert backtrace into treenode %p", treenode); TMSG(FENCE, "backtrace below"); bool bt_ins = ENABLED(BT_INSERT); if (ENABLED(FENCE)) { ENABLE(BT_INSERT); } cct_node_t* path = cct_insert_raw_backtrace(treenode, path_beg, path_end); if (! bt_ins) DISABLE(BT_INSERT); // Put lush as_info class correction here // N.B. If 'frm' is a 1-to-1 bichord and 'path' is not (i.e., 'path' // is M-to-1 or 1-to-M), then update the association of 'path' to // reflect that 'path' is now a proxy for two bichord types (1-to-1 // and M-to-1 or 1-to-M) cct_addr_t* addr = hpcrun_cct_addr(path); lush_assoc_t as_frm = lush_assoc_info__get_assoc(path_beg->as_info); lush_assoc_t as_path = lush_assoc_info__get_assoc(addr->as_info); if (as_frm == LUSH_ASSOC_1_to_1 && as_path != LUSH_ASSOC_1_to_1) { // INVARIANT: path->as_info should be either M-to-1 or 1-to-M lush_assoc_info__set_assoc(hpcrun_cct_addr(path)->as_info, LUSH_ASSOC_1_to_1); } return path; }
int __wrap__xlsmpSyncRegionItemL2Fence(long x, long y, long z) { int ret; TMSG(IDLE, "enter fence"); IDLE(); ret = __real__xlsmpSyncRegionItemL2Fence(x,y,z); WORKING(); TMSG(IDLE, "exit fence"); return ret; }
int __wrap_pthread_cond_wait(pthread_cond_t *c, pthread_mutex_t *m) { int ret; TMSG(IDLE, "enter cond"); IDLE(); ret = __real_pthread_cond_wait(c, m); WORKING(); TMSG(IDLE, "exit cond"); return ret; }
int __wrap_pthread_yield() { int ret; TMSG(IDLE, "enter yield"); IDLE(); ret = __real_pthread_yield(); WORKING(); TMSG(IDLE, "exit yield"); return ret; }
int __wrap__xlsmpBarrier_TPO(long x, long y) { int ret; TMSG(IDLE, "enter barrier"); IDLE(); ret = __real__xlsmpBarrier_TPO(x, y); WORKING(); TMSG(IDLE, "exit barrier"); return ret; }
int __wrap__xlsmpSyncWorkShareItemL2(long x, long y, long z, long other) { int ret; TMSG(IDLE, "enter sync"); IDLE(); ret = __real__xlsmpSyncWorkShareItemL2(x,y,z,other); WORKING(); TMSG(IDLE, "exit sync"); return ret; }
int __wrap__xlsmpWSDoSetup_TPO(long a1, long a2, long a3, long a4, long a5, long a6, long a7, long a8) { int ret; TMSG(IDLE, "enter dosetup"); IDLE(); ret = __real__xlsmpWSDoSetup_TPO(a1, a2, a3, a4, a5, a6, a7, a8); WORKING(); TMSG(IDLE, "exit dosetup"); return ret; }
cct_node_t* hpcrun_cct_record_backtrace_w_metric(cct_bundle_t* cct, bool partial, bool thread_stop, frame_t* bt_beg, frame_t* bt_last, bool tramp_found, int metricId, uint64_t metricIncr) { TMSG(FENCE, "Recording backtrace"); TMSG(BT_INSERT, "Record backtrace w metric to id %d, incr = %d", metricId, metricIncr); thread_data_t* td = hpcrun_get_thread_data(); cct_node_t* cct_cursor = cct->tree_root; TMSG(FENCE, "Initially picking tree root = %p", cct_cursor); if (tramp_found) { // start insertion below caller's frame, which is marked with the trampoline cct_cursor = hpcrun_cct_parent(td->tramp_cct_node); TMSG(FENCE, "Tramp found ==> cursor = %p", cct_cursor); } if (partial) { cct_cursor = cct->partial_unw_root; TMSG(FENCE, "Partial unwind ==> cursor = %p", cct_cursor); } if (thread_stop) { cct_cursor = cct->thread_root; TMSG(FENCE, "Thread stop ==> cursor = %p", cct_cursor); } TMSG(FENCE, "sanity check cursor = %p", cct_cursor); TMSG(FENCE, "further sanity check: bt_last frame = (%d, %p)", bt_last->ip_norm.lm_id, bt_last->ip_norm.lm_ip); return hpcrun_cct_insert_backtrace_w_metric(cct_cursor, metricId, bt_last, bt_beg, (cct_metric_data_t){.i = metricIncr});
static void add_source(sample_source_t* ss) { TMSG(SS_ALL, "%s", ss->name); if (in_sources(ss)) { return; } *ss_insert = ss; ss->next_sel = NULL; ss_insert = &(ss->next_sel); ss->sel_idx = n_sources++; TMSG(SS_ALL, "Sample source %s has selection index %d", ss->name, ss->sel_idx); TMSG(SS_ALL, "# sources now = %d", n_sources); }
static void METHOD_FN(start) { TMSG(NONE_CTL,"starting NONE"); TD_GET(ss_state)[self->sel_idx] = START; }
static void METHOD_FN(init) { TMSG(ITIMER_CTL, "init"); blame_shift_source_register(bs_type_timer); self->state = INIT; }
static void METHOD_FN(start) { if (! hpcrun_td_avail()){ return; // in the unlikely event that we are trying to start, but thread data is unavailable, // assume that all sample source ops are suspended. } TMSG(_TST_CTL,"starting _tst w value = (%d,%d), interval = (%d,%d)", itimer.it_interval.tv_sec, itimer.it_interval.tv_usec, itimer.it_value.tv_sec, itimer.it_value.tv_usec); if (setitimer(HPCRUN_PROFILE_TIMER, &itimer, NULL) != 0) { EMSG("setitimer failed (%d): %s", errno, strerror(errno)); hpcrun_ssfail_start("_tst"); } #ifdef USE_ELAPSED_TIME_FOR_WALLCLOCK int ret = time_getTimeCPU(&TD_GET(last_time_us)); if (ret != 0) { EMSG("time_getTimeCPU (clock_gettime) failed!"); abort(); } #endif TD_GET(ss_state)[self->sel_idx] = START; }
// Allocate space and init a thread's memstore. // If failure, shutdown sampling and leave old memstore in place. void hpcrun_make_memstore(hpcrun_meminfo_t *mi, int is_child) { void *addr; hpcrun_mem_init(); // If in the child after fork(), then continue to use the parent's // memstore if it looks ok, else mmap a new one. Note: we can't // reset the memstore to empty unless we delete everything that was // created via hpcrun_malloc() (cct, ui_tree, ...). if (is_child && mi->mi_start != NULL && mi->mi_start <= mi->mi_low && mi->mi_low <= mi->mi_high && mi->mi_high <= mi->mi_start + mi->mi_size) { return; } addr = hpcrun_mmap_anon(memsize); if (addr == NULL) { if (! out_of_mem_mesg) { EMSG("%s: out of memory, shutting down sampling", __func__); out_of_mem_mesg = 1; } hpcrun_disable_sampling(); return; } mi->mi_start = addr; mi->mi_size = memsize; mi->mi_low = mi->mi_start; mi->mi_high = mi->mi_start + memsize; TMSG(MALLOC, "new memstore: [%p, %p)", mi->mi_start, mi->mi_high); }
// Look up environ variables and pagesize. static void hpcrun_mem_init(void) { static int init_done = 0; char *str; long ans; if (init_done) return; #ifdef _SC_PAGESIZE if ((ans = sysconf(_SC_PAGESIZE)) > 0) { pagesize = ans; } #endif str = getenv(HPCRUN_MEMSIZE); if (str != NULL && sscanf(str, "%ld", &ans) == 1) { memsize = hpcrun_align_pagesize(ans); } str = getenv(HPCRUN_LOW_MEMSIZE); if (str != NULL && sscanf(str, "%ld", &ans) == 1) { low_memsize = ans; } else { low_memsize = memsize/40; if (low_memsize < MIN_LOW_MEMSIZE) low_memsize = MIN_LOW_MEMSIZE; } TMSG(MALLOC, "%s: pagesize = %ld, memsize = %ld, " "low memsize = %ld, extra mmap = %d", __func__, pagesize, memsize, low_memsize, allow_extra_mmap); init_done = 1; }
static int ignore_this_thread() { if (ignore_thread == THREAD_DOINIT) { ignore_thread = THREAD_SAMPLING; char *string = getenv("HPCRUN_IGNORE_THREAD"); if (string) { // eliminate special cases by adding comma delimiters at front and back char all_str[1024]; sprintf(all_str, ",%s,", string); int myid = monitor_get_thread_num(); char myid_str[20]; sprintf(myid_str, ",%d,", myid); if (strstr(all_str, myid_str)) { ignore_thread = THREAD_NOSAMPLING; TMSG(IGNORE, "Thread %d ignore sampling", myid); } } } return ignore_thread == THREAD_NOSAMPLING; }
int __wrap__xlsmp_DynamicChunkCall(long a1, long a2, long a3, long a4, long a5) { int ret; #ifdef BLAME_DYN_CHUNK TMSG(IDLE, "enter chunk"); WORKING(); #endif ret = __real__xlsmp_DynamicChunkCall(a1, a2, a3, a4, a5); #ifdef BLAME_DYN_CHUNK IDLE(); TMSG(IDLE, "exit chunk"); #endif return ret; }
// Factor out the body of the start method so we can restart itimer // without messages in the case that we are interrupting our own code. // safe = 1 if not inside our code, so ok to print debug messages // static void hpcrun_restart_timer(sample_source_t *self, int safe) { int ret; // if thread data is unavailable, assume that all sample source ops // are suspended. if (! hpcrun_td_avail()) { if (safe) { TMSG(ITIMER_CTL, "Thread data unavailable ==> sampling suspended"); } return; } thread_data_t *td = hpcrun_get_thread_data(); if (safe) { TMSG(ITIMER_HANDLER, "starting %s: value = (%d,%d), interval = (%d,%d)", the_event_name, itval_start.it_value.tv_sec, itval_start.it_value.tv_usec, itval_start.it_interval.tv_sec, itval_start.it_interval.tv_usec); } ret = hpcrun_start_timer(td); if (ret != 0) { if (safe) { TMSG(ITIMER_CTL, "setitimer failed to start!!"); EMSG("setitimer failed (%d): %s", errno, strerror(errno)); } hpcrun_ssfail_start("itimer"); } #ifdef USE_ELAPSED_TIME_FOR_WALLCLOCK ret = time_getTimeReal(&TD_GET(last_time_us)); if (ret != 0) { if (safe) { EMSG("time_getTimeReal (clock_gettime) failed!"); } monitor_real_abort(); } #endif TD_GET(ss_state)[self->sel_idx] = START; }
static void vrecord(void *from, void *to, validation_status vstat) { hpcrun_validation_counts[vstat]++; if ( ENABLED(VALID_RECORD_ALL) || (vstat == UNW_ADDR_WRONG) ){ TMSG(UNW_VALID,"%p->%p (%s)", from, to, vstat2s(vstat)); } }
// Look up pagesize if this is possible // void hpcrun_mmap_init(void) { static bool init_done = false; long ans; if (init_done) return; #ifdef _SC_PAGESIZE if ((ans = sysconf(_SC_PAGESIZE)) > 0) { TMSG(MMAP, "sysconf gives pagesize = %ld", ans); pagesize = ans; } #endif TMSG(MMAP, "pagesize = %ld", pagesize); init_done = true; }
static void update_cursor_with_troll(hpcrun_unw_cursor_t* cursor, int offset) { if (ENABLED(NO_TROLLING)){ TMSG(TROLL, "Trolling disabled"); hpcrun_unw_throw(); } unsigned int tmp_ra_offset; int ret = stack_troll(cursor->sp, &tmp_ra_offset, &deep_validate_return_addr, (void *)cursor); if (ret != TROLL_INVALID) { void **next_sp = ((void **)((unsigned long) cursor->sp + tmp_ra_offset)); void *next_pc = *next_sp; void *ra_loc = (void*) next_sp; // the current base pointer is a good assumption for the caller's BP void **next_bp = (void **) cursor->bp; next_sp += 1; if ( next_sp <= cursor->sp){ TMSG(TROLL,"Something weird happened! trolling from %p" " resulted in sp not advancing", cursor->pc_unnorm); hpcrun_unw_throw(); } ip_normalized_t next_pc_norm = ip_normalized_NULL; cursor->intvl = hpcrun_addr_to_interval(((char *)next_pc) + offset, next_pc, &next_pc_norm); if (cursor->intvl) { TMSG(TROLL,"Trolling advances cursor to pc = %p, sp = %p", next_pc, next_sp); TMSG(TROLL,"TROLL SUCCESS pc = %p", cursor->pc_unnorm); cursor->pc_unnorm = next_pc; cursor->bp = next_bp; cursor->sp = next_sp; cursor->ra_loc = ra_loc; cursor->pc_norm = next_pc_norm; cursor->flags = 1; // trolling_used return; // success! } TMSG(TROLL, "No interval found for trolled pc, dropping sample," " cursor pc = %p", cursor->pc_unnorm); // fall through for error handling } else { TMSG(TROLL, "Troll failed: dropping sample, cursor pc = %p", cursor->pc_unnorm); TMSG(TROLL,"TROLL FAILURE pc = %p", cursor->pc_unnorm); // fall through for error handling } // assert(0); hpcrun_unw_throw(); }
static void METHOD_FN(stop) { int rc; rc = setitimer(HPCRUN_PROFILE_TIMER, &zerotimer, NULL); TMSG(ITIMER_CTL,"stopping _tst"); TD_GET(ss_state)[self->sel_idx] = STOP; }
// Reclaim the freeable CCT memory at the low end. void hpcrun_reclaim_freeable_mem(void) { hpcrun_meminfo_t *mi = &TD_GET(memstore); mi->mi_low = mi->mi_start; TD_GET(mem_low) = 0; num_reclaims++; TMSG(MALLOC, "%s: %d", __func__, num_reclaims); }
static cct_node_t* cct_insert_raw_backtrace(cct_node_t* cct, frame_t* path_beg, frame_t* path_end) { TMSG(BT_INSERT, "%s : start", __func__); if ( (path_beg < path_end) || (!cct)) { TMSG(BT_INSERT, "No insert effect, cct = %p, path_beg = %p, path_end = %p", cct, path_beg, path_end); return cct; } ip_normalized_t parent_routine = ip_normalized_NULL; for(; path_beg >= path_end; path_beg--){ if ( (! retain_recursion) && (path_beg >= path_end + 1) && ip_normalized_eq(&(path_beg->the_function), &(parent_routine)) && ip_normalized_eq(&(path_beg->the_function), &((path_beg-1)->the_function))) { TMSG(REC_COMPRESS, "recursive routine compression!"); } else { cct_addr_t tmp = (cct_addr_t) {.as_info = path_beg->as_info, .ip_norm = path_beg->ip_norm, .lip = path_beg->lip}; TMSG(BT_INSERT, "inserting addr (%d, %p)", tmp.ip_norm.lm_id, tmp.ip_norm.lm_ip); cct = hpcrun_cct_insert_addr(cct, &tmp); } parent_routine = path_beg->the_function; } hpcrun_cct_terminate_path(cct); return cct; } static cct_node_t* help_hpcrun_backtrace2cct(cct_bundle_t* cct, ucontext_t* context, int metricId, uint64_t metricIncr, int skipInner); // // void hpcrun_set_retain_recursion_mode(bool mode) { TMSG(REC_COMPRESS, "retain_recursion set to %s", mode ? "true" : "false"); retain_recursion = mode; }
// We can't downgrade the lock during fnbounds_unmap_closed_dsos() // because it acquires the fnbounds lock before the dl-iterate lock, // and that is a LOR with sampling. // void hpcrun_post_dlclose(void *handle, int ret) { int outermost = (dlopen_num_writers == 1); TMSG(LOADMAP, "dlclose: handle = %p", handle); fnbounds_unmap_closed_dsos(); if (outermost) { TD_GET(inside_dlfcn) = false; } hpcrun_dlopen_write_unlock(); }
void ui_dump(unw_interval_t* u) { if (!u) { return; } TMSG(INTV, " [%p, %p) ty=%s flgs=%d sp_arg=%d fp_arg=%d ra_arg=%d", (void*)u->common.start, (void*)u->common.end, framety_string(u->ty), u->flgs, u->sp_arg, u->fp_arg, u->ra_arg); //TMSG(INTV, " next=%p prev=%p", u->common.next, u->common.prev); }
// // utility routine that does 3 things: // 1) Generate a std backtrace // 2) Modifies the backtrace according to a passed in function // 3) enters the generated backtrace in the cct // cct_node_t* hpcrun_bt2cct(cct_bundle_t *cct, ucontext_t* context, int metricId, uint64_t metricIncr, bt_mut_fn bt_fn, bt_fn_arg arg, int isSync) { cct_node_t* n = NULL; if (hpcrun_isLogicalUnwind()) { #ifdef LATER TMSG(LUSH,"lush backtrace2cct invoked"); n = lush_backtrace2cct(cct, context, metricId, metricIncr, skipInner, isSync); #endif } else { TMSG(LUSH,"regular (NON-lush) bt2cct invoked"); n = help_hpcrun_bt2cct(cct, context, metricId, metricIncr, bt_fn, arg); } // N.B.: for lush_backtrace() it may be that n = NULL return n; }
static fence_enum_t hpcrun_check_fence(void* ip) { fence_enum_t rv = FENCE_NONE; if (monitor_unwind_process_bottom_frame(ip)) rv = FENCE_MAIN; else if (monitor_unwind_thread_bottom_frame(ip)) rv = FENCE_THREAD; if (ENABLED(FENCE_UNW) && rv != FENCE_NONE) TMSG(FENCE_UNW, "%s", fence_enum_name(rv)); return rv; }
static void METHOD_FN(shutdown) { METHOD_CALL(self, stop); // make sure stop has been called TMSG(ITIMER_CTL, "shutdown %s", the_event_name); // delete the realtime timer to avoid a timer leak if (use_realtime || use_cputime) { thread_data_t *td = hpcrun_get_thread_data(); hpcrun_delete_real_timer(td); } self->state = UNINIT; }