unwind_interval *
process_unconditional_branch(xed_decoded_inst_t *xptr, bool irdebug,
interval_arg_t *iarg, mem_alloc m_alloc)
{
unwind_interval *next = iarg->current;
if ((iarg->highwatermark).state == HW_UNINITIALIZED) {
(iarg->highwatermark).uwi = iarg->current;
(iarg->highwatermark).state = HW_INITIALIZED;
}
reset_to_canonical_interval(xptr, &next, irdebug, iarg, m_alloc);
TMSG(TAIL_CALL,"checking for tail call via unconditional branch @ %p",iarg->ins);
void *possible = x86_get_branch_target(iarg->ins, xptr);
if (possible == NULL) {
TMSG(TAIL_CALL,"indirect unconditional branch ==> possible tail call");
UWI_RECIPE(next)->has_tail_calls = true;
}
else if ((possible >= iarg->end) || ((uintptr_t)possible < UWI_START_ADDR(iarg->first))) {
TMSG(TAIL_CALL,"unconditional branch to address %p outside of current routine (%p to %p)",
possible, UWI_START_ADDR(iarg->first), iarg->end);
UWI_RECIPE(next)->has_tail_calls = true;
}
return next;
}
//
// Returns: address of freeable region at the high end,
// else NULL on failure.
//
void *
hpcrun_malloc_freeable(size_t size)
{
return hpcrun_malloc(size);
// For now, don't bother with freeable memory.
#if 0
hpcrun_meminfo_t *mi = &TD_GET(memstore);
void *addr, *ans;
size = round_up(size);
addr = mi->mi_low + size;
// Recoverable out of memory.
if (addr >= mi->mi_high) {
TD_GET(mem_low) = 1;
TMSG(MALLOC, "%s: size = %ld, failure: temporary out of memory",
__func__, size);
num_failures++;
return NULL;
}
// Low on memory.
if (addr + low_memsize > mi->mi_high) {
TD_GET(mem_low) = 1;
TMSG(MALLOC, "%s: low on memory, setting epoch flush flag", __func__);
}
ans = mi->mi_low;
mi->mi_low = addr;
total_freeable += size;
TMSG(MALLOC, "%s: size = %ld, addr = %p", __func__, size, addr);
return ans;
#endif
}
// See usage in header.
cct_node_t*
hpcrun_cct_insert_backtrace(cct_node_t* treenode, frame_t* path_beg, frame_t* path_end)
{
TMSG(FENCE, "insert backtrace into treenode %p", treenode);
TMSG(FENCE, "backtrace below");
bool bt_ins = ENABLED(BT_INSERT);
if (ENABLED(FENCE)) {
ENABLE(BT_INSERT);
}
cct_node_t* path = cct_insert_raw_backtrace(treenode, path_beg, path_end);
if (! bt_ins) DISABLE(BT_INSERT);
// Put lush as_info class correction here
// N.B. If 'frm' is a 1-to-1 bichord and 'path' is not (i.e., 'path'
// is M-to-1 or 1-to-M), then update the association of 'path' to
// reflect that 'path' is now a proxy for two bichord types (1-to-1
// and M-to-1 or 1-to-M)
cct_addr_t* addr = hpcrun_cct_addr(path);
lush_assoc_t as_frm = lush_assoc_info__get_assoc(path_beg->as_info);
lush_assoc_t as_path = lush_assoc_info__get_assoc(addr->as_info);
if (as_frm == LUSH_ASSOC_1_to_1 && as_path != LUSH_ASSOC_1_to_1) {
// INVARIANT: path->as_info should be either M-to-1 or 1-to-M
lush_assoc_info__set_assoc(hpcrun_cct_addr(path)->as_info, LUSH_ASSOC_1_to_1);
}
return path;
}
int
__wrap__xlsmpSyncRegionItemL2Fence(long x, long y, long z)
{
int ret;
TMSG(IDLE, "enter fence");
IDLE();
ret = __real__xlsmpSyncRegionItemL2Fence(x,y,z);
WORKING();
TMSG(IDLE, "exit fence");
return ret;
}
int
__wrap_pthread_cond_wait(pthread_cond_t *c, pthread_mutex_t *m)
{
int ret;
TMSG(IDLE, "enter cond");
IDLE();
ret = __real_pthread_cond_wait(c, m);
WORKING();
TMSG(IDLE, "exit cond");
return ret;
}
int
__wrap_pthread_yield()
{
int ret;
TMSG(IDLE, "enter yield");
IDLE();
ret = __real_pthread_yield();
WORKING();
TMSG(IDLE, "exit yield");
return ret;
}
int
__wrap__xlsmpBarrier_TPO(long x, long y)
{
int ret;
TMSG(IDLE, "enter barrier");
IDLE();
ret = __real__xlsmpBarrier_TPO(x, y);
WORKING();
TMSG(IDLE, "exit barrier");
return ret;
}
int
__wrap__xlsmpSyncWorkShareItemL2(long x, long y, long z, long other)
{
int ret;
TMSG(IDLE, "enter sync");
IDLE();
ret = __real__xlsmpSyncWorkShareItemL2(x,y,z,other);
WORKING();
TMSG(IDLE, "exit sync");
return ret;
}
int
__wrap__xlsmpWSDoSetup_TPO(long a1, long a2, long a3, long a4,
long a5, long a6, long a7, long a8)
{
int ret;
TMSG(IDLE, "enter dosetup");
IDLE();
ret = __real__xlsmpWSDoSetup_TPO(a1, a2, a3, a4,
a5, a6, a7, a8);
WORKING();
TMSG(IDLE, "exit dosetup");
return ret;
}
cct_node_t*
hpcrun_cct_record_backtrace_w_metric(cct_bundle_t* cct, bool partial, bool thread_stop,
frame_t* bt_beg, frame_t* bt_last, bool tramp_found,
int metricId, uint64_t metricIncr)
{
TMSG(FENCE, "Recording backtrace");
TMSG(BT_INSERT, "Record backtrace w metric to id %d, incr = %d", metricId, metricIncr);
thread_data_t* td = hpcrun_get_thread_data();
cct_node_t* cct_cursor = cct->tree_root;
TMSG(FENCE, "Initially picking tree root = %p", cct_cursor);
if (tramp_found) {
// start insertion below caller's frame, which is marked with the trampoline
cct_cursor = hpcrun_cct_parent(td->tramp_cct_node);
TMSG(FENCE, "Tramp found ==> cursor = %p", cct_cursor);
}
if (partial) {
cct_cursor = cct->partial_unw_root;
TMSG(FENCE, "Partial unwind ==> cursor = %p", cct_cursor);
}
if (thread_stop) {
cct_cursor = cct->thread_root;
TMSG(FENCE, "Thread stop ==> cursor = %p", cct_cursor);
}
TMSG(FENCE, "sanity check cursor = %p", cct_cursor);
TMSG(FENCE, "further sanity check: bt_last frame = (%d, %p)", bt_last->ip_norm.lm_id, bt_last->ip_norm.lm_ip);
return hpcrun_cct_insert_backtrace_w_metric(cct_cursor, metricId,
bt_last, bt_beg,
(cct_metric_data_t){.i = metricIncr});
static void
add_source(sample_source_t* ss)
{
TMSG(SS_ALL, "%s", ss->name);
if (in_sources(ss)) {
return;
}
*ss_insert = ss;
ss->next_sel = NULL;
ss_insert = &(ss->next_sel);
ss->sel_idx = n_sources++;
TMSG(SS_ALL, "Sample source %s has selection index %d", ss->name, ss->sel_idx);
TMSG(SS_ALL, "# sources now = %d", n_sources);
}
static void
METHOD_FN(start)
{
TMSG(NONE_CTL,"starting NONE");
TD_GET(ss_state)[self->sel_idx] = START;
}
static void
METHOD_FN(init)
{
TMSG(ITIMER_CTL, "init");
blame_shift_source_register(bs_type_timer);
self->state = INIT;
}
static void
METHOD_FN(start)
{
if (! hpcrun_td_avail()){
return; // in the unlikely event that we are trying to start, but thread data is unavailable,
// assume that all sample source ops are suspended.
}
TMSG(_TST_CTL,"starting _tst w value = (%d,%d), interval = (%d,%d)",
itimer.it_interval.tv_sec,
itimer.it_interval.tv_usec,
itimer.it_value.tv_sec,
itimer.it_value.tv_usec);
if (setitimer(HPCRUN_PROFILE_TIMER, &itimer, NULL) != 0) {
EMSG("setitimer failed (%d): %s", errno, strerror(errno));
hpcrun_ssfail_start("_tst");
}
#ifdef USE_ELAPSED_TIME_FOR_WALLCLOCK
int ret = time_getTimeCPU(&TD_GET(last_time_us));
if (ret != 0) {
EMSG("time_getTimeCPU (clock_gettime) failed!");
abort();
}
#endif
TD_GET(ss_state)[self->sel_idx] = START;
}
// Allocate space and init a thread's memstore.
// If failure, shutdown sampling and leave old memstore in place.
void
hpcrun_make_memstore(hpcrun_meminfo_t *mi, int is_child)
{
void *addr;
hpcrun_mem_init();
// If in the child after fork(), then continue to use the parent's
// memstore if it looks ok, else mmap a new one. Note: we can't
// reset the memstore to empty unless we delete everything that was
// created via hpcrun_malloc() (cct, ui_tree, ...).
if (is_child && mi->mi_start != NULL
&& mi->mi_start <= mi->mi_low && mi->mi_low <= mi->mi_high
&& mi->mi_high <= mi->mi_start + mi->mi_size) {
return;
}
addr = hpcrun_mmap_anon(memsize);
if (addr == NULL) {
if (! out_of_mem_mesg) {
EMSG("%s: out of memory, shutting down sampling", __func__);
out_of_mem_mesg = 1;
}
hpcrun_disable_sampling();
return;
}
mi->mi_start = addr;
mi->mi_size = memsize;
mi->mi_low = mi->mi_start;
mi->mi_high = mi->mi_start + memsize;
TMSG(MALLOC, "new memstore: [%p, %p)", mi->mi_start, mi->mi_high);
}
// Look up environ variables and pagesize.
static void
hpcrun_mem_init(void)
{
static int init_done = 0;
char *str;
long ans;
if (init_done)
return;
#ifdef _SC_PAGESIZE
if ((ans = sysconf(_SC_PAGESIZE)) > 0) {
pagesize = ans;
}
#endif
str = getenv(HPCRUN_MEMSIZE);
if (str != NULL && sscanf(str, "%ld", &ans) == 1) {
memsize = hpcrun_align_pagesize(ans);
}
str = getenv(HPCRUN_LOW_MEMSIZE);
if (str != NULL && sscanf(str, "%ld", &ans) == 1) {
low_memsize = ans;
} else {
low_memsize = memsize/40;
if (low_memsize < MIN_LOW_MEMSIZE)
low_memsize = MIN_LOW_MEMSIZE;
}
TMSG(MALLOC, "%s: pagesize = %ld, memsize = %ld, "
"low memsize = %ld, extra mmap = %d",
__func__, pagesize, memsize, low_memsize, allow_extra_mmap);
init_done = 1;
}
static int
ignore_this_thread()
{
if (ignore_thread == THREAD_DOINIT) {
ignore_thread = THREAD_SAMPLING;
char *string = getenv("HPCRUN_IGNORE_THREAD");
if (string) {
// eliminate special cases by adding comma delimiters at front and back
char all_str[1024];
sprintf(all_str, ",%s,", string);
int myid = monitor_get_thread_num();
char myid_str[20];
sprintf(myid_str, ",%d,", myid);
if (strstr(all_str, myid_str)) {
ignore_thread = THREAD_NOSAMPLING;
TMSG(IGNORE, "Thread %d ignore sampling", myid);
}
}
}
return ignore_thread == THREAD_NOSAMPLING;
}
int
__wrap__xlsmp_DynamicChunkCall(long a1, long a2, long a3, long a4,
long a5)
{
int ret;
#ifdef BLAME_DYN_CHUNK
TMSG(IDLE, "enter chunk");
WORKING();
#endif
ret = __real__xlsmp_DynamicChunkCall(a1, a2, a3, a4,
a5);
#ifdef BLAME_DYN_CHUNK
IDLE();
TMSG(IDLE, "exit chunk");
#endif
return ret;
}
// Factor out the body of the start method so we can restart itimer
// without messages in the case that we are interrupting our own code.
// safe = 1 if not inside our code, so ok to print debug messages
//
static void
hpcrun_restart_timer(sample_source_t *self, int safe)
{
int ret;
// if thread data is unavailable, assume that all sample source ops
// are suspended.
if (! hpcrun_td_avail()) {
if (safe) {
TMSG(ITIMER_CTL, "Thread data unavailable ==> sampling suspended");
}
return;
}
thread_data_t *td = hpcrun_get_thread_data();
if (safe) {
TMSG(ITIMER_HANDLER, "starting %s: value = (%d,%d), interval = (%d,%d)",
the_event_name,
itval_start.it_value.tv_sec,
itval_start.it_value.tv_usec,
itval_start.it_interval.tv_sec,
itval_start.it_interval.tv_usec);
}
ret = hpcrun_start_timer(td);
if (ret != 0) {
if (safe) {
TMSG(ITIMER_CTL, "setitimer failed to start!!");
EMSG("setitimer failed (%d): %s", errno, strerror(errno));
}
hpcrun_ssfail_start("itimer");
}
#ifdef USE_ELAPSED_TIME_FOR_WALLCLOCK
ret = time_getTimeReal(&TD_GET(last_time_us));
if (ret != 0) {
if (safe) {
EMSG("time_getTimeReal (clock_gettime) failed!");
}
monitor_real_abort();
}
#endif
TD_GET(ss_state)[self->sel_idx] = START;
}
static void
vrecord(void *from, void *to, validation_status vstat)
{
hpcrun_validation_counts[vstat]++;
if ( ENABLED(VALID_RECORD_ALL) || (vstat == UNW_ADDR_WRONG) ){
TMSG(UNW_VALID,"%p->%p (%s)", from, to, vstat2s(vstat));
}
}
// Look up pagesize if this is possible
//
void
hpcrun_mmap_init(void)
{
static bool init_done = false;
long ans;
if (init_done)
return;
#ifdef _SC_PAGESIZE
if ((ans = sysconf(_SC_PAGESIZE)) > 0) {
TMSG(MMAP, "sysconf gives pagesize = %ld", ans);
pagesize = ans;
}
#endif
TMSG(MMAP, "pagesize = %ld", pagesize);
init_done = true;
}
static void
update_cursor_with_troll(hpcrun_unw_cursor_t* cursor, int offset)
{
if (ENABLED(NO_TROLLING)){
TMSG(TROLL, "Trolling disabled");
hpcrun_unw_throw();
}
unsigned int tmp_ra_offset;
int ret = stack_troll(cursor->sp, &tmp_ra_offset, &deep_validate_return_addr,
(void *)cursor);
if (ret != TROLL_INVALID) {
void **next_sp = ((void **)((unsigned long) cursor->sp + tmp_ra_offset));
void *next_pc = *next_sp;
void *ra_loc = (void*) next_sp;
// the current base pointer is a good assumption for the caller's BP
void **next_bp = (void **) cursor->bp;
next_sp += 1;
if ( next_sp <= cursor->sp){
TMSG(TROLL,"Something weird happened! trolling from %p"
" resulted in sp not advancing", cursor->pc_unnorm);
hpcrun_unw_throw();
}
ip_normalized_t next_pc_norm = ip_normalized_NULL;
cursor->intvl = hpcrun_addr_to_interval(((char *)next_pc) + offset,
next_pc, &next_pc_norm);
if (cursor->intvl) {
TMSG(TROLL,"Trolling advances cursor to pc = %p, sp = %p",
next_pc, next_sp);
TMSG(TROLL,"TROLL SUCCESS pc = %p", cursor->pc_unnorm);
cursor->pc_unnorm = next_pc;
cursor->bp = next_bp;
cursor->sp = next_sp;
cursor->ra_loc = ra_loc;
cursor->pc_norm = next_pc_norm;
cursor->flags = 1; // trolling_used
return; // success!
}
TMSG(TROLL, "No interval found for trolled pc, dropping sample,"
" cursor pc = %p", cursor->pc_unnorm);
// fall through for error handling
}
else {
TMSG(TROLL, "Troll failed: dropping sample, cursor pc = %p",
cursor->pc_unnorm);
TMSG(TROLL,"TROLL FAILURE pc = %p", cursor->pc_unnorm);
// fall through for error handling
}
// assert(0);
hpcrun_unw_throw();
}
static void
METHOD_FN(stop)
{
int rc;
rc = setitimer(HPCRUN_PROFILE_TIMER, &zerotimer, NULL);
TMSG(ITIMER_CTL,"stopping _tst");
TD_GET(ss_state)[self->sel_idx] = STOP;
}
// Reclaim the freeable CCT memory at the low end.
void
hpcrun_reclaim_freeable_mem(void)
{
hpcrun_meminfo_t *mi = &TD_GET(memstore);
mi->mi_low = mi->mi_start;
TD_GET(mem_low) = 0;
num_reclaims++;
TMSG(MALLOC, "%s: %d", __func__, num_reclaims);
}
static cct_node_t*
cct_insert_raw_backtrace(cct_node_t* cct,
frame_t* path_beg, frame_t* path_end)
{
TMSG(BT_INSERT, "%s : start", __func__);
if ( (path_beg < path_end) || (!cct)) {
TMSG(BT_INSERT, "No insert effect, cct = %p, path_beg = %p, path_end = %p",
cct, path_beg, path_end);
return cct;
}
ip_normalized_t parent_routine = ip_normalized_NULL;
for(; path_beg >= path_end; path_beg--){
if ( (! retain_recursion) &&
(path_beg >= path_end + 1) &&
ip_normalized_eq(&(path_beg->the_function), &(parent_routine)) &&
ip_normalized_eq(&(path_beg->the_function), &((path_beg-1)->the_function))) {
TMSG(REC_COMPRESS, "recursive routine compression!");
}
else {
cct_addr_t tmp = (cct_addr_t) {.as_info = path_beg->as_info, .ip_norm = path_beg->ip_norm, .lip = path_beg->lip};
TMSG(BT_INSERT, "inserting addr (%d, %p)", tmp.ip_norm.lm_id, tmp.ip_norm.lm_ip);
cct = hpcrun_cct_insert_addr(cct, &tmp);
}
parent_routine = path_beg->the_function;
}
hpcrun_cct_terminate_path(cct);
return cct;
}
static cct_node_t*
help_hpcrun_backtrace2cct(cct_bundle_t* cct, ucontext_t* context,
int metricId, uint64_t metricIncr,
int skipInner);
//
//
void
hpcrun_set_retain_recursion_mode(bool mode)
{
TMSG(REC_COMPRESS, "retain_recursion set to %s", mode ? "true" : "false");
retain_recursion = mode;
}
// We can't downgrade the lock during fnbounds_unmap_closed_dsos()
// because it acquires the fnbounds lock before the dl-iterate lock,
// and that is a LOR with sampling.
//
void
hpcrun_post_dlclose(void *handle, int ret)
{
int outermost = (dlopen_num_writers == 1);
TMSG(LOADMAP, "dlclose: handle = %p", handle);
fnbounds_unmap_closed_dsos();
if (outermost) {
TD_GET(inside_dlfcn) = false;
}
hpcrun_dlopen_write_unlock();
}
void
ui_dump(unw_interval_t* u)
{
if (!u) {
return;
}
TMSG(INTV, " [%p, %p) ty=%s flgs=%d sp_arg=%d fp_arg=%d ra_arg=%d",
(void*)u->common.start, (void*)u->common.end,
framety_string(u->ty), u->flgs, u->sp_arg, u->fp_arg, u->ra_arg);
//TMSG(INTV, " next=%p prev=%p", u->common.next, u->common.prev);
}
//
// utility routine that does 3 things:
// 1) Generate a std backtrace
// 2) Modifies the backtrace according to a passed in function
// 3) enters the generated backtrace in the cct
//
cct_node_t*
hpcrun_bt2cct(cct_bundle_t *cct, ucontext_t* context,
int metricId, uint64_t metricIncr,
bt_mut_fn bt_fn, bt_fn_arg arg, int isSync)
{
cct_node_t* n = NULL;
if (hpcrun_isLogicalUnwind()) {
#ifdef LATER
TMSG(LUSH,"lush backtrace2cct invoked");
n = lush_backtrace2cct(cct, context, metricId, metricIncr, skipInner,
isSync);
#endif
}
else {
TMSG(LUSH,"regular (NON-lush) bt2cct invoked");
n = help_hpcrun_bt2cct(cct, context, metricId, metricIncr, bt_fn, arg);
}
// N.B.: for lush_backtrace() it may be that n = NULL
return n;
}
static fence_enum_t
hpcrun_check_fence(void* ip)
{
fence_enum_t rv = FENCE_NONE;
if (monitor_unwind_process_bottom_frame(ip))
rv = FENCE_MAIN;
else if (monitor_unwind_thread_bottom_frame(ip))
rv = FENCE_THREAD;
if (ENABLED(FENCE_UNW) && rv != FENCE_NONE)
TMSG(FENCE_UNW, "%s", fence_enum_name(rv));
return rv;
}
static void
METHOD_FN(shutdown)
{
METHOD_CALL(self, stop); // make sure stop has been called
TMSG(ITIMER_CTL, "shutdown %s", the_event_name);
// delete the realtime timer to avoid a timer leak
if (use_realtime || use_cputime) {
thread_data_t *td = hpcrun_get_thread_data();
hpcrun_delete_real_timer(td);
}
self->state = UNINIT;
}
