inline uint64_t
elapsed_usec() const
{
compiler_barrier();
const uint64_t t0 = start_;
const uint64_t t1 = cur_usec(m_);
compiler_barrier();
return t1 - t0;
}
inline uint64_t
lap()
{
compiler_barrier();
const uint64_t t0 = start_;
const uint64_t t1 = cur_usec(m_);
start_ = t1;
compiler_barrier();
return t1 - t0;
}
/**
* Converts a key to a gni memory handle
*
* @param key gnix memory registration key
* @param mhdl gni memory handle
*/
void _gnix_convert_key_to_mhdl(
gnix_mr_key_t *key,
gni_mem_handle_t *mhdl)
{
_gnix_convert_key_to_mhdl_no_crc(key, mhdl);
compiler_barrier();
GNI_MEMHNDL_SET_CRC((*mhdl));
}
int NOINLINE b(int x)
{
int r;
compiler_barrier();
if (x)
r = a();
else
r = c(1);
return r + 1;
}
u32_t _get_elapsed_program_time(void)
{
u32_t rtc_elapsed, rtc_past_copy;
if (!expected_sys_ticks) {
return 0;
}
/* Read rtc_past before RTC_COUNTER */
rtc_past_copy = rtc_past;
/* Make sure that compiler will not reverse access to RTC and
* rtc_past.
*/
compiler_barrier();
rtc_elapsed = (RTC_COUNTER - rtc_past_copy) & RTC_MASK;
/* Convert number of Machine cycles to SYS_TICKS */
return (rtc_elapsed / sys_clock_hw_cycles_per_tick);
}
u64_t _get_elapsed_clock_time(void)
{
u64_t elapsed;
u32_t rtc_elapsed, rtc_past_copy;
/* Read _sys_clock_tick_count and rtc_past before RTC_COUNTER */
elapsed = _sys_clock_tick_count;
rtc_past_copy = rtc_past;
/* Make sure that compiler will not reverse access to RTC and
* variables above.
*/
compiler_barrier();
rtc_elapsed = (RTC_COUNTER - rtc_past_copy) & RTC_MASK;
if (rtc_elapsed >= sys_clock_hw_cycles_per_tick) {
/* Update total number of SYS_TICKS passed */
elapsed += (rtc_elapsed / sys_clock_hw_cycles_per_tick);
}
return elapsed;
}
u32_t _timer_cycle_get_32(void)
{
u32_t ticked_cycles;
u32_t elapsed_cycles;
/* Number of timer cycles announced as ticks so far. */
ticked_cycles = _sys_clock_tick_count * sys_clock_hw_cycles_per_tick;
/* Make sure that compiler will not reverse access to RTC and
* _sys_clock_tick_count.
*/
compiler_barrier();
/* Number of timer cycles since last announced tick we know about.
*
* The value of RTC_COUNTER is not reset on tick, so it will
* compensate potentialy missed update of _sys_clock_tick_count
* which could have happen between the ticked_cycles calculation
* and the code below.
*/
elapsed_cycles = (RTC_COUNTER - ticked_cycles) & RTC_MASK;
return ticked_cycles + elapsed_cycles;
}
unsigned long plthook_entry(unsigned long *ret_addr, unsigned long child_idx,
unsigned long module_id, struct mcount_regs *regs)
{
struct sym *sym;
unsigned long child_ip;
struct mcount_thread_data *mtdp;
struct mcount_ret_stack *rstack;
struct ftrace_trigger tr = {
.flags = 0,
};
bool skip = false;
enum filter_result filtered;
if (unlikely(mcount_should_stop()))
return 0;
mtd.recursion_guard = true;
mtdp = get_thread_data();
if (unlikely(check_thread_data(mtdp))) {
mcount_prepare();
mtdp = get_thread_data();
assert(mtdp);
}
/*
* There was a recursion like below:
*
* plthook_entry -> mcount_entry -> mcount_prepare -> xmalloc
* -> plthook_entry
*/
if (mtdp->plthook_guard)
goto out;
if (check_dynsym_idxlist(&skip_idxlist, child_idx))
goto out;
sym = find_dynsym(&symtabs, child_idx);
pr_dbg3("[%d] enter %lx: %s\n", child_idx, sym->addr, sym->name);
child_ip = sym ? sym->addr : 0;
if (child_ip == 0) {
pr_err_ns("invalid function idx found! (idx: %d, %#lx)\n",
(int) child_idx, child_idx);
}
filtered = mcount_entry_filter_check(mtdp, sym->addr, &tr);
if (filtered != FILTER_IN) {
/*
* Skip recording but still hook the return address,
* otherwise it cannot trace further invocations due to
* the overwritten PLT entry by the resolver function.
*/
skip = true;
/* but if we don't have rstack, just bail out */
if (filtered == FILTER_RSTACK)
goto out;
}
mtdp->plthook_guard = true;
rstack = &mtdp->rstack[mtdp->idx++];
rstack->depth = mtdp->record_idx;
rstack->dyn_idx = child_idx;
rstack->parent_loc = ret_addr;
rstack->parent_ip = *ret_addr;
rstack->child_ip = child_ip;
rstack->start_time = skip ? 0 : mcount_gettime();
rstack->end_time = 0;
rstack->flags = skip ? MCOUNT_FL_NORECORD : 0;
mcount_entry_filter_record(mtdp, rstack, &tr, regs);
*ret_addr = (unsigned long)plthook_return;
if (check_dynsym_idxlist(&setjmp_idxlist, child_idx))
setup_jmpbuf_rstack(mtdp, mtdp->idx - 1);
else if (check_dynsym_idxlist(&longjmp_idxlist, child_idx))
rstack->flags |= MCOUNT_FL_LONGJMP;
else if (check_dynsym_idxlist(&vfork_idxlist, child_idx)) {
rstack->flags |= MCOUNT_FL_VFORK;
prepare_vfork(mtdp, rstack);
}
/* force flush rstack on some special functions */
if (check_dynsym_idxlist(&flush_idxlist, child_idx)) {
record_trace_data(mtdp, rstack, NULL);
}
if (plthook_dynsym_resolved[child_idx]) {
volatile unsigned long *resolved_addr = plthook_dynsym_addr + child_idx;
/* ensure resolved address was set */
while (!*resolved_addr)
cpu_relax();
mtdp->recursion_guard = false;
return *resolved_addr;
}
mtdp->plthook_addr = plthook_got_ptr[3 + child_idx];
out:
mtdp->recursion_guard = false;
return 0;
}
unsigned long plthook_exit(long *retval)
{
int dyn_idx;
unsigned long new_addr;
struct mcount_thread_data *mtdp;
struct mcount_ret_stack *rstack;
mtdp = get_thread_data();
assert(mtdp);
mtdp->recursion_guard = true;
again:
rstack = &mtdp->rstack[mtdp->idx - 1];
if (unlikely(rstack->flags & (MCOUNT_FL_LONGJMP | MCOUNT_FL_VFORK))) {
if (rstack->flags & MCOUNT_FL_LONGJMP) {
restore_jmpbuf_rstack(mtdp, mtdp->idx + 1);
rstack->flags &= ~MCOUNT_FL_LONGJMP;
goto again;
}
if (rstack->flags & MCOUNT_FL_VFORK)
setup_vfork(mtdp);
}
if (unlikely(vfork_parent))
rstack = restore_vfork(mtdp, rstack);
dyn_idx = rstack->dyn_idx;
if (dyn_idx == MCOUNT_INVALID_DYNIDX)
pr_err_ns("<%d> invalid dynsym idx: %d\n", mtdp->idx, dyn_idx);
pr_dbg3("[%d] exit %lx: %s\n", dyn_idx,
plthook_dynsym_addr[dyn_idx],
find_dynsym(&symtabs, dyn_idx)->name);
if (!(rstack->flags & MCOUNT_FL_NORECORD))
rstack->end_time = mcount_gettime();
mcount_exit_filter_record(mtdp, rstack, retval);
if (!plthook_dynsym_resolved[dyn_idx]) {
#ifndef SINGLE_THREAD
static pthread_mutex_t resolver_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_lock(&resolver_mutex);
#endif
if (!plthook_dynsym_resolved[dyn_idx]) {
new_addr = plthook_got_ptr[3 + dyn_idx];
/* restore GOT so plt_hooker keep called */
plthook_got_ptr[3 + dyn_idx] = mtdp->plthook_addr;
plthook_dynsym_addr[dyn_idx] = new_addr;
plthook_dynsym_resolved[dyn_idx] = true;
}
#ifndef SINGLE_THREAD
pthread_mutex_unlock(&resolver_mutex);
#endif
}
compiler_barrier();
mtdp->idx--;
mtdp->recursion_guard = false;
mtdp->plthook_guard = false;
return rstack->parent_ip;
}
VOID_TASK_IMPL_2(sylvan_skiplist_assign_next, sylvan_skiplist_t, l, MTBDD, dd)
{
if (dd == mtbdd_false || dd == mtbdd_true) return;
uint32_t trace[SL_DEPTH];
uint32_t loc = 0, loc_next = 0, k = SL_DEPTH-1;
for (;;) {
/* invariant: [loc].dd < dd */
/* note: this is always true for loc==0 */
sl_bucket *e = l->buckets + loc;
loc_next = (*(volatile uint32_t*)&e->next[k]) & 0x7fffffff;
if (loc_next != 0 && l->buckets[loc_next].dd == dd) {
/* found */
return;
} else if (loc_next != 0 && l->buckets[loc_next].dd < dd) {
/* go right */
loc = loc_next;
} else if (k > 0) {
/* go down */
trace[k] = loc;
k--;
} else if (!(e->next[0] & 0x80000000) && cas(&e->next[0], loc_next, loc_next|0x80000000)) {
/* locked */
break;
}
}
/* claim next item */
const uint64_t next = __sync_fetch_and_add(&l->next, 1);
if (next >= l->size) {
fprintf(stderr, "Out of cheese exception, no more blocks available\n");
exit(1);
}
/* fill next item */
sl_bucket *a = l->buckets + next;
a->dd = dd;
a->next[0] = loc_next;
compiler_barrier();
l->buckets[loc].next[0] = next;
/* determine height */
uint64_t h = 1 + __builtin_clz(LACE_TRNG) / 2;
if (h > SL_DEPTH) h = SL_DEPTH;
/* go up and create links */
for (k=1;k<h;k++) {
loc = trace[k];
for (;;) {
sl_bucket *e = l->buckets + loc;
/* note, at k>0, no locks on edges */
uint32_t loc_next = *(volatile uint32_t*)&e->next[k];
if (loc_next != 0 && l->buckets[loc_next].dd < dd) {
loc = loc_next;
} else {
a->next[k] = loc_next;
if (cas(&e->next[k], loc_next, next)) break;
}
}
}
}