static int
fbt_prov_entry(pf_info_t *infp, instr_t *instr, int size, int modrm)
{
fbt_probe_t *fbt;
/***********************************************/
/* Avoid patching a patched probe point. */
/***********************************************/
if (*instr == 0xcc)
return 1;
/***********************************************/
/* This is temporary. Because of the issue */
/* of %RIP relative addressing modes */
/* potentially not being addressable in our */
/* single-step jump buffer, disable those */
/* probes which look like one of these. */
/***********************************************/
if (modrm >= 0 && (instr[modrm] & 0xc7) == 0x05)
return 1;
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = infp->name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, infp->modname,
infp->name, FBT_ENTRY, 3, fbt);
num_probes++;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = infp->mp; // ctl;
fbt->fbtp_loadcnt = get_refcount(infp->mp);
fbt->fbtp_rval = DTRACE_INVOP_ANY;
/***********************************************/
/* Save potential overwrite of instruction */
/* and length, because we will need the */
/* entire instruction when we single step */
/* over it. */
/***********************************************/
fbt->fbtp_savedval = *instr;
fbt->fbtp_inslen = size;
fbt->fbtp_type = 0; /* entry */
//if (modrm >= 0 && (instr[modrm] & 0xc7) == 0x05) printk("modrm %s %p rm=%d\n", name, instr, modrm);
fbt->fbtp_modrm = modrm;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = infp->symndx;
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
infp->pmp->fbt_nentries++;
infp->retptr = NULL;
return 1;
}
/*ARGSUSED*/
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg,id)
fbt_probe_t *fbt = parg, *next, *hash, *last;
int ndx;
do {
/*
* Now we need to remove this probe from the fbt_probetab.
*/
ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
last = NULL;
hash = fbt_probetab[ndx];
while (hash != fbt) {
ASSERT(hash != NULL);
last = hash;
hash = hash->fbtp_hashnext;
}
if (last != NULL) {
last->fbtp_hashnext = fbt->fbtp_hashnext;
} else {
fbt_probetab[ndx] = fbt->fbtp_hashnext;
}
next = fbt->fbtp_next;
kmem_free(fbt, sizeof (fbt_probe_t));
fbt = next;
} while (fbt != NULL);
}
static void
fbt_disable(void *arg, dtrace_id_t id, void *parg)
{
fbt_probe_t *fbt = parg, *hash;
modctl_t *ctl = fbt->fbtp_ctl;
ASSERT(ctl->nenabled > 0);
ctl->nenabled--;
if ((ctl->loadcnt != fbt->fbtp_loadcnt))
return;
for (; fbt != NULL; fbt = fbt->fbtp_probenext) {
fbt->fbtp_enabled--;
for (hash = fbt_probetab[FBT_ADDR2NDX(fbt->fbtp_patchpoint)];
hash != NULL; hash = hash->fbtp_hashnext) {
if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
for (; hash != NULL; hash = hash->fbtp_tracenext)
if (hash->fbtp_enabled > 0)
break;
break;
}
}
if (hash == NULL)
fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
}
}
static void
fbt_destroy_one(fbt_probe_t *fbt)
{
fbt_probe_t *hash, *hashprev, *next;
int ndx;
ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
for (hash = fbt_probetab[ndx], hashprev = NULL; hash != NULL;
hashprev = hash, hash = hash->fbtp_hashnext) {
if (hash == fbt) {
if ((next = fbt->fbtp_tracenext) != NULL)
next->fbtp_hashnext = hash->fbtp_hashnext;
else
next = hash->fbtp_hashnext;
if (hashprev != NULL)
hashprev->fbtp_hashnext = next;
else
fbt_probetab[ndx] = next;
goto free;
} else if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
for (next = hash; next->fbtp_tracenext != NULL;
next = next->fbtp_tracenext) {
if (fbt == next->fbtp_tracenext) {
next->fbtp_tracenext =
fbt->fbtp_tracenext;
goto free;
}
}
}
}
panic("probe %p not found in hash table", fbt);
free:
free(fbt, M_FBT);
}
int
fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
{
struct trapframe *frame = (struct trapframe *)stack;
solaris_cpu_t *cpu = &solaris_cpu[curcpu];
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
register_t fifthparam;
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
cpu->cpu_dtrace_caller = addr;
/* Get 5th parameter from stack */
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
fifthparam = *(register_t *)frame->tf_usr_sp;
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
dtrace_probe(fbt->fbtp_id, frame->tf_r0,
frame->tf_r1, frame->tf_r2,
frame->tf_r3, fifthparam);
cpu->cpu_dtrace_caller = 0;
return (fbt->fbtp_rval | (fbt->fbtp_savedval << DTRACE_INVOP_SHIFT));
}
}
return (0);
}
static int
fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
{
solaris_cpu_t *cpu = &solaris_cpu[curcpu];
uintptr_t stack0, stack1, stack2, stack3, stack4;
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
fbt->fbtp_invop_cnt++;
if (fbt->fbtp_roffset == 0) {
int i = 0;
/*
* When accessing the arguments on the stack,
* we must protect against accessing beyond
* the stack. We can safely set NOFAULT here
* -- we know that interrupts are already
* disabled.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
cpu->cpu_dtrace_caller = stack[i++];
stack0 = stack[i++];
stack1 = stack[i++];
stack2 = stack[i++];
stack3 = stack[i++];
stack4 = stack[i++];
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
dtrace_probe(fbt->fbtp_id, stack0, stack1,
stack2, stack3, stack4);
cpu->cpu_dtrace_caller = 0;
} else {
#ifdef __amd64__
/*
* On amd64, we instrument the ret, not the
* leave. We therefore need to set the caller
* to assure that the top frame of a stack()
* action is correct.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
cpu->cpu_dtrace_caller = stack[0];
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
#endif
dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
rval, 0, 0, 0);
cpu->cpu_dtrace_caller = 0;
}
return (fbt->fbtp_rval);
}
}
return (0);
}
int *
fbt_get_instr_buf(instr_t *addr)
{
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if (fbt->fbtp_patchpoint == addr)
return fbt->fbtp_instr_buf;
}
return NULL;
}
static int
fbt_is_patched(char *name, instr_t *addr)
{
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if (fbt->fbtp_patchpoint == addr) {
dtrace_printf("fbt:dup patch: %p %s\n", addr, name);
return 1;
}
}
return 0;
}
int
fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
{
struct trapframe *frame = (struct trapframe *)stack;
solaris_cpu_t *cpu = &solaris_cpu[curcpu];
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
uintptr_t tmp;
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
fbt->fbtp_invop_cnt++;
if (fbt->fbtp_roffset == 0) {
cpu->cpu_dtrace_caller = addr;
dtrace_probe(fbt->fbtp_id, frame->fixreg[3],
frame->fixreg[4], frame->fixreg[5],
frame->fixreg[6], frame->fixreg[7]);
cpu->cpu_dtrace_caller = 0;
} else {
dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
rval, 0, 0, 0);
/*
* The caller doesn't have the fbt item, so
* fixup tail calls here.
*/
if (fbt->fbtp_rval == DTRACE_INVOP_JUMP) {
frame->srr0 = (uintptr_t)fbt->fbtp_patchpoint;
tmp = fbt->fbtp_savedval & FBT_BR_MASK;
/* Sign extend. */
if (tmp & 0x02000000)
#ifdef __powerpc64__
tmp |= 0xfffffffffc000000ULL;
#else
tmp |= 0xfc000000UL;
#endif
frame->srr0 += tmp;
}
cpu->cpu_dtrace_caller = 0;
}
return (fbt->fbtp_rval);
}
}
return (0);
}
/*ARGSUSED*/
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
fbt_probe_t *fbt = parg, *next, *hash, *last;
struct modctl *ctl = fbt->fbtp_ctl;
struct module *mp = ctl;
int ndx;
do {
//printk("refc=%d load=%d\n", get_refcount(mp), fbt->fbtp_loadcnt);
if (mp != NULL && get_refcount(mp) == fbt->fbtp_loadcnt) {
if ((get_refcount(mp) == fbt->fbtp_loadcnt &&
mp->state == MODULE_STATE_LIVE)) {
par_module_t *pmp = par_alloc(PARD_FBT, mp, sizeof *pmp, NULL);
if (pmp && --pmp->fbt_nentries == 0)
par_free(PARD_FBT, pmp);
}
}
/*
* Now we need to remove this probe from the fbt_probetab.
*/
ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
last = NULL;
hash = fbt_probetab[ndx];
while (hash != fbt) {
ASSERT(hash != NULL);
last = hash;
hash = hash->fbtp_hashnext;
}
if (last != NULL) {
last->fbtp_hashnext = fbt->fbtp_hashnext;
} else {
fbt_probetab[ndx] = fbt->fbtp_hashnext;
}
next = fbt->fbtp_next;
kmem_free(fbt, sizeof (fbt_probe_t));
fbt = next;
} while (fbt != NULL);
}
/*ARGSUSED*/
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
fbt_probe_t *fbt = parg, *next, *hash, *last;
struct modctl *ctl = fbt->fbtp_ctl;
int ndx;
do {
if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
ctl->mod_loaded)) {
((struct module *)
(ctl->mod_mp))->fbt_nentries--;
}
}
/*
* Now we need to remove this probe from the fbt_probetab.
*/
ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
last = NULL;
hash = fbt_probetab[ndx];
while (hash != fbt) {
ASSERT(hash != NULL);
last = hash;
hash = hash->fbtp_hashnext;
}
if (last != NULL) {
last->fbtp_hashnext = fbt->fbtp_hashnext;
} else {
fbt_probetab[ndx] = fbt->fbtp_hashnext;
}
next = fbt->fbtp_next;
kmem_free(fbt, sizeof (fbt_probe_t));
fbt = next;
} while (fbt != NULL);
}
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
fbt_probe_t *fbt = parg, *next, *hash, *last;
modctl_t *ctl;
int ndx;
do {
ctl = fbt->fbtp_ctl;
ctl->fbt_nentries--;
/*
* Now we need to remove this probe from the fbt_probetab.
*/
ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
last = NULL;
hash = fbt_probetab[ndx];
while (hash != fbt) {
ASSERT(hash != NULL);
last = hash;
hash = hash->fbtp_hashnext;
}
if (last != NULL) {
last->fbtp_hashnext = fbt->fbtp_hashnext;
} else {
fbt_probetab[ndx] = fbt->fbtp_hashnext;
}
next = fbt->fbtp_next;
free(fbt, M_FBT);
fbt = next;
} while (fbt != NULL);
}
int
fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval)
{
solaris_cpu_t *cpu;
fbt_probe_t *fbt;
cpu = &solaris_cpu[curcpu];
fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
cpu->cpu_dtrace_caller = addr;
dtrace_probe(fbt->fbtp_id, frame->a0,
frame->a1, frame->a2,
frame->a3, frame->a4);
cpu->cpu_dtrace_caller = 0;
return (fbt->fbtp_savedval);
}
}
return (0);
}
int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
char *modname = opaque;
const char *name = symval->name;
fbt_probe_t *fbt, *retfbt;
uint32_t *instr, *limit;
int popm;
if (strncmp(name, "dtrace_", 7) == 0 &&
strncmp(name, "dtrace_safe_", 12) != 0) {
/*
* Anything beginning with "dtrace_" may be called
* from probe context unless it explicitly indicates
* that it won't be called from probe context by
* using the prefix "dtrace_safe_".
*/
return (0);
}
if (name[0] == '_' && name[1] == '_')
return (0);
instr = (uint32_t *)symval->value;
limit = (uint32_t *)(symval->value + symval->size);
/*
* va_arg functions has first instruction of
* sub sp, sp, #?
*/
if ((*instr & 0xfffff000) == FBT_SUBSP)
instr++;
/*
* check if insn is a pushm with LR
*/
if ((*instr & 0xffff0000) != FBT_PUSHM ||
(*instr & (1 << LR)) == 0)
return (0);
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 2, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_BREAKPOINT;
fbt->fbtp_rval = DTRACE_INVOP_PUSHM;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
popm = FBT_POPM | ((*instr) & 0x3FFF) | 0x8000;
retfbt = NULL;
again:
for (; instr < limit; instr++) {
if (*instr == popm)
break;
else if ((*instr & 0xff000000) == FBT_JUMP) {
uint32_t *target, *start;
int offset;
offset = (*instr & 0xffffff);
offset <<= 8;
offset /= 64;
target = instr + (2 + offset);
start = (uint32_t *)symval->value;
if (target >= limit || target < start)
break;
}
}
if (instr >= limit)
return (0);
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 2, fbt);
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
if ((*instr & 0xff000000) == FBT_JUMP)
fbt->fbtp_rval = DTRACE_INVOP_B;
else
fbt->fbtp_rval = DTRACE_INVOP_POPM;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_BREAKPOINT;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr++;
goto again;
}
static int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
char *modname = opaque;
const char *name = symval->name;
fbt_probe_t *fbt, *retfbt;
int j;
int size;
u_int8_t *instr, *limit;
if (strncmp(name, "dtrace_", 7) == 0 &&
strncmp(name, "dtrace_safe_", 12) != 0) {
/*
* Anything beginning with "dtrace_" may be called
* from probe context unless it explicitly indicates
* that it won't be called from probe context by
* using the prefix "dtrace_safe_".
*/
return (0);
}
if (name[0] == '_' && name[1] == '_')
return (0);
size = symval->size;
instr = (u_int8_t *) symval->value;
limit = (u_int8_t *) symval->value + symval->size;
#ifdef __amd64__
while (instr < limit) {
if (*instr == FBT_PUSHL_EBP)
break;
if ((size = dtrace_instr_size(instr)) <= 0)
break;
instr += size;
}
if (instr >= limit || *instr != FBT_PUSHL_EBP) {
/*
* We either don't save the frame pointer in this
* function, or we ran into some disassembly
* screw-up. Either way, we bail.
*/
return (0);
}
#else
if (instr[0] != FBT_PUSHL_EBP)
return (0);
if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 &&
instr[2] == FBT_MOVL_ESP_EBP1_V0) &&
!(instr[1] == FBT_MOVL_ESP_EBP0_V1 &&
instr[2] == FBT_MOVL_ESP_EBP1_V1))
return (0);
#endif
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
retfbt = NULL;
again:
if (instr >= limit)
return (0);
/*
* If this disassembly fails, then we've likely walked off into
* a jump table or some other unsuitable area. Bail out of the
* disassembly now.
*/
if ((size = dtrace_instr_size(instr)) <= 0)
return (0);
#ifdef __amd64__
/*
* We only instrument "ret" on amd64 -- we don't yet instrument
* ret imm16, largely because the compiler doesn't seem to
* (yet) emit them in the kernel...
*/
if (*instr != FBT_RET) {
instr += size;
goto again;
}
#else
if (!(size == 1 &&
(*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) &&
(*(instr + 1) == FBT_RET ||
*(instr + 1) == FBT_RET_IMM16))) {
instr += size;
goto again;
}
#endif
/*
* We (desperately) want to avoid erroneously instrumenting a
* jump table, especially given that our markers are pretty
* short: two bytes on x86, and just one byte on amd64. To
* determine if we're looking at a true instruction sequence
* or an inline jump table that happens to contain the same
* byte sequences, we resort to some heuristic sleeze: we
* treat this instruction as being contained within a pointer,
* and see if that pointer points to within the body of the
* function. If it does, we refuse to instrument it.
*/
for (j = 0; j < sizeof (uintptr_t); j++) {
caddr_t check = (caddr_t) instr - j;
uint8_t *ptr;
if (check < symval->value)
break;
if (check + sizeof (caddr_t) > (caddr_t)limit)
continue;
ptr = *(uint8_t **)check;
if (ptr >= (uint8_t *) symval->value && ptr < limit) {
instr += size;
goto again;
}
}
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
#ifndef __amd64__
if (*instr == FBT_POPL_EBP) {
fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
} else {
ASSERT(*instr == FBT_LEAVE);
fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
}
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *) symval->value) + 1;
#else
ASSERT(*instr == FBT_RET);
fbt->fbtp_rval = DTRACE_INVOP_RET;
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *) symval->value);
#endif
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr += size;
goto again;
}
static int
fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval, trap_instr_t *tinfo)
{
uintptr_t stack0, stack1, stack2, stack3, stack4;
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
//HERE();
//int dtrace_here = 1;
//if (dtrace_here) printk("fbt_invop:addr=%lx stack=%p eax=%lx\n", addr, stack, (long) rval);
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
//if (dtrace_here) printk("patchpoint: %p rval=%x\n", fbt->fbtp_patchpoint, fbt->fbtp_rval);
if ((uintptr_t)fbt->fbtp_patchpoint != addr)
continue;
/***********************************************/
/* If probe is not enabled, but still */
/* fired, then it *might have* overran. Likely cause is */
/* cpu cache consistency issue - some other */
/* CPU hasnt seen the breakpoint being */
/* removed. Flag it so we can show in */
/* /proc/dtrace/fbt. */
/* Additionally, another provider (eg prov) */
/* is sitting on the same function, so we */
/* have to broadcast to both/all providers. */
/***********************************************/
if (!fbt->fbtp_enabled) {
fbt->fbtp_overrun = TRUE;
}
/***********************************************/
/* Always handle the probe - if we dont, */
/* nobody else will know what to do with */
/* it. Its possible somebody else does want */
/* it, e.g. INSTR, but we cant know who. */
/* Probably need to call all providers, */
/* rather than just the first one. */
/***********************************************/
if (1) {
tinfo->t_opcode = fbt->fbtp_savedval;
//printk("fbt: opc=%p %p\n", tinfo->t_opcode, fbt->fbtp_savedval);
tinfo->t_inslen = fbt->fbtp_inslen;
tinfo->t_modrm = fbt->fbtp_modrm;
if (!tinfo->t_doprobe)
return fbt->fbtp_rval;
fbt->fbtp_fired++;
if (fbt->fbtp_roffset == 0) {
struct pt_regs *ptregs = (struct pt_regs *) stack;
/*
* When accessing the arguments on the stack,
* we must protect against accessing beyond
* the stack. We can safely set NOFAULT here
* -- we know that interrupts are already
* disabled.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
CPU->cpu_dtrace_caller = ptregs->r_pc;
stack0 = ptregs->c_arg0;
stack1 = ptregs->c_arg1;
stack2 = ptregs->c_arg2;
stack3 = ptregs->c_arg3;
stack4 = ptregs->c_arg4;
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
dtrace_probe(fbt->fbtp_id, stack0, stack1,
stack2, stack3, stack4);
CPU->cpu_dtrace_caller = NULL;
} else {
#ifdef __amd64
/*
* On amd64, we instrument the ret, not the
* leave. We therefore need to set the caller
* to assure that the top frame of a stack()
* action is correct.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
CPU->cpu_dtrace_caller = stack[0];
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
#endif
dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
rval, 0, 0, 0);
CPU->cpu_dtrace_caller = NULL;
}
return (fbt->fbtp_rval);
}
}
//HERE();
return (0);
}
static int
fbt_prov_return(pf_info_t *infp, instr_t *instr, int size)
{
fbt_probe_t *fbt;
fbt_probe_t *retfbt = infp->retptr;
# if defined(__i386) || defined(__amd64)
if (*instr == 0xcc)
return 1;
# endif
/***********************************************/
/* Sanity check for bad things happening. */
/***********************************************/
if (fbt_is_patched(infp->name, instr)) {
return 0;
}
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = infp->name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, infp->modname,
infp->name, FBT_RETURN, 3, fbt);
num_probes++;
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
infp->retptr = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = infp->mp; //ctl;
fbt->fbtp_loadcnt = get_refcount(infp->mp);
/***********************************************/
/* Swapped sense of the following ifdef */
/* around so we are consistent. */
/***********************************************/
fbt->fbtp_rval = DTRACE_INVOP_ANY;
#if defined(__amd64)
ASSERT(*instr == FBT_RET);
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)infp->st_value);
#elif defined(__i386)
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)infp->st_value) + 1;
#elif defined(__arm__)
fbt->fbtp_roffset = 0;
#endif
/***********************************************/
/* Save potential overwrite of instruction */
/* and length, because we will need the */
/* entire instruction when we single step */
/* over it. */
/***********************************************/
fbt->fbtp_savedval = *instr;
fbt->fbtp_inslen = size;
fbt->fbtp_type = 1; /* return */
fbt->fbtp_modrm = -1;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = infp->symndx;
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
infp->pmp->fbt_nentries++;
return 1;
}
int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
char *modname = opaque;
const char *name = symval->name;
fbt_probe_t *fbt, *retfbt;
int j;
uint32_t *instr, *limit;
#ifdef __powerpc64__
/*
* PowerPC64 uses '.' prefixes on symbol names, ignore it, but only
* allow symbols with the '.' prefix, so that we don't get the function
* descriptor instead.
*/
if (name[0] == '.')
name++;
else
return (0);
#endif
if (strncmp(name, "dtrace_", 7) == 0 &&
strncmp(name, "dtrace_safe_", 12) != 0) {
/*
* Anything beginning with "dtrace_" may be called
* from probe context unless it explicitly indicates
* that it won't be called from probe context by
* using the prefix "dtrace_safe_".
*/
return (0);
}
if (name[0] == '_' && name[1] == '_')
return (0);
instr = (uint32_t *) symval->value;
limit = (uint32_t *) (symval->value + symval->size);
for (; instr < limit; instr++)
if (*instr == FBT_MFLR_R0)
break;
if (*instr != FBT_MFLR_R0)
return (0);
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, FBT_AFRAMES, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_rval = DTRACE_INVOP_MFLR_R0;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
retfbt = NULL;
again:
if (instr >= limit)
return (0);
/*
* We (desperately) want to avoid erroneously instrumenting a
* jump table. To determine if we're looking at a true instruction
* sequence or an inline jump table that happens to contain the same
* byte sequences, we resort to some heuristic sleeze: we treat this
* instruction as being contained within a pointer, and see if that
* pointer points to within the body of the function. If it does, we
* refuse to instrument it.
*/
{
uint32_t *ptr;
ptr = *(uint32_t **)instr;
if (ptr >= (uint32_t *) symval->value && ptr < limit) {
instr++;
goto again;
}
}
if (*instr != FBT_MTLR_R0) {
instr++;
goto again;
}
instr++;
for (j = 0; j < 12 && instr < limit; j++, instr++) {
if ((*instr == FBT_BCTR) || (*instr == FBT_BLR) ||
FBT_IS_JUMP(*instr))
break;
}
if (!(*instr == FBT_BCTR || *instr == FBT_BLR || FBT_IS_JUMP(*instr)))
goto again;
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, FBT_AFRAMES, fbt);
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
if (*instr == FBT_BCTR)
fbt->fbtp_rval = DTRACE_INVOP_BCTR;
else if (*instr == FBT_BLR)
fbt->fbtp_rval = DTRACE_INVOP_RET;
else
fbt->fbtp_rval = DTRACE_INVOP_JUMP;
fbt->fbtp_roffset =
(uintptr_t)((uint8_t *)instr - (uint8_t *)symval->value);
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr += 4;
goto again;
}
static void
fbt_provide_function(struct modctl *mp, par_module_t *pmp,
char *modname, char *name, uint8_t *st_value,
uint8_t *instr, uint8_t *limit, int symndx)
{
int do_print = FALSE;
int invop = 0;
fbt_probe_t *fbt, *retfbt;
int size;
int modrm;
/***********************************************/
/* Dont let us register anything on the */
/* notifier list(s) we are on, else we will */
/* have recursion trouble. */
/***********************************************/
if (on_notifier_list(instr)) {
printk("fbt_provide_function: Skip %s:%s - on notifier_chain\n",
modname, name);
return;
}
# define UNHANDLED_FBT() if (do_print || dtrace_unhandled) { \
printk("fbt:unhandled instr %s:%p %02x %02x %02x %02x\n", \
name, instr, instr[0], instr[1], instr[2], instr[3]); \
}
/***********************************************/
/* Make sure we dont try and handle data or */
/* bad instructions. */
/***********************************************/
if ((size = dtrace_instr_size_modrm(instr, &modrm)) <= 0)
return;
/***********************************************/
/* Allow us to work on a single function */
/* for debugging/tracing else we generate */
/* too much printk() output and swamp the */
/* log daemon. */
/***********************************************/
// do_print = strncmp(name, "update_process", 9) == NULL;
#ifdef __amd64
// am not happy with 0xe8 CALLR instructions, so disable them for now.
if (*instr == 0xe8) return;
invop = DTRACE_INVOP_ANY;
switch (instr[0] & 0xf0) {
case 0x00:
case 0x10:
case 0x20:
case 0x30:
break;
case 0x40:
if (instr[0] == 0x48 && instr[1] == 0xcf)
return;
break;
case 0x50:
case 0x60:
case 0x70:
case 0x80:
case 0x90:
case 0xa0:
case 0xb0:
break;
case 0xc0:
/***********************************************/
/* We cannot single step an IRET for now, */
/* so just ditch them as potential probes. */
/***********************************************/
if (instr[0] == 0xcf)
return;
break;
case 0xd0:
case 0xe0:
break;
case 0xf0:
/***********************************************/
/* This doesnt work - if we single step a */
/* HLT, well, the kernel doesnt really like */
/* it. */
/***********************************************/
if (instr[0] == 0xf4)
return;
// printk("fbt:F instr %s:%p size=%d %02x %02x %02x %02x %02x %02x\n", name, instr, size, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]);
break;
}
#else
/***********************************************/
/* GCC generates lots of different */
/* assembler functions plus we have inline */
/* assembler to deal with. We break the */
/* opcodes down into groups, which helped */
/* during debugging, or if we need to */
/* single out specific opcodes, but for */
/* now, we can pretty much allow all */
/* opcodes. (Apart from HLT - which I may */
/* get around to fixing). */
/***********************************************/
invop = DTRACE_INVOP_ANY;
switch (instr[0] & 0xf0) {
case 0x00:
case 0x10:
case 0x20:
case 0x30:
case 0x40:
case 0x50:
case 0x60:
case 0x70:
case 0x80:
case 0x90:
case 0xa0:
case 0xb0:
break;
case 0xc0:
/***********************************************/
/* We cannot single step an IRET for now, */
/* so just ditch them as potential probes. */
/***********************************************/
if (instr[0] == 0xcf)
return;
break;
case 0xd0:
case 0xe0:
break;
case 0xf0:
/***********************************************/
/* This doesnt work - if we single step a */
/* HLT, well, the kernel doesnt really like */
/* it. */
/***********************************************/
if (instr[0] == 0xf4)
return;
// printk("fbt:F instr %s:%p size=%d %02x %02x %02x %02x %02x %02x\n", name, instr, size, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]);
break;
}
#endif
/***********************************************/
/* Handle fact we may not be ready to cope */
/* with this instruction yet. */
/***********************************************/
if (invop == 0) {
UNHANDLED_FBT();
return;
}
/***********************************************/
/* Make sure this doesnt overlap another */
/* sym. We are in trouble when this happens */
/* - eg we will mistaken what the emulation */
/* is for, but also, it means something */
/* strange happens, like kernel is reusing */
/* a page (eg for init/exit section of a */
/* module). */
/***********************************************/
if (fbt_is_patched(name, instr))
return;
/*if (modrm >= 0 && (instr[modrm] & 0xc7) == 0x05) {
static char buf[128];
sprintf(buf, "MODRM_%s", name);
name = buf;
}
*/
/***********************************************/
/* Special code here for debugging - we can */
/* make the name of a probe a function of */
/* the instruction, so we can validate and */
/* binary search to find faulty instruction */
/* stepping code in cpu_x86.c */
/***********************************************/
if (fbt_name_opcodes) {
static char buf[128];
if (fbt_name_opcodes == 2)
snprintf(buf, sizeof buf, "x%02x%02x_%s", *instr, instr[1], name);
else
snprintf(buf, sizeof buf, "x%02x_%s", *instr, name);
name = buf;
}
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
num_probes++;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = mp; // ctl;
fbt->fbtp_loadcnt = get_refcount(mp);
fbt->fbtp_rval = invop;
/***********************************************/
/* Save potential overwrite of instruction */
/* and length, because we will need the */
/* entire instruction when we single step */
/* over it. */
/***********************************************/
fbt->fbtp_savedval = *instr;
fbt->fbtp_inslen = size;
//if (modrm >= 0 && (instr[modrm] & 0xc7) == 0x05) printk("modrm %s %p rm=%d\n", name, instr, modrm);
fbt->fbtp_modrm = modrm;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = symndx;
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
if (do_print)
printk("%d:alloc entry-patchpoint: %s %p invop=%x sz=%d %02x %02x %02x\n",
__LINE__,
name,
fbt->fbtp_patchpoint,
fbt->fbtp_rval,
fbt->fbtp_inslen,
instr[0], instr[1], instr[2]);
pmp->fbt_nentries++;
retfbt = NULL;
/***********************************************/
/* This point is part of a loop */
/* (implemented with goto) to find the end */
/* part of a function. Original Solaris */
/* code assumes a single exit, via RET, for */
/* amd64, but is more accepting for i386. */
/* However, with GCC as a compiler a lot */
/* more things can happen - very */
/* specifically, we can have multiple exit */
/* points in a function. So we need to find */
/* each of those. */
/***********************************************/
again:
if (instr >= limit) {
return;
}
/*
* If this disassembly fails, then we've likely walked off into
* a jump table or some other unsuitable area. Bail out of the
* disassembly now.
*/
if ((size = dtrace_instr_size(instr)) <= 0)
return;
//HERE();
#ifdef __amd64
/*
* We only instrument "ret" on amd64 -- we don't yet instrument
* ret imm16, largely because the compiler doesn't seem to
* (yet) emit them in the kernel...
*/
switch (*instr) {
case FBT_RET:
invop = DTRACE_INVOP_RET;
break;
default:
instr += size;
goto again;
}
#else
switch (*instr) {
/* case FBT_POPL_EBP:
invop = DTRACE_INVOP_POPL_EBP;
break;
case FBT_LEAVE:
invop = DTRACE_INVOP_LEAVE;
break;
*/
case FBT_RET:
invop = DTRACE_INVOP_RET;
break;
case FBT_RET_IMM16:
invop = DTRACE_INVOP_RET_IMM16;
break;
/***********************************************/
/* Some functions can end in a jump, e.g. */
/* security_file_permission has an indirect */
/* jmp. Dont think we care too much about */
/* this, but we could handle direct and */
/* indirect jumps out of the function body. */
/***********************************************/
default:
instr += size;
goto again;
}
#endif
/***********************************************/
/* Sanity check for bad things happening. */
/***********************************************/
if (fbt_is_patched(name, instr)) {
instr += size;
goto again;
}
/*
* We have a winner!
*/
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
num_probes++;
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = mp; //ctl;
fbt->fbtp_loadcnt = get_refcount(mp);
/***********************************************/
/* Swapped sense of the following ifdef */
/* around so we are consistent. */
/***********************************************/
fbt->fbtp_rval = invop;
#ifdef __amd64
ASSERT(*instr == FBT_RET);
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)st_value);
#else
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)st_value) + 1;
#endif
/***********************************************/
/* Save potential overwrite of instruction */
/* and length, because we will need the */
/* entire instruction when we single step */
/* over it. */
/***********************************************/
fbt->fbtp_savedval = *instr;
fbt->fbtp_inslen = size;
fbt->fbtp_modrm = -1;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = symndx;
if (do_print) {
printk("%d:alloc return-patchpoint: %s %p: %02x %02x invop=%d\n", __LINE__, name, instr, instr[0], instr[1], invop);
}
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
pmp->fbt_nentries++;
instr += size;
goto again;
}
/*ARGSUSED*/
static void
fbt_provide_module(void *arg, struct modctl *ctl)
{
struct module *mp = ctl->mod_mp;
char *str = mp->strings;
int nsyms = mp->nsyms;
Shdr *symhdr = mp->symhdr;
char *modname = ctl->mod_modname;
char *name;
fbt_probe_t *fbt, *retfbt;
size_t symsize;
int i, size;
/*
* Employees of dtrace and their families are ineligible. Void
* where prohibited.
*/
if (strcmp(modname, "dtrace") == 0)
return;
if (ctl->mod_requisites != NULL) {
struct modctl_list *list;
list = (struct modctl_list *)ctl->mod_requisites;
for (; list != NULL; list = list->modl_next) {
if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
return;
}
}
/*
* KMDB is ineligible for instrumentation -- it may execute in
* any context, including probe context.
*/
if (strcmp(modname, "kmdbmod") == 0)
return;
if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) {
/*
* If this module doesn't (yet) have its string or symbol
* table allocated, clear out.
*/
return;
}
symsize = symhdr->sh_entsize;
if (mp->fbt_nentries) {
/*
* This module has some FBT entries allocated; we're afraid
* to screw with it.
*/
return;
}
for (i = 1; i < nsyms; i++) {
uint8_t *instr, *limit;
Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
int j;
if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
/*
* Weak symbols are not candidates. This could be made to
* work (where weak functions and their underlying function
* appear as two disjoint probes), but it's not simple.
*/
if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
continue;
name = str + sym->st_name;
if (strstr(name, "dtrace_") == name &&
strstr(name, "dtrace_safe_") != name) {
/*
* Anything beginning with "dtrace_" may be called
* from probe context unless it explitly indicates
* that it won't be called from probe context by
* using the prefix "dtrace_safe_".
*/
continue;
}
if (strstr(name, "kdi_") == name ||
strstr(name, "_kdi_") != NULL) {
/*
* Any function name beginning with "kdi_" or
* containing the string "_kdi_" is a part of the
* kernel debugger interface and may be called in
* arbitrary context -- including probe context.
*/
continue;
}
/*
* Due to 4524008, _init and _fini may have a bloated st_size.
* While this bug was fixed quite some time ago, old drivers
* may be lurking. We need to develop a better solution to
* this problem, such that correct _init and _fini functions
* (the vast majority) may be correctly traced. One solution
* may be to scan through the entire symbol table to see if
* any symbol overlaps with _init. If none does, set a bit in
* the module structure that this module has correct _init and
* _fini sizes. This will cause some pain the first time a
* module is scanned, but at least it would be O(N) instead of
* O(N log N)...
*/
if (strcmp(name, "_init") == 0)
continue;
if (strcmp(name, "_fini") == 0)
continue;
/*
* In order to be eligible, the function must begin with the
* following sequence:
*
* pushl %esp
* movl %esp, %ebp
*
* Note that there are two variants of encodings that generate
* the movl; we must check for both. For 64-bit, we would
* normally insist that a function begin with the following
* sequence:
*
* pushq %rbp
* movq %rsp, %rbp
*
* However, the compiler for 64-bit often splits these two
* instructions -- and the first instruction in the function
* is often not the pushq. As a result, on 64-bit we look
* for any "pushq %rbp" in the function and we instrument
* this with a breakpoint instruction.
*/
instr = (uint8_t *)sym->st_value;
limit = (uint8_t *)(sym->st_value + sym->st_size);
#ifdef __amd64
while (instr < limit) {
if (*instr == FBT_PUSHL_EBP)
break;
if ((size = dtrace_instr_size(instr)) <= 0)
break;
instr += size;
}
if (instr >= limit || *instr != FBT_PUSHL_EBP) {
/*
* We either don't save the frame pointer in this
* function, or we ran into some disassembly
* screw-up. Either way, we bail.
*/
continue;
}
#else
if (instr[0] != FBT_PUSHL_EBP)
continue;
if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 &&
instr[2] == FBT_MOVL_ESP_EBP1_V0) &&
!(instr[1] == FBT_MOVL_ESP_EBP0_V1 &&
instr[2] == FBT_MOVL_ESP_EBP1_V1))
continue;
#endif
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = ctl;
fbt->fbtp_loadcnt = ctl->mod_loadcnt;
fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = i;
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
mp->fbt_nentries++;
retfbt = NULL;
again:
if (instr >= limit)
continue;
/*
* If this disassembly fails, then we've likely walked off into
* a jump table or some other unsuitable area. Bail out of the
* disassembly now.
*/
if ((size = dtrace_instr_size(instr)) <= 0)
continue;
#ifdef __amd64
/*
* We only instrument "ret" on amd64 -- we don't yet instrument
* ret imm16, largely because the compiler doesn't seem to
* (yet) emit them in the kernel...
*/
if (*instr != FBT_RET) {
instr += size;
goto again;
}
#else
if (!(size == 1 &&
(*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) &&
(*(instr + 1) == FBT_RET ||
*(instr + 1) == FBT_RET_IMM16))) {
instr += size;
goto again;
}
#endif
/*
* We (desperately) want to avoid erroneously instrumenting a
* jump table, especially given that our markers are pretty
* short: two bytes on x86, and just one byte on amd64. To
* determine if we're looking at a true instruction sequence
* or an inline jump table that happens to contain the same
* byte sequences, we resort to some heuristic sleeze: we
* treat this instruction as being contained within a pointer,
* and see if that pointer points to within the body of the
* function. If it does, we refuse to instrument it.
*/
for (j = 0; j < sizeof (uintptr_t); j++) {
uintptr_t check = (uintptr_t)instr - j;
uint8_t *ptr;
if (check < sym->st_value)
break;
if (check + sizeof (uintptr_t) > (uintptr_t)limit)
continue;
ptr = *(uint8_t **)check;
if (ptr >= (uint8_t *)sym->st_value && ptr < limit) {
instr += size;
goto again;
}
}
/*
* We have a winner!
*/
fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = ctl;
fbt->fbtp_loadcnt = ctl->mod_loadcnt;
#ifndef __amd64
if (*instr == FBT_POPL_EBP) {
fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
} else {
ASSERT(*instr == FBT_LEAVE);
fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
}
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)sym->st_value) + 1;
#else
ASSERT(*instr == FBT_RET);
fbt->fbtp_rval = DTRACE_INVOP_RET;
fbt->fbtp_roffset =
(uintptr_t)(instr - (uint8_t *)sym->st_value);
#endif
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt->fbtp_symndx = i;
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
mp->fbt_nentries++;
instr += size;
goto again;
}
}
int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
fbt_probe_t *fbt, *retfbt;
uint32_t *instr, *limit;
const char *name;
char *modname;
int patchval;
int rval;
modname = opaque;
name = symval->name;
/* Check if function is excluded from instrumentation */
if (fbt_excluded(name))
return (0);
instr = (uint32_t *)(symval->value);
limit = (uint32_t *)(symval->value + symval->size);
/* Look for sd operation */
for (; instr < limit; instr++) {
/* Look for a non-compressed store of ra to sp */
if (match_opcode(*instr, (MATCH_SD | RS2_RA | RS1_SP),
(MASK_SD | RS2_MASK | RS1_MASK))) {
rval = DTRACE_INVOP_SD;
patchval = FBT_PATCHVAL;
break;
}
/* Look for a 'C'-compressed store of ra to sp. */
if (check_c_sdsp(&instr)) {
rval = DTRACE_INVOP_C_SDSP;
patchval = FBT_C_PATCHVAL;
break;
}
}
if (instr >= limit)
return (0);
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = patchval;
fbt->fbtp_rval = rval;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
retfbt = NULL;
again:
for (; instr < limit; instr++) {
/* Look for non-compressed return */
if (match_opcode(*instr, (MATCH_JALR | (X_RA << RS1_SHIFT)),
(MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) {
rval = DTRACE_INVOP_RET;
patchval = FBT_PATCHVAL;
break;
}
/* Look for 'C'-compressed return */
if (check_c_ret(&instr)) {
rval = DTRACE_INVOP_C_RET;
patchval = FBT_C_PATCHVAL;
break;
}
}
if (instr >= limit)
return (0);
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
} else {
retfbt->fbtp_probenext = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
fbt->fbtp_rval = rval;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = patchval;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr++;
goto again;
}
int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
fbt_probe_t *fbt, *retfbt;
uint32_t *instr, *limit;
const char *name;
char *modname;
modname = opaque;
name = symval->name;
/* Check if function is excluded from instrumentation */
if (fbt_excluded(name))
return (0);
instr = (uint32_t *)(symval->value);
limit = (uint32_t *)(symval->value + symval->size);
/* Look for store double to ra register */
for (; instr < limit; instr++) {
if ((*instr & LDSD_RA_SP_MASK) == SD_RA_SP)
break;
}
if (instr >= limit)
return (0);
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_rval = DTRACE_INVOP_SD;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
retfbt = NULL;
again:
for (; instr < limit; instr++) {
if ((*instr & LDSD_RA_SP_MASK) == LD_RA_SP) {
break;
}
}
if (instr >= limit)
return (0);
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
} else {
retfbt->fbtp_next = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
fbt->fbtp_rval = DTRACE_INVOP_LD;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = FBT_PATCHVAL;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr++;
goto again;
}
static int
fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval, trap_instr_t *tinfo)
{
uintptr_t stack0, stack1, stack2, stack3, stack4;
fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
//HERE();
if (dtrace_here) printk("fbt_invop:addr=%lx stack=%p eax=%lx\n", addr, stack, (long) rval);
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if (dtrace_here) printk("patchpoint: %p rval=%x\n", fbt->fbtp_patchpoint, fbt->fbtp_rval);
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
tinfo->t_opcode = fbt->fbtp_savedval;
tinfo->t_inslen = fbt->fbtp_inslen;
tinfo->t_modrm = fbt->fbtp_modrm;
if (!tinfo->t_doprobe)
return fbt->fbtp_rval;
if (fbt->fbtp_roffset == 0) {
/*
* When accessing the arguments on the stack,
* we must protect against accessing beyond
* the stack. We can safely set NOFAULT here
* -- we know that interrupts are already
* disabled.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
CPU->cpu_dtrace_caller = stack[0];
stack0 = stack[1];
stack1 = stack[2];
stack2 = stack[3];
stack3 = stack[4];
stack4 = stack[5];
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
dtrace_probe(fbt->fbtp_id, stack0, stack1,
stack2, stack3, stack4);
CPU->cpu_dtrace_caller = NULL;
} else {
#ifdef __amd64
/*
* On amd64, we instrument the ret, not the
* leave. We therefore need to set the caller
* to assure that the top frame of a stack()
* action is correct.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
CPU->cpu_dtrace_caller = stack[0];
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
CPU_DTRACE_BADADDR);
#endif
dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
rval, 0, 0, 0);
CPU->cpu_dtrace_caller = NULL;
}
return (fbt->fbtp_rval);
}
}
//HERE();
return (0);
}
