void
dtrace_vtime_disable(void)
{
dtrace_vtime_state_t state, nstate = 0;
do {
state = dtrace_vtime_active;
switch (state) {
case DTRACE_VTIME_ACTIVE:
nstate = DTRACE_VTIME_INACTIVE;
break;
case DTRACE_VTIME_ACTIVE_TNF:
nstate = DTRACE_VTIME_INACTIVE_TNF;
break;
case DTRACE_VTIME_INACTIVE:
case DTRACE_VTIME_INACTIVE_TNF:
panic("DTrace virtual time already disabled");
/*NOTREACHED*/
}
} while (dtrace_cas32((uint32_t *)&dtrace_vtime_active,
state, nstate) != state);
}
void
dtrace_xcall2(processorid_t cpu, dtrace_xcall_t func, void *arg)
{ int c;
int cpu_id = smp_processor_id();
int cpus_todo = 0;
# if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 24)
typedef struct cpumask cpumask_t;
//#define cpu_set(c, mask) cpumask_set_cpu(c, &(mask))
//#define cpus_clear(mask) cpumask_clear(&mask)
# endif
cpumask_t mask;
/***********************************************/
/* If we had an internal panic, stop doing */
/* xcalls. Shouldnt happen, but useful */
/* during debugging so we can diagnose what */
/* caused the panic. */
/***********************************************/
if (dtrace_shutdown)
return;
/***********************************************/
/* Special case - just 'us'. */
/***********************************************/
cnt_xcall1++;
if (cpu_id == cpu) {
local_irq_disable();
//dtrace_printf("[%d] sole cnt=%lu\n", smp_processor_id(), cnt_xcall1);
func(arg);
local_irq_enable();
return;
}
/***********************************************/
/* Set up the cpu mask to do just the */
/* relevant cpu. */
/***********************************************/
if (cpu != DTRACE_CPUALL) {
//dtrace_printf("just me %d %d\n", cpu_id, cpu);
cpu = 1 << cpu;
}
//dtrace_printf("xcall %d f=%p\n", cpu_id, func);
cnt_xcall2++;
if (xcall_levels[cpu_id]++)
cnt_xcall3++;
/***********************************************/
/* Set up the rendezvous with the other */
/* targetted cpus. We use a nearly square */
/* NCPU*NCPU matrix to allow for any cpu to */
/* wait for any other. We have two slots */
/* per cpu - because we may be in an */
/* interrupt. */
/* */
/* The interrupt slave will service all */
/* queued calls - sometimes it will be */
/* lucky and see multiple, especially if we */
/* are heavily loaded. */
/***********************************************/
cpus_clear(mask);
for (c = 0; c < nr_cpus; c++) {
struct xcalls *xc = &xcalls[cpu_id][c];
unsigned int cnt;
/***********************************************/
/* Dont set ourselves - we dont want our */
/* cpu to be taking an IPI interrupt and */
/* doing the work twice. We inline */
/* ourselves below. */
/***********************************************/
if ((cpu & (1 << c)) == 0 || c == cpu_id) {
continue;
}
/***********************************************/
/* Is this safe? We want to avoid an IPI */
/* call if the other cpu is idle/not doing */
/* dtrace work. If thats the case and we */
/* are calling dtrace_sync, then we can */
/* avoid the xcall. */
/***********************************************/
if ((void *) func == (void *) dtrace_sync_func &&
cpu_core[c].cpuc_probe_level == 0) {
cpu &= ~(1 << c);
cnt_xcall7++;
continue;
}
//dtrace_printf("xcall %p\n", func);
xc->xc_func = func;
xc->xc_arg = arg;
/***********************************************/
/* Spinlock in case the interrupt hasnt */
/* fired. This should be very rare, and */
/* when it happens, we would be hanging for */
/* 100m iterations (about 1s). We reduce */
/* the chance of a hit by using the */
/* NCPU*NCPU*2 array approach. These things */
/* happen when buffers are full or user is */
/* ^C-ing dtrace. */
/***********************************************/
for (cnt = 0; dtrace_cas32((void *) &xc->xc_state, XC_WORKING, XC_WORKING) == XC_WORKING; cnt++) {
/***********************************************/
/* Avoid noise for tiny windows. */
/***********************************************/
if ((cnt == 0 && xcall_debug) || !(xcall_debug && cnt == 50)) {
dtrace_printf("[%d] cpu%d in wrong state (state=%d)\n",
smp_processor_id(), c, xc->xc_state);
}
// xcall_slave2();
if (cnt == 100 * 1000 * 1000) {
dtrace_printf("[%d] cpu%d - busting lock\n",
smp_processor_id(), c);
break;
}
}
if ((cnt && xcall_debug) || (!xcall_debug && cnt > 50)) {
dtrace_printf("[%d] cpu%d in wrong state (state=%d) %u cycles\n",
smp_processor_id(), c, xc->xc_state, cnt);
}
/***********************************************/
/* As soon as we set xc_state and BEFORE */
/* the apic call, the cpu may see the */
/* change since it may be taking an IPI */
/* interrupt for someone else. We need to */
/* be careful with barriers (I think - */
/* although the clflush/wmb may be */
/* redundant). */
/***********************************************/
xc->xc_state = XC_WORKING;
// clflush(&xc->xc_state);
// smp_wmb();
cpu_set(c, mask);
cpus_todo++;
}
smp_mb();
/***********************************************/
/* Now tell the other cpus to do some work. */
/***********************************************/
if (cpus_todo)
send_ipi_interrupt(&mask, ipi_vector);
/***********************************************/
/* Check for ourselves. */
/***********************************************/
if (cpu & (1 << cpu_id)) {
func(arg);
}
if (xcall_debug)
dtrace_printf("[%d] getting ready.... (%ld) mask=%x func=%p\n", smp_processor_id(), cnt_xcall1, *(int *) &mask, func);
/***********************************************/
/* Wait for the cpus we invoked the IPI on. */
/* Cycle thru the cpus, to avoid mutual */
/* deadlock between one cpu trying to call */
/* us whilst we are calling them. */
/***********************************************/
while (cpus_todo > 0) {
for (c = 0; c < nr_cpus && cpus_todo > 0; c++) {
xcall_slave2();
if (c == cpu_id || (cpu & (1 << c)) == 0)
continue;
/***********************************************/
/* Wait a little while for this cpu to */
/* respond before going on to the next one. */
/***********************************************/
if (ack_wait(c, 100)) {
cpus_todo--;
cpu &= ~(1 << c);
}
}
}
// smp_mb();
xcall_levels[cpu_id]--;
}
int
ack_wait(int c, int attempts)
{
unsigned long cnt = 0;
int cnt1 = 0;
volatile struct xcalls *xc = &xcalls[smp_processor_id()][c];
/***********************************************/
/* Avoid holding on to a stale cache line. */
/***********************************************/
while (dtrace_cas32((void *) &xc->xc_state, XC_WORKING, XC_WORKING) != XC_IDLE) {
if (attempts-- <= 0)
return 0;
barrier();
/***********************************************/
/* Be HT friendly. */
/***********************************************/
// smt_pause();
cnt_xcall6++;
/***********************************************/
/* Keep track of the max. */
/***********************************************/
if (cnt > cnt_xcall5)
cnt_xcall5 = cnt;
/***********************************************/
/* On my Dual Core 2.26GHz system, we are */
/* seeing counters in the range of hundreds */
/* to maybe 2,000,000 for more extreme */
/* cases. (This is inside a VM). During */
/* debugging, we found problems with the */
/* two cores not seeing each other -- */
/* possibly because I wasnt doing the right */
/* things to ensure memory barriers were in */
/* place. */
/* */
/* We dont want to wait forever because */
/* that will crash/hang your machine, but */
/* we do need to give up if its taken far */
/* too long. */
/***********************************************/
// if (cnt++ == 50 * 1000 * 1000UL) {
if (cnt++ == 1 * 1000 * 1000UL) {
cnt = 0;
cnt_xcall4++;
if (cnt1 == 0) {
/***********************************************/
/* Looks like we are having trouble getting */
/* the interrupt, so try for an NMI. */
/***********************************************/
cpumask_t mask;
cpus_clear(mask);
cpu_set(c, mask);
// nmi_masks[c] = 1;
// send_ipi_interrupt(&mask, 2); //NMI_VECTOR);
}
if (1) {
// set_console_on(1);
dtrace_printf("ack_wait cpu=%d xcall %staking too long! c=%d [xcall1=%lu]\n",
smp_processor_id(),
cnt1 ? "STILL " : "",
c, cnt_xcall1);
//dump_stack();
// set_console_on(0);
}
if (cnt1++ > 3) {
dump_xcalls();
dtrace_linux_panic("xcall taking too long");
break;
}
}
}
if (xcall_debug) {
dtrace_printf("[%d] ack_wait finished c=%d cnt=%lu (max=%lu)\n", smp_processor_id(), c, cnt, cnt_xcall5);
}
return 1;
}
