void MT_trace_irq_on(void)
{
struct sched_stop_event *e;
e = & __raw_get_cpu_var(IRQ_disable_mon);
e->last_ts = e->cur_ts;
e->cur_ts = 0 ;
e->last_te = sched_clock();
}
// SoftIRQ monitor
void mt_trace_SoftIRQ_start(int sq_num)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(SoftIRQ_mon);
b->cur_ts = sched_clock();
b->cur_event = (unsigned long)sq_num;
}
static void do_init_timer(struct timer_list *timer, unsigned int flags,
const char *name)
{
struct tvec_base *base = __raw_get_cpu_var(tvec_bases);
timer->entry.next = NULL;
timer->base = (void *)((unsigned long)base | flags);
timer->slack = -1;
}
static void at91_unmute_pic(void)
{
struct at91_gpio_chip *prev, *chip = NULL;
unsigned long muted;
unsigned i;
at91_sys_write(AT91_AIC_IECR, __raw_get_cpu_var(__ipipe_muted_irqs)[0]);
for (i = 0; i < gpio_banks; i++) {
prev = chip;
chip = &gpio_chip[i];
if (!(*chip->nonroot_gpios))
continue;
muted = __raw_get_cpu_var(__ipipe_muted_irqs)
[i + PIN_BASE / 32];
__raw_writel(muted, chip->regbase + PIO_IER);
}
}
/**
* hrtimer_get_res - get the timer resolution for a clock
* @which_clock: which clock to query
* @tp: pointer to timespec variable to store the resolution
*
* Store the resolution of the clock selected by @which_clock in the
* variable pointed to by @tp.
*/
int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
{
struct hrtimer_cpu_base *cpu_base;
cpu_base = &__raw_get_cpu_var(hrtimer_bases);
*tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution);
return 0;
}
static void watchdog_disable(unsigned int cpu)
{
struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
watchdog_set_prio(SCHED_NORMAL, 0);
hrtimer_cancel(hrtimer);
/* disable the perf event */
watchdog_nmi_disable(cpu);
}
// SoftTimer monitor
void mt_trace_sft_start(void *func)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(sft_mon);
b->cur_ts = sched_clock();
b->cur_event = (unsigned long)func;
b->cur_count++;
}
// IRQ off monitor
void MT_trace_irq_off(void)
{
struct sched_stop_event *e;
struct stack_trace *trace;
e = & __raw_get_cpu_var(IRQ_disable_mon);
e->cur_ts = sched_clock();
/*save timestap*/
__raw_get_cpu_var(TS_irq_off) = sched_clock();
trace = &__raw_get_cpu_var(MT_stack_trace);
/*save backtraces*/
trace->nr_entries = 0;
trace->max_entries = MAX_STACK_TRACE_DEPTH; //32
trace->skip = 0;
save_stack_trace_tsk(current, trace);
}
void mt_trace_ISR_end(int irq)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(ISR_mon);
WARN_ON(b->cur_event != irq);
b->last_event = b->cur_event;
b->last_ts = b->cur_ts;
b->last_te = sched_clock();
b->cur_event = 0;
b->cur_ts = 0;
event_duration_check(b);
aee_rr_rec_last_irq_exit(smp_processor_id(), irq, b->last_te);
//reset HRTimer function counter
b = & __raw_get_cpu_var(hrt_mon);
reset_event_count(b);
}
// ISR monitor
void mt_trace_ISR_start(int irq)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(ISR_mon);
b->cur_ts = sched_clock();
b->cur_event = (unsigned long)irq;
aee_rr_rec_last_irq_enter(smp_processor_id(), irq, b->cur_ts);
}
static inline unsigned int get_and_clear_irq_fired(void)
{
/* This is potentially not atomic since we might migrate if
* preemptions are not disabled. As a tradeoff between
* accuracy and tracing overheads, this seems acceptable.
* If it proves to be a problem, then one could add a callback
* from the migration code to invalidate irq_fired_count.
*/
return atomic_xchg(&__raw_get_cpu_var(irq_fired_count), 0);
}
void mt_trace_SoftIRQ_end(int sq_num)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(SoftIRQ_mon);
WARN_ON(b->cur_event != sq_num);
b->last_event = b->cur_event;
b->last_ts = b->cur_ts;
b->last_te = sched_clock();
b->cur_event = 0;
b->cur_ts = 0;
event_duration_check(b);
//reset soft timer function counter
b = & __raw_get_cpu_var(sft_mon);
reset_event_count(b);
//reset tasklet function counter
b = & __raw_get_cpu_var(tasklet_mon);
reset_event_count(b);
}
void touch_nmi_watchdog(void)
{
/*
* Using __raw here because some code paths have
* preemption enabled. If preemption is enabled
* then interrupts should be enabled too, in which
* case we shouldn't have to worry about the watchdog
* going off.
*/
__raw_get_cpu_var(watchdog_nmi_touch) = true;
touch_softlockup_watchdog();
}
static irqreturn_t timer_handler(int irq, void *dev_id)
{
struct clock_event_device *evt = *(struct clock_event_device **)dev_id;
//#ifdef CONFIG_MT_SCHED_MONITOR
#if 0
// add timer event tracer for wdt debug
__raw_get_cpu_var(local_timer_ts) = sched_clock();
if (generic_timer_ack()) {
evt->event_handler(evt);
__raw_get_cpu_var(local_timer_te) = sched_clock();
return IRQ_HANDLED;
}
__raw_get_cpu_var(local_timer_te) = sched_clock();
return IRQ_NONE;
#else
if (generic_timer_ack()) {
evt->event_handler(evt);
return IRQ_HANDLED;
}
return IRQ_NONE;
#endif
}
void mt_trace_sft_end(void *func)
{
struct sched_block_event *b;
b = & __raw_get_cpu_var(sft_mon);
WARN_ON(b->cur_event != (unsigned long)func);
b->last_event = b->cur_event;
b->last_ts = b->cur_ts;
b->last_te = sched_clock();
b->cur_event = 0;
b->cur_ts = 0;
event_duration_check(b);
}
void __sched io_schedule(void)
{
#ifndef DDE_LINUX
struct rq *rq = &__raw_get_cpu_var(runqueues);
delayacct_blkio_start();
atomic_inc(&rq->nr_iowait);
#endif
schedule();
#ifndef DDE_LINUX
atomic_dec(&rq->nr_iowait);
delayacct_blkio_end();
#endif
}
static void at91_mute_pic(void)
{
struct at91_gpio_chip *prev, *chip = NULL;
unsigned long unmasked, muted;
unsigned i;
for (i = 0; i < gpio_banks; i++) {
prev = chip;
chip = &gpio_chip[i];
if (!(*chip->nonroot_gpios))
continue;
unmasked = __raw_readl(chip->regbase + PIO_IMR);
muted = unmasked & __ipipe_irqbits[i + 1];
__raw_get_cpu_var(__ipipe_muted_irqs)
[i + PIN_BASE / 32] = muted;
__raw_writel(muted, chip->regbase + PIO_IDR);
}
unmasked = at91_sys_read(AT91_AIC_IMR);
muted = unmasked & __ipipe_irqbits[0];
__raw_get_cpu_var(__ipipe_muted_irqs)[0] = muted;
at91_sys_write(AT91_AIC_IDCR, muted);
}
static void restart_watchdog_hrtimer(void *info)
{
struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
int ret;
/*
* No need to cancel and restart hrtimer if it is currently executing
* because it will reprogram itself with the new period now.
* We should never see it unqueued here because we are running per-cpu
* with interrupts disabled.
*/
ret = hrtimer_try_to_cancel(hrtimer);
if (ret == 1)
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
HRTIMER_MODE_REL_PINNED);
}
static void
__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{
struct cpumask *cpus_in_cluster_ptr;
struct cpumask *ipi_mask_ptr;
unsigned int cpu, this_cpu;
unsigned long flags;
u32 dest;
x2apic_wrmsr_fence();
local_irq_save(flags);
this_cpu = smp_processor_id();
/*
* We are to modify mask, so we need an own copy
* and be sure it's manipulated with irq off.
*/
ipi_mask_ptr = __raw_get_cpu_var(ipi_mask);
cpumask_copy(ipi_mask_ptr, mask);
/*
* The idea is to send one IPI per cluster.
*/
for_each_cpu(cpu, ipi_mask_ptr) {
unsigned long i;
cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
dest = 0;
/* Collect cpus in cluster. */
for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
dest |= per_cpu(x86_cpu_to_logical_apicid, i);
}
if (!dest)
continue;
__x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
/*
* Cluster sibling cpus should be discared now so
* we would not send IPI them second time.
*/
cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
}
static void watchdog_enable(unsigned int cpu)
{
struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
/* kick off the timer for the hardlockup detector */
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer->function = watchdog_timer_fn;
/* Enable the perf event */
watchdog_nmi_enable(cpu);
/* done here because hrtimer_start can only pin to smp_processor_id() */
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
HRTIMER_MODE_REL_PINNED);
/* initialize timestamp */
watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
__touch_watchdog();
}
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
struct hrtimer_cpu_base *cpu_base;
memset(timer, 0, sizeof(struct hrtimer));
cpu_base = &__raw_get_cpu_var(hrtimer_bases);
if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
clock_id = CLOCK_MONOTONIC;
timer->base = &cpu_base->clock_base[clock_id];
hrtimer_init_timer_hres(timer);
#ifdef CONFIG_TIMER_STATS
timer->start_site = NULL;
timer->start_pid = -1;
memset(timer->start_comm, 0, TASK_COMM_LEN);
#endif
}
void MT_trace_hardirqs_off(void)
{
if(unlikely(__raw_get_cpu_var(mtsched_mon_enabled) & 0x2)){
if( 0 == current->pid) /* Ignore swap thread */
return;
if( __raw_get_cpu_var(MT_trace_in_sched))
return;
if( __raw_get_cpu_var(MT_trace_in_resume_console))
return;
if(__raw_get_cpu_var(MT_tracing_cpu) == 0){
MT_trace_irq_off();
__raw_get_cpu_var(t_irq_off) = sched_clock();
}
__raw_get_cpu_var(MT_tracing_cpu) = 1;
}
}
void wake_up_klogd(void)
{
if (waitqueue_active(&log_wait))
__raw_get_cpu_var(printk_pending) = 1;
}
/*
* __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
* interrupt protection log is maintained here for each domain. Hw
* interrupts are off on entry.
*/
int __ipipe_handle_irq(struct pt_regs *regs)
{
struct ipipe_domain *this_domain, *next_domain;
int irq, vector = regs->orig_ax;
struct list_head *head, *pos;
struct pt_regs *tick_regs;
int m_ack;
if (vector < 0) {
irq = __get_cpu_var(vector_irq)[~vector];
BUG_ON(irq < 0);
m_ack = 0;
} else { /* This is a self-triggered one. */
irq = vector;
m_ack = 1;
}
this_domain = ipipe_current_domain;
if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
head = &this_domain->p_link;
else {
head = __ipipe_pipeline.next;
next_domain = list_entry(head, struct ipipe_domain, p_link);
if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
if (!m_ack && next_domain->irqs[irq].acknowledge)
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
__ipipe_dispatch_wired(next_domain, irq);
goto finalize_nosync;
}
}
/* Ack the interrupt. */
pos = head;
while (pos != &__ipipe_pipeline) {
next_domain = list_entry(pos, struct ipipe_domain, p_link);
if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
__ipipe_set_irq_pending(next_domain, irq);
if (!m_ack && next_domain->irqs[irq].acknowledge) {
next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
m_ack = 1;
}
}
if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
break;
pos = next_domain->p_link.next;
}
/*
* If the interrupt preempted the head domain, then do not
* even try to walk the pipeline, unless an interrupt is
* pending for it.
*/
if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
!__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
goto finalize_nosync;
/*
* Now walk the pipeline, yielding control to the highest
* priority domain that has pending interrupt(s) or
* immediately to the current domain if the interrupt has been
* marked as 'sticky'. This search does not go beyond the
* current domain in the pipeline.
*/
__ipipe_walk_pipeline(head);
finalize_nosync:
/*
* Given our deferred dispatching model for regular IRQs, we
* only record CPU regs for the last timer interrupt, so that
* the timer handler charges CPU times properly. It is assumed
* that other interrupt handlers don't actually care for such
* information.
*/
if (irq == __ipipe_hrtimer_irq) {
tick_regs = &__raw_get_cpu_var(__ipipe_tick_regs);
tick_regs->flags = regs->flags;
tick_regs->cs = regs->cs;
tick_regs->ip = regs->ip;
tick_regs->bp = regs->bp;
#ifdef CONFIG_X86_64
tick_regs->ss = regs->ss;
tick_regs->sp = regs->sp;
#endif
if (!__ipipe_root_domain_p)
tick_regs->flags &= ~X86_EFLAGS_IF;
}
if (user_mode(regs) && (current->ipipe_flags & PF_EVTRET) != 0) {
current->ipipe_flags &= ~PF_EVTRET;
__ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
}
if (!__ipipe_root_domain_p ||
test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
return 0;
return 1;
}
static unsigned long iommu_range_alloc(struct device *dev,
struct iommu_table *tbl,
unsigned long npages,
unsigned long *handle,
unsigned long mask,
unsigned int align_order)
{
unsigned long n, end, start;
unsigned long limit;
int largealloc = npages > 15;
int pass = 0;
unsigned long align_mask;
unsigned long boundary_size;
unsigned long flags;
unsigned int pool_nr;
struct iommu_pool *pool;
align_mask = 0xffffffffffffffffl >> (64 - align_order);
/* This allocator was derived from x86_64's bit string search */
/* Sanity check */
if (unlikely(npages == 0)) {
if (printk_ratelimit())
WARN_ON(1);
return DMA_ERROR_CODE;
}
if (should_fail_iommu(dev))
return DMA_ERROR_CODE;
/*
* We don't need to disable preemption here because any CPU can
* safely use any IOMMU pool.
*/
pool_nr = __raw_get_cpu_var(iommu_pool_hash) & (tbl->nr_pools - 1);
if (largealloc)
pool = &(tbl->large_pool);
else
pool = &(tbl->pools[pool_nr]);
spin_lock_irqsave(&(pool->lock), flags);
again:
if ((pass == 0) && handle && *handle &&
(*handle >= pool->start) && (*handle < pool->end))
start = *handle;
else
start = pool->hint;
limit = pool->end;
/* The case below can happen if we have a small segment appended
* to a large, or when the previous alloc was at the very end of
* the available space. If so, go back to the initial start.
*/
if (start >= limit)
start = pool->start;
if (limit + tbl->it_offset > mask) {
limit = mask - tbl->it_offset + 1;
/* If we're constrained on address range, first try
* at the masked hint to avoid O(n) search complexity,
* but on second pass, start at 0 in pool 0.
*/
if ((start & mask) >= limit || pass > 0) {
spin_unlock(&(pool->lock));
pool = &(tbl->pools[0]);
spin_lock(&(pool->lock));
start = pool->start;
} else {
start &= mask;
}
}
if (dev)
boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1 << tbl->it_page_shift);
else
boundary_size = ALIGN(1UL << 32, 1 << tbl->it_page_shift);
/* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
n = iommu_area_alloc(tbl->it_map, limit, start, npages, tbl->it_offset,
boundary_size >> tbl->it_page_shift, align_mask);
if (n == -1) {
if (likely(pass == 0)) {
/* First try the pool from the start */
pool->hint = pool->start;
pass++;
goto again;
} else if (pass <= tbl->nr_pools) {
/* Now try scanning all the other pools */
spin_unlock(&(pool->lock));
pool_nr = (pool_nr + 1) & (tbl->nr_pools - 1);
pool = &tbl->pools[pool_nr];
spin_lock(&(pool->lock));
pool->hint = pool->start;
pass++;
goto again;
} else {
/* Give up */
spin_unlock_irqrestore(&(pool->lock), flags);
return DMA_ERROR_CODE;
}
}
end = n + npages;
/* Bump the hint to a new block for small allocs. */
if (largealloc) {
/* Don't bump to new block to avoid fragmentation */
pool->hint = end;
} else {
/* Overflow will be taken care of at the next allocation */
pool->hint = (end + tbl->it_blocksize - 1) &
~(tbl->it_blocksize - 1);
}
/* Update handle for SG allocations */
if (handle)
*handle = end;
spin_unlock_irqrestore(&(pool->lock), flags);
return n;
}
void touch_softlockup_watchdog_sync(void)
{
__raw_get_cpu_var(softlockup_touch_sync) = true;
__raw_get_cpu_var(watchdog_touch_ts) = 0;
}
static inline void clear_irq_fired(void)
{
atomic_set(&__raw_get_cpu_var(irq_fired_count), 0);
}
void touch_softlockup_watchdog(void)
{
__raw_get_cpu_var(watchdog_touch_ts) = 0;
}
