asmlinkage void __sched __ipipe_preempt_schedule_irq(void) { struct ipipe_percpu_domain_data *p; unsigned long flags; BUG_ON(!hard_irqs_disabled()); local_irq_save(flags); hard_local_irq_enable(); preempt_schedule_irq(); /* Ok, may reschedule now. */ hard_local_irq_disable(); /* * Flush any pending interrupt that may have been logged after * preempt_schedule_irq() stalled the root stage before * returning to us, and now. */ p = ipipe_this_cpu_root_context(); if (unlikely(__ipipe_ipending_p(p))) { add_preempt_count(PREEMPT_ACTIVE); trace_hardirqs_on(); __clear_bit(IPIPE_STALL_FLAG, &p->status); __ipipe_sync_stage(); sub_preempt_count(PREEMPT_ACTIVE); } __ipipe_restore_root_nosync(flags); }
static void __local_bh_disable(unsigned long ip) { unsigned long flags; WARN_ON_ONCE(in_irq()); raw_local_irq_save(flags); add_preempt_count(SOFTIRQ_OFFSET); /* * Were softirqs turned off above: */ if (softirq_count() == SOFTIRQ_OFFSET) trace_softirqs_off(ip); raw_local_irq_restore(flags); }
/* * Were softirqs turned off above: */ if (softirq_count() == cnt) trace_softirqs_off(ip); raw_local_irq_restore(flags); if (preempt_count() == cnt) trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); } #else /* !CONFIG_TRACE_IRQFLAGS */ static inline void __local_bh_disable(unsigned long ip, unsigned int cnt) { add_preempt_count(cnt); barrier(); }
/** * nohz_restart_sched_tick - restart the idle tick from the idle task * * Restart the idle tick when the CPU is woken up from idle */ void tick_nohz_restart_sched_tick(void) { int cpu = smp_processor_id(); struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); unsigned long ticks; ktime_t now, delta; if (!ts->tick_stopped) return; /* Update jiffies first */ now = ktime_get(); local_irq_disable(); tick_do_update_jiffies64(now); cpu_clear(cpu, nohz_cpu_mask); /* Account the idle time */ delta = ktime_sub(now, ts->idle_entrytime); ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); /* * We stopped the tick in idle. Update process times would miss the * time we slept as update_process_times does only a 1 tick * accounting. Enforce that this is accounted to idle ! */ ticks = jiffies - ts->idle_jiffies; /* * We might be one off. Do not randomly account a huge number of ticks! */ if (ticks && ticks < LONG_MAX) { add_preempt_count(HARDIRQ_OFFSET); account_system_time(current, HARDIRQ_OFFSET, jiffies_to_cputime(ticks)); sub_preempt_count(HARDIRQ_OFFSET); } /* * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; hrtimer_cancel(&ts->sched_timer); ts->sched_timer.expires = ts->idle_tick; while (1) { /* Forward the time to expire in the future */ hrtimer_forward(&ts->sched_timer, now, tick_period); if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { hrtimer_start(&ts->sched_timer, ts->sched_timer.expires, HRTIMER_MODE_ABS); /* Check, if the timer was already in the past */ if (hrtimer_active(&ts->sched_timer)) break; } else { if (!tick_program_event(ts->sched_timer.expires, 0)) break; } /* Update jiffies and reread time */ tick_do_update_jiffies64(now); now = ktime_get(); } local_irq_enable(); }
/* * Were softirqs turned off above: */ if (softirq_count() == SOFTIRQ_OFFSET) trace_softirqs_off(ip); raw_local_irq_restore(flags); } #else /* !CONFIG_TRACE_IRQFLAGS */ static inline void __local_bh_disable(unsigned long ip) { add_preempt_count(SOFTIRQ_OFFSET); barrier(); }
static int kona_timer_unit_test_run(const char *clk_name, unsigned long clk_rate) { struct kona_timer *lkt; int i; unsigned long flags; unsigned long req_cycle; pr_info("%s started!\n", __func__); lkt = kona_timer_request((char *)clk_name, -1); if (lkt == NULL) { pr_err("kona_timer_request returned error\n"); return -EINVAL; } pr_info("%s ch_num: %d acquired!\n", clk_name, lkt->ch_num); timer_m = (struct timer_measure *) kmalloc(sizeof(struct timer_measure), GFP_KERNEL); if (!timer_m) { pr_err("%s memory allocation failed!\n", __func__); return -EINVAL; } spin_lock_init(&timer_m->lock); #ifdef CONFIG_PREEMPT /* Ensure that cond_resched() won't try to preempt anybody */ add_preempt_count(PREEMPT_ACTIVE); #endif /*---------------------------------------------------------------------- * Test 1 : 1 clock cycle test * - set up 1 clock cycle timer request and measure the expiration * time. */ msleep(50); pr_info("=== Test case 1 ===\n"); pr_info("1 clock tick one shot for 1 ms\n"); req_cycle = 1; atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); timer_m->start[timer_m->counter] = (u64)kona_timer_get_counter(lkt); timer_m->req_cycle[timer_m->counter] = req_cycle; spin_unlock_irqrestore(&timer_m->lock, flags); kona_timer_unit_test_program(lkt, MODE_ONESHOT, req_cycle); mdelay(1); kona_timer_stop(lkt); /* Error check */ print_unit_test_result(atomic_read(&timer_exp_count), 1, clk_rate); if (atomic_read(&timer_exp_count) != 1) goto error; /*---------------------------------------------------------------------- * Test 2 : peridic 1 clock timer test * - set up periodic 1 clock cycle timer request. SW will handle * periodic timer on every timer expiration. Wait for certain time * and check how many timers were expired and mesaure the time. */ msleep(50); pr_info("=== Test case 2 ===\n"); pr_info("Periodic 1 clock tick for 1 ms\n"); req_cycle = 1; atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); timer_m->start[timer_m->counter] = (u64)kona_timer_get_counter(lkt); for (i = 0; i < TIMER_MEASURE_MAX; i++) timer_m->req_cycle[i] = req_cycle; spin_unlock_irqrestore(&timer_m->lock, flags); kona_timer_unit_test_program(lkt, MODE_PERIODIC, req_cycle); /* mdelay() may be delayed by busy timer request. Should be short */ mdelay(1); kona_timer_stop(lkt); pr_info("Total expiration count: %d\n", atomic_read(&timer_exp_count)); print_unit_test_result(atomic_read(&timer_exp_count), atomic_read(&timer_exp_count), clk_rate); /*---------------------------------------------------------------------- * Test 3 : one-shot timer test with various time values * - test various short clock cycles and check * timer expiration and real run time of timer. */ msleep(50); pr_info("=== Test case 3 ===\n"); pr_info("0~20 clock tick test for 1s, 50ms delay between each req\n"); atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); spin_unlock_irqrestore(&timer_m->lock, flags); for (i = 0; i < 20; i++) { spin_lock_irqsave(&timer_m->lock, flags); timer_m->busy = 1; timer_m->counter = i; timer_m->start[i] = (u64)kona_timer_get_counter(lkt); timer_m->req_cycle[i] = i; kona_timer_unit_test_program(lkt, MODE_ONESHOT, i); timer_m->busy = 0; spin_unlock_irqrestore(&timer_m->lock, flags); mdelay(50); kona_timer_stop(lkt); } print_unit_test_result(atomic_read(&timer_exp_count), 20, clk_rate); if (atomic_read(&timer_exp_count) != 20) goto error; /*---------------------------------------------------------------------- * Test 4 : one-shot timer test with various time values and delays. * - test various short clock cycles with various short delays. * Verify the timer expiration and timer run-time. */ msleep(50); pr_info("=== Test case 4 ===\n"); pr_info("0~29 cycle timer test, 0~29 + 3 clock cycle wait\n"); pr_info(" Ex)1 clock : wait for 4 clock cycle time\n"); pr_info(" Ex)10 clock : wait for 13 clock cycle time\n"); atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); spin_unlock_irqrestore(&timer_m->lock, flags); for (i = 0; i < 30; i++) { spin_lock_irqsave(&timer_m->lock, flags); timer_m->busy = 1; timer_m->counter = i; timer_m->start[i] = (u64)kona_timer_get_counter(lkt); timer_m->req_cycle[i] = i; kona_timer_unit_test_program(lkt, MODE_ONESHOT, i); timer_m->busy = 0; spin_unlock_irqrestore(&timer_m->lock, flags); kona_clock_delay(lkt, i + 3); } kona_timer_stop(lkt); pr_info("Total expiration count: %d\n", atomic_read(&timer_exp_count)); print_unit_test_result(atomic_read(&timer_exp_count), 30, clk_rate); /*---------------------------------------------------------------------- * Test 5 : one-shot timer test with short-long time/delay. * - test various short/long clock cycles with various short/long * delays. Verify the timer expiration and timer run-time. */ msleep(50); pr_info("=== Test case 5 ===\n"); pr_info("short and long timer test\n"); pr_info(" short delay: 0~19 clock cycle wait\n"); pr_info(" long delay: 50 ms wait\n"); atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); spin_unlock_irqrestore(&timer_m->lock, flags); for (i = 0; i < 20; i++) { /* Short timer request */ spin_lock_irqsave(&timer_m->lock, flags); timer_m->busy = 1; timer_m->counter = 2 * i; timer_m->start[2 * i] = (u64)kona_timer_get_counter(lkt); timer_m->req_cycle[2 * i] = i; kona_timer_unit_test_program(lkt, MODE_ONESHOT, i); timer_m->busy = 0; spin_unlock_irqrestore(&timer_m->lock, flags); kona_clock_delay(lkt, i + 1); /* Long timer request */ req_cycle = clk_rate / 50; /* 20ms */ spin_lock_irqsave(&timer_m->lock, flags); timer_m->busy = 1; timer_m->counter = 2 * i + 1; timer_m->start[2 * i + 1] = (u64)kona_timer_get_counter(lkt); timer_m->req_cycle[2 * i + 1] = req_cycle; kona_timer_unit_test_program(lkt, MODE_ONESHOT, req_cycle); timer_m->busy = 0; spin_unlock_irqrestore(&timer_m->lock, flags); mdelay(1000/50 * 2); /* 40 ms */ } kona_timer_stop(lkt); pr_info("Total expiration count: %d\n", atomic_read(&timer_exp_count)); print_unit_test_result(atomic_read(&timer_exp_count), 20 * 2, clk_rate); /*---------------------------------------------------------------------- * Test 6 : periodic 1s timer test * wait for 10.5 sec with 1s periodic timer request */ msleep(50); pr_info("=== Test case 6 ===\n"); pr_info("Periodic 1s timer test for 10.5s\n"); req_cycle = clk_rate; atomic_set(&timer_exp_count, 0); spin_lock_irqsave(&timer_m->lock, flags); memset(timer_m, 0, sizeof(struct timer_measure)); timer_m->start[timer_m->counter] = (u64)kona_timer_get_counter(lkt); for (i = 0; i < TIMER_MEASURE_MAX; i++) timer_m->req_cycle[i] = req_cycle; spin_unlock_irqrestore(&timer_m->lock, flags); kona_timer_unit_test_program(lkt, MODE_PERIODIC, clk_rate); msleep(10500); kona_timer_stop(lkt); print_unit_test_result(atomic_read(&timer_exp_count), atomic_read(&timer_exp_count), clk_rate); if (atomic_read(&timer_exp_count) != 10) goto error; msleep(50); /* * End of kona timer unit test */ kona_timer_free(lkt); pr_info("%s Passed!\n", __func__); #ifdef CONFIG_PREEMPT sub_preempt_count(PREEMPT_ACTIVE); #endif kfree(timer_m); timer_m = NULL; return 0; error: kona_timer_stop(lkt); kona_timer_free(lkt); pr_err("%s Failed\n", __func__); #ifdef CONFIG_PREEMPT sub_preempt_count(PREEMPT_ACTIVE); #endif kfree(timer_m); timer_m = NULL; return -EINVAL; }
void local_bh_disable(void) { add_preempt_count(SOFTIRQ_DISABLE_OFFSET); }