void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
if (machine_is_ag5evm())
sh73a0_secondary_init(cpu);
}
asmlinkage void do_notify_resume(struct pt_regs *regs,
unsigned int thread_flags)
{
/*
* The assembly code enters us with IRQs off, but it hasn't
* informed the tracing code of that for efficiency reasons.
* Update the trace code with the current status.
*/
trace_hardirqs_off();
do {
if (thread_flags & _TIF_NEED_RESCHED) {
schedule();
} else {
local_irq_enable();
if (thread_flags & _TIF_SIGPENDING)
do_signal(regs);
if (thread_flags & _TIF_NOTIFY_RESUME) {
clear_thread_flag(TIF_NOTIFY_RESUME);
tracehook_notify_resume(regs);
}
if (thread_flags & _TIF_FOREIGN_FPSTATE)
fpsimd_restore_current_state();
}
local_irq_disable();
thread_flags = READ_ONCE(current_thread_info()->flags);
} while (thread_flags & _TIF_WORK_MASK);
}
void enabled_wait(void)
{
struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
unsigned long long idle_time;
unsigned long psw_mask;
trace_hardirqs_on();
/* Wait for external, I/O or machine check interrupt. */
psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT |
PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
clear_cpu_flag(CIF_NOHZ_DELAY);
/* Call the assembler magic in entry.S */
psw_idle(idle, psw_mask);
trace_hardirqs_off();
/* Account time spent with enabled wait psw loaded as idle time. */
write_seqcount_begin(&idle->seqcount);
idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
idle->idle_time += idle_time;
idle->idle_count++;
account_idle_time(idle_time);
write_seqcount_end(&idle->seqcount);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* the primary core may have used a "cross call" soft interrupt
* to get this processor out of WFI in the BootMonitor - make
* sure that we are no longer being sent this soft interrupt
*/
smp_cross_call_done(cpumask_of_cpu(cpu));
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
gic_cpu_init(0, gic_cpu_base_addr);
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
pen_release = -1;
smp_wmb();
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
asmlinkage void __cpuinit secondary_start_kernel(void)
{
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
printk("CPU%u: Booted secondary processor\n", cpu);
atomic_inc(&mm->mm_count);
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
cpu_set_reserved_ttbr0();
flush_tlb_all();
preempt_disable();
trace_hardirqs_off();
if (cpu_ops[cpu]->cpu_postboot)
cpu_ops[cpu]->cpu_postboot();
set_cpu_online(cpu, true);
complete(&cpu_running);
smp_store_cpu_info(cpu);
notify_cpu_starting(cpu);
local_dbg_enable();
local_irq_enable();
local_fiq_enable();
cpu_startup_entry(CPUHP_ONLINE);
}
/* Initialization routine for secondary CPUs after they are brought out of
* reset.
*/
void platform_secondary_init(unsigned int cpu)
{
printk(KERN_DEBUG "%s: cpu:%d\n", __func__, cpu);
#ifdef CONFIG_HOTPLUG_CPU
WARN_ON(msm_pm_platform_secondary_init(cpu));
#endif
trace_hardirqs_off();
/* Edge trigger PPIs except AVS_SVICINT and AVS_SVICINTSWDONE */
writel(0xFFFFD7FF, MSM_QGIC_DIST_BASE + GIC_DIST_CONFIG + 4);
/* RUMI does not adhere to GIC spec by enabling STIs by default.
* Enable/clear is supposed to be RO for STIs, but is RW on RUMI.
*/
if (machine_is_msm8x60_surf() ||
machine_is_msm8x60_ffa() ||
machine_is_msm8x60_rumi3())
writel(0x0000FFFF, MSM_QGIC_DIST_BASE + GIC_DIST_ENABLE_SET);
/*
* setup GIC (GIC number NOT CPU number and the base address of the
* GIC CPU interface
*/
gic_cpu_init(0, MSM_QGIC_CPU_BASE);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
mpcore_cpu_init();
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
pen_release = -1;
smp_wmb();
clean_dcache_area((void *) &pen_release, sizeof(pen_release));
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
if (is_sh73a0())
sh73a0_secondary_init(cpu);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* the primary core may have used a "cross call" soft interrupt
* to get this processor out of WFI in the BootMonitor - make
* sure that we are no longer being sent this soft interrupt
*/
smp_cross_call_done(cpumask_of_cpu(cpu));
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
if (machine_is_realview_eb() && core_tile_eb11mp())
gic_cpu_init(0, __io_address(REALVIEW_EB11MP_GIC_CPU_BASE));
else if (machine_is_realview_pb11mp())
gic_cpu_init(0, __io_address(REALVIEW_TC11MP_GIC_CPU_BASE));
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
pen_release = -1;
smp_wmb();
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
/*
* This is the secondary CPU boot entry. We're using this CPUs
* idle thread stack, but a set of temporary page tables.
*/
asmlinkage void __cpuinit secondary_start_kernel(void)
{
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
/*
* All kernel threads share the same mm context; grab a
* reference and switch to it.
*/
atomic_inc(&mm->mm_count);
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
printk("CPU%u: Booted secondary processor\n", cpu);
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
*/
cpu_set_reserved_ttbr0();
flush_tlb_all();
preempt_disable();
trace_hardirqs_off();
if (cpu_ops[cpu]->cpu_postboot)
cpu_ops[cpu]->cpu_postboot();
/*
* Enable GIC and timers.
*/
smp_store_cpu_info(cpu);
notify_cpu_starting(cpu);
/*
* OK, now it's safe to let the boot CPU continue. Wait for
* the CPU migration code to notice that the CPU is online
* before we continue.
*/
set_cpu_online(cpu, true);
complete(&cpu_running);
local_dbg_enable();
/*
* Setup the percpu timer for this CPU.
*/
percpu_timer_setup();
local_irq_enable();
local_async_enable();
/*
* OK, it's off to the idle thread for us
*/
cpu_startup_entry(CPUHP_ONLINE);
}
/*
* __ipipe_do_sync_stage() -- Flush the pending IRQs for the current
* domain (and processor). This routine flushes the interrupt log (see
* "Optimistic interrupt protection" from D. Stodolsky et al. for more
* on the deferred interrupt scheme). Every interrupt that occurred
* while the pipeline was stalled gets played.
*
* WARNING: CPU migration may occur over this routine.
*/
void __ipipe_do_sync_stage(void)
{
struct ipipe_percpu_domain_data *p;
struct ipipe_domain *ipd;
int irq;
p = __ipipe_current_context;
ipd = p->domain;
__set_bit(IPIPE_STALL_FLAG, &p->status);
smp_wmb();
if (ipd == ipipe_root_domain)
trace_hardirqs_off();
for (;;) {
irq = __ipipe_next_irq(p);
if (irq < 0)
break;
/*
* Make sure the compiler does not reorder wrongly, so
* that all updates to maps are done before the
* handler gets called.
*/
barrier();
if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
continue;
if (ipd != ipipe_head_domain)
hard_local_irq_enable();
if (likely(ipd != ipipe_root_domain)) {
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
__ipipe_run_irqtail(irq);
hard_local_irq_disable();
} else if (ipipe_virtual_irq_p(irq)) {
irq_enter();
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
irq_exit();
root_stall_after_handler();
hard_local_irq_disable();
while (__ipipe_check_root_resched())
__ipipe_preempt_schedule_irq();
} else {
ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie);
root_stall_after_handler();
hard_local_irq_disable();
}
p = __ipipe_current_context;
}
if (ipd == ipipe_root_domain)
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
}
/*
* This is the secondary CPU boot entry. We're using this CPUs
* idle thread stack, but a set of temporary page tables.
*/
asmlinkage void __cpuinit secondary_start_kernel(void)
{
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
printk("CPU%u: Booted secondary processor\n", cpu);
/*
* All kernel threads share the same mm context; grab a
* reference and switch to it.
*/
atomic_inc(&mm->mm_count);
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
*/
cpu_set_reserved_ttbr0();
flush_tlb_all();
preempt_disable();
trace_hardirqs_off();
/*
* Let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
write_pen_release(INVALID_HWID);
/*
* Synchronise with the boot thread.
*/
raw_spin_lock(&boot_lock);
raw_spin_unlock(&boot_lock);
/*
* OK, now it's safe to let the boot CPU continue. Wait for
* the CPU migration code to notice that the CPU is online
* before we continue.
*/
set_cpu_online(cpu, true);
complete(&cpu_running);
/*
* Enable GIC and timers.
*/
notify_cpu_starting(cpu);
local_irq_enable();
local_fiq_enable();
/*
* OK, it's off to the idle thread for us
*/
cpu_startup_entry(CPUHP_ONLINE);
}
void __ipipe_restore_root_nosync(unsigned long x)
{
struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context();
if (raw_irqs_disabled_flags(x)) {
__set_bit(IPIPE_STALL_FLAG, &p->status);
trace_hardirqs_off();
} else {
trace_hardirqs_on();
__clear_bit(IPIPE_STALL_FLAG, &p->status);
}
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
if (is_sh73a0())
sh73a0_secondary_init(cpu);
if (is_r8a7779())
r8a7779_secondary_init(cpu);
if (is_emev2())
emev2_secondary_init(cpu);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
gic_cpu_init(0, IO_ADDRESS(TEGRA_ARM_PERIF_BASE) + 0x100);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
#ifdef CONFIG_HOTPLUG_CPU
cpu_set(cpu, cpu_init_map);
INIT_COMPLETION(per_cpu(cpu_killed, cpu));
#endif
spin_unlock(&boot_lock);
}
void secondary_start_kernel(void)
{
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
init_mmu();
#ifdef CONFIG_DEBUG_KERNEL
if (boot_secondary_processors == 0) {
pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
__func__, boot_secondary_processors, cpu);
for (;;)
__asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
}
pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
__func__, boot_secondary_processors, cpu);
#endif
/* Init EXCSAVE1 */
secondary_trap_init();
/* All kernel threads share the same mm context. */
mmget(mm);
mmgrab(mm);
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
enter_lazy_tlb(mm, current);
preempt_disable();
trace_hardirqs_off();
calibrate_delay();
notify_cpu_starting(cpu);
secondary_init_irq();
local_timer_setup(cpu);
set_cpu_online(cpu, true);
local_irq_enable();
complete(&cpu_running);
cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
}
void sti_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
write_pen_release(-1);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* If any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
gic_cpu_init(0, gic_cpu_base_addr);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
gic_cpu_init(0, IO_ADDRESS(TEGRA_ARM_PERIF_BASE) + 0x100);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
/*
* do_IRQ()
* Primary interface for handling IRQ() requests.
*
* This function is force executed when the PIC switches a thread to it.
*/
asmlinkage void do_IRQ(struct pt_regs *regs, unsigned int irq)
{
struct pt_regs *oldregs;
struct thread_info *ti = current_thread_info();
STOPWATCH_DECLARE;
trace_hardirqs_off();
/*
* Mark that we are inside of an interrupt and
* that interrupts are disabled.
*/
oldregs = set_irq_regs(regs);
ti->interrupt_nesting++;
irq_kernel_stack_check(irq, regs);
/*
* Start the interrupt sequence
*/
irq_enter();
/*
* Execute the IRQ handler and any pending SoftIRQ requests.
*/
BUG_ON(!irqs_disabled());
STOPWATCH_START();
generic_handle_irq(irq);
STOPWATCH_END(&irq_watches[irq]);
BUG_ON(!irqs_disabled());
/*
* Exit the interrupt and process softirqs if needed.
*/
STOPWATCH_START();
irq_exit();
STOPWATCH_END(&irq_watches[INTERRUPT_COUNT]);
BUG_ON(!irqs_disabled());
/*
* Outside of an interrupt (or nested exit).
*/
trace_hardirqs_on();
ti->interrupt_nesting--;
set_irq_regs(oldregs);
}
void platform_secondary_init(unsigned int cpu)
{
printk(KERN_DEBUG "%s: cpu:%d\n", __func__, cpu);
trace_hardirqs_off();
writel(0xFFFFD7FF, MSM_QGIC_DIST_BASE + GIC_DIST_CONFIG + 4);
if (machine_is_msm8x60_surf() ||
machine_is_msm8x60_ffa() ||
machine_is_msm8x60_rumi3())
writel(0x0000FFFF, MSM_QGIC_DIST_BASE + GIC_DIST_ENABLE_SET);
gic_cpu_init(0, MSM_QGIC_CPU_BASE);
}
asmlinkage void do_IRQ(int irq, struct pt_regs * regs)
{
unsigned long sp;
struct pt_regs *old_regs;
trace_hardirqs_off();
old_regs = set_irq_regs(regs);
irq_enter();
sp = rdsp();
if (unlikely((sp & (PAGE_SIZE - 1)) < (PAGE_SIZE/8))) {
printk("do_IRQ: stack overflow: %lX\n", sp);
show_stack(NULL, (unsigned long *)sp);
}
generic_handle_irq(irq);
irq_exit();
set_irq_regs(old_regs);
}
/* Initialization routine for secondary CPUs after they are brought out of
* reset.
*/
void __cpuinit platform_secondary_init(unsigned int cpu)
{
pr_debug("CPU%u: Booted secondary processor\n", cpu);
WARN_ON(msm_platform_secondary_init(cpu));
trace_hardirqs_off();
/* Edge trigger PPIs except AVS_SVICINT and AVS_SVICINTSWDONE */
writel(0xFFFFD7FF, MSM_QGIC_DIST_BASE + GIC_DIST_CONFIG + 4);
/* RUMI does not adhere to GIC spec by enabling STIs by default.
* Enable/clear is supposed to be RO for STIs, but is RW on RUMI.
*/
if (!machine_is_msm8x60_sim())
writel(0x0000FFFF, MSM_QGIC_DIST_BASE + GIC_DIST_ENABLE_SET);
gic_secondary_init(0);
}
void smp_send_stop(void)
{
int cpu, rc;
/* Disable all interrupts/machine checks */
__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
trace_hardirqs_off();
/* stop all processors */
for_each_online_cpu(cpu) {
if (cpu == smp_processor_id())
continue;
do {
rc = sigp(cpu, sigp_stop);
} while (rc == sigp_busy);
while (!cpu_stopped(cpu))
cpu_relax();
}
}
asmlinkage int
do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
{
/*
* The assembly code enters us with IRQs off, but it hasn't
* informed the tracing code of that for efficiency reasons.
* Update the trace code with the current status.
*/
trace_hardirqs_off();
do {
if (likely(thread_flags & _TIF_NEED_RESCHED)) {
schedule();
} else {
if (unlikely(!user_mode(regs)))
return 0;
local_irq_enable();
if (thread_flags & _TIF_SIGPENDING) {
int restart = do_signal(regs, syscall);
if (unlikely(restart)) {
/*
* Restart without handlers.
* Deal with it without leaving
* the kernel space.
*/
return restart;
}
syscall = 0;
} else if (thread_flags & _TIF_UPROBE) {
uprobe_notify_resume(regs);
} else {
clear_thread_flag(TIF_NOTIFY_RESUME);
tracehook_notify_resume(regs);
rseq_handle_notify_resume(NULL, regs);
}
}
local_irq_disable();
thread_flags = current_thread_info()->flags;
} while (thread_flags & _TIF_WORK_MASK);
return 0;
}
void __irq_entry do_IRQ(struct pt_regs *regs)
{
unsigned int irq;
struct pt_regs *old_regs = set_irq_regs(regs);
trace_hardirqs_off();
irq_enter();
irq = xintc_get_irq();
next_irq:
BUG_ON(!irq);
generic_handle_irq(irq);
irq = xintc_get_irq();
if (irq != -1U) {
pr_debug("next irq: %d\n", irq);
++concurrent_irq;
goto next_irq;
}
irq_exit();
set_irq_regs(old_regs);
trace_hardirqs_on();
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
if (omap_type() != OMAP2_DEVICE_TYPE_GP)
/* Enable NS access to SMP bit */
omap4_secure_dispatcher(PPA_SERVICE_NS_SMP, 4, 0, 0, 0, 0, 0);
else
omap_smc1(0x114, 0x810);
/*
* If any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
gic_cpu_init(0, gic_cpu_base_addr);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
void __cpuinit platform_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
gic_cpu_init(0, __io_address(UX500_GIC_CPU_BASE));
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
pen_release = -1;
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
static void __cpuinit brcm_secondary_init(unsigned int cpu)
{
trace_hardirqs_off();
/*
* if any interrupts are already enabled for the primary
* core (e.g. timer irq), then they will not have been enabled
* for us: do so
*/
mpcore_cpu_init();
/*
* let the primary processor know we're out of the
* pen, then head off into the C entry point
*/
write_pen_release(-1);
/*
* Synchronise with the boot thread.
*/
spin_lock(&boot_lock);
spin_unlock(&boot_lock);
}
/*
* This is the secondary CPU boot entry. We're using this CPUs
* idle thread stack, but a set of temporary page tables.
*/
asmlinkage void secondary_start_kernel(void)
{
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
/*
* All kernel threads share the same mm context; grab a
* reference and switch to it.
*/
atomic_inc(&mm->mm_count);
current->active_mm = mm;
set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
*/
cpu_uninstall_idmap();
preempt_disable();
trace_hardirqs_off();
/*
* If the system has established the capabilities, make sure
* this CPU ticks all of those. If it doesn't, the CPU will
* fail to come online.
*/
verify_local_cpu_capabilities();
if (cpu_ops[cpu]->cpu_postboot)
cpu_ops[cpu]->cpu_postboot();
/*
* Log the CPU info before it is marked online and might get read.
*/
cpuinfo_store_cpu();
/*
* Enable GIC and timers.
*/
notify_cpu_starting(cpu);
store_cpu_topology(cpu);
/*
* OK, now it's safe to let the boot CPU continue. Wait for
* the CPU migration code to notice that the CPU is online
* before we continue.
*/
pr_info("CPU%u: Booted secondary processor [%08x]\n",
cpu, read_cpuid_id());
update_cpu_boot_status(CPU_BOOT_SUCCESS);
set_cpu_online(cpu, true);
complete(&cpu_running);
local_irq_enable();
local_async_enable();
/*
* OK, it's off to the idle thread for us
*/
cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
}
