asmlinkage void __cpuinit secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); printk("CPU%u: Booted secondary processor\n", cpu); atomic_inc(&mm->mm_count); current->active_mm = mm; cpumask_set_cpu(cpu, mm_cpumask(mm)); set_my_cpu_offset(per_cpu_offset(smp_processor_id())); cpu_set_reserved_ttbr0(); flush_tlb_all(); preempt_disable(); trace_hardirqs_off(); if (cpu_ops[cpu]->cpu_postboot) cpu_ops[cpu]->cpu_postboot(); set_cpu_online(cpu, true); complete(&cpu_running); smp_store_cpu_info(cpu); notify_cpu_starting(cpu); local_dbg_enable(); local_irq_enable(); local_fiq_enable(); cpu_startup_entry(CPUHP_ONLINE); }
/* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ asmlinkage void __cpuinit secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); /* * All kernel threads share the same mm context; grab a * reference and switch to it. */ atomic_inc(&mm->mm_count); current->active_mm = mm; cpumask_set_cpu(cpu, mm_cpumask(mm)); set_my_cpu_offset(per_cpu_offset(smp_processor_id())); printk("CPU%u: Booted secondary processor\n", cpu); /* * TTBR0 is only used for the identity mapping at this stage. Make it * point to zero page to avoid speculatively fetching new entries. */ cpu_set_reserved_ttbr0(); flush_tlb_all(); preempt_disable(); trace_hardirqs_off(); if (cpu_ops[cpu]->cpu_postboot) cpu_ops[cpu]->cpu_postboot(); /* * Enable GIC and timers. */ smp_store_cpu_info(cpu); notify_cpu_starting(cpu); /* * OK, now it's safe to let the boot CPU continue. Wait for * the CPU migration code to notice that the CPU is online * before we continue. */ set_cpu_online(cpu, true); complete(&cpu_running); local_dbg_enable(); /* * Setup the percpu timer for this CPU. */ percpu_timer_setup(); local_irq_enable(); local_async_enable(); /* * OK, it's off to the idle thread for us */ cpu_startup_entry(CPUHP_ONLINE); }
/* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ asmlinkage void __cpuinit secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); printk("CPU%u: Booted secondary processor\n", cpu); /* * All kernel threads share the same mm context; grab a * reference and switch to it. */ atomic_inc(&mm->mm_count); current->active_mm = mm; cpumask_set_cpu(cpu, mm_cpumask(mm)); /* * TTBR0 is only used for the identity mapping at this stage. Make it * point to zero page to avoid speculatively fetching new entries. */ cpu_set_reserved_ttbr0(); flush_tlb_all(); preempt_disable(); trace_hardirqs_off(); /* * Let the primary processor know we're out of the * pen, then head off into the C entry point */ write_pen_release(INVALID_HWID); /* * Synchronise with the boot thread. */ raw_spin_lock(&boot_lock); raw_spin_unlock(&boot_lock); /* * OK, now it's safe to let the boot CPU continue. Wait for * the CPU migration code to notice that the CPU is online * before we continue. */ set_cpu_online(cpu, true); complete(&cpu_running); /* * Enable GIC and timers. */ notify_cpu_starting(cpu); local_irq_enable(); local_fiq_enable(); /* * OK, it's off to the idle thread for us */ cpu_startup_entry(CPUHP_ONLINE); }
/* * __cpu_suspend * * arg: argument to pass to the finisher function * fn: finisher function pointer * */ int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) { struct mm_struct *mm = current->active_mm; int ret; unsigned long flags; /* * From this point debug exceptions are disabled to prevent * updates to mdscr register (saved and restored along with * general purpose registers) from kernel debuggers. */ local_dbg_save(flags); /* * Function graph tracer state gets incosistent when the kernel * calls functions that never return (aka suspend finishers) hence * disable graph tracing during their execution. */ pause_graph_tracing(); /* * mm context saved on the stack, it will be restored when * the cpu comes out of reset through the identity mapped * page tables, so that the thread address space is properly * set-up on function return. */ ret = __cpu_suspend_enter(arg, fn); if (ret == 0) { /* * We are resuming from reset with TTBR0_EL1 set to the * idmap to enable the MMU; restore the active_mm mappings in * TTBR0_EL1 unless the active_mm == &init_mm, in which case * the thread entered __cpu_suspend with TTBR0_EL1 set to * reserved TTBR0 page tables and should be restored as such. */ if (mm == &init_mm) cpu_set_reserved_ttbr0(); else cpu_switch_mm(mm->pgd, mm); flush_tlb_all(); /* * Restore per-cpu offset before any kernel * subsystem relying on it has a chance to run. */ set_my_cpu_offset(per_cpu_offset(smp_processor_id())); /* * Restore HW breakpoint registers to sane values * before debug exceptions are possibly reenabled * through local_dbg_restore. */ if (hw_breakpoint_restore) hw_breakpoint_restore(NULL); } unpause_graph_tracing(); /* * Restore pstate flags. OS lock and mdscr have been already * restored, so from this point onwards, debugging is fully * renabled if it was enabled when core started shutdown. */ local_dbg_restore(flags); return ret; }
/** * cpu_suspend * * @arg: argument to pass to the finisher function */ int cpu_suspend(unsigned long arg) { struct mm_struct *mm = current->active_mm; int ret, cpu = smp_processor_id(); unsigned long flags; /* * If cpu_ops have not been registered or suspend * has not been initialized, cpu_suspend call fails early. */ if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) return -EOPNOTSUPP; /* * From this point debug exceptions are disabled to prevent * updates to mdscr register (saved and restored along with * general purpose registers) from kernel debuggers. */ local_dbg_save(flags); /* * mm context saved on the stack, it will be restored when * the cpu comes out of reset through the identity mapped * page tables, so that the thread address space is properly * set-up on function return. */ ret = __cpu_suspend(arg); if (ret == 0) { /* * We are resuming from reset with TTBR0_EL1 set to the * idmap to enable the MMU; restore the active_mm mappings in * TTBR0_EL1 unless the active_mm == &init_mm, in which case * the thread entered __cpu_suspend with TTBR0_EL1 set to * reserved TTBR0 page tables and should be restored as such. */ if (mm == &init_mm) cpu_set_reserved_ttbr0(); else cpu_switch_mm(mm->pgd, mm); flush_tlb_all(); /* * Restore per-cpu offset before any kernel * subsystem relying on it has a chance to run. */ set_my_cpu_offset(per_cpu_offset(cpu)); /* * Restore HW breakpoint registers to sane values * before debug exceptions are possibly reenabled * through local_dbg_restore. */ if (hw_breakpoint_restore) hw_breakpoint_restore(NULL); } /* * Restore pstate flags. OS lock and mdscr have been already * restored, so from this point onwards, debugging is fully * renabled if it was enabled when core started shutdown. */ local_dbg_restore(flags); return ret; }
/* * cpu_suspend * * arg: argument to pass to the finisher function * fn: finisher function pointer * */ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) { struct mm_struct *mm = current->active_mm; int ret; unsigned long flags; /* * From this point debug exceptions are disabled to prevent * updates to mdscr register (saved and restored along with * general purpose registers) from kernel debuggers. */ local_dbg_save(flags); /* * mm context saved on the stack, it will be restored when * the cpu comes out of reset through the identity mapped * page tables, so that the thread address space is properly * set-up on function return. */ ret = __cpu_suspend_enter(arg, fn); if (ret == 0) { /* * We are resuming from reset with TTBR0_EL1 set to the * idmap to enable the MMU; set the TTBR0 to the reserved * page tables to prevent speculative TLB allocations, flush * the local tlb and set the default tcr_el1.t0sz so that * the TTBR0 address space set-up is properly restored. * If the current active_mm != &init_mm we entered cpu_suspend * with mappings in TTBR0 that must be restored, so we switch * them back to complete the address space configuration * restoration before returning. */ cpu_set_reserved_ttbr0(); local_flush_tlb_all(); cpu_set_default_tcr_t0sz(); if (mm != &init_mm) cpu_switch_mm(mm->pgd, mm); /* * Restore per-cpu offset before any kernel * subsystem relying on it has a chance to run. */ set_my_cpu_offset(per_cpu_offset(smp_processor_id())); /* * Restore HW breakpoint registers to sane values * before debug exceptions are possibly reenabled * through local_dbg_restore. */ if (hw_breakpoint_restore) hw_breakpoint_restore(NULL); } /* * Restore pstate flags. OS lock and mdscr have been already * restored, so from this point onwards, debugging is fully * renabled if it was enabled when core started shutdown. */ local_dbg_restore(flags); return ret; }
/* * Copies length bytes, starting at src_start into an new page, * perform cache maintentance, then maps it at the specified address low * address as executable. * * This is used by hibernate to copy the code it needs to execute when * overwriting the kernel text. This function generates a new set of page * tables, which it loads into ttbr0. * * Length is provided as we probably only want 4K of data, even on a 64K * page system. */ static int create_safe_exec_page(void *src_start, size_t length, unsigned long dst_addr, phys_addr_t *phys_dst_addr, void *(*allocator)(gfp_t mask), gfp_t mask) { int rc = 0; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long dst = (unsigned long)allocator(mask); if (!dst) { rc = -ENOMEM; goto out; } memcpy((void *)dst, src_start, length); flush_icache_range(dst, dst + length); pgdp = pgd_offset_raw(allocator(mask), dst_addr); if (pgd_none(READ_ONCE(*pgdp))) { pudp = allocator(mask); if (!pudp) { rc = -ENOMEM; goto out; } pgd_populate(&init_mm, pgdp, pudp); } pudp = pud_offset(pgdp, dst_addr); if (pud_none(READ_ONCE(*pudp))) { pmdp = allocator(mask); if (!pmdp) { rc = -ENOMEM; goto out; } pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, dst_addr); if (pmd_none(READ_ONCE(*pmdp))) { ptep = allocator(mask); if (!ptep) { rc = -ENOMEM; goto out; } pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, dst_addr); set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); /* * Load our new page tables. A strict BBM approach requires that we * ensure that TLBs are free of any entries that may overlap with the * global mappings we are about to install. * * For a real hibernate/resume cycle TTBR0 currently points to a zero * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI * runtime services), while for a userspace-driven test_resume cycle it * points to userspace page tables (and we must point it at a zero page * ourselves). Elsewhere we only (un)install the idmap with preemption * disabled, so T0SZ should be as required regardless. */ cpu_set_reserved_ttbr0(); local_flush_tlb_all(); write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); isb(); *phys_dst_addr = virt_to_phys((void *)dst); out: return rc; }
/* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); /* * All kernel threads share the same mm context; grab a * reference and switch to it. */ atomic_inc(&mm->mm_count); current->active_mm = mm; set_my_cpu_offset(per_cpu_offset(smp_processor_id())); pr_debug("CPU%u: Booted secondary processor\n", cpu); /* * TTBR0 is only used for the identity mapping at this stage. Make it * point to zero page to avoid speculatively fetching new entries. */ cpu_set_reserved_ttbr0(); local_flush_tlb_all(); cpu_set_default_tcr_t0sz(); preempt_disable(); trace_hardirqs_off(); /* * If the system has established the capabilities, make sure * this CPU ticks all of those. If it doesn't, the CPU will * fail to come online. */ verify_local_cpu_capabilities(); if (cpu_ops[cpu]->cpu_postboot) cpu_ops[cpu]->cpu_postboot(); /* * Log the CPU info before it is marked online and might get read. */ cpuinfo_store_cpu(); /* * Enable GIC and timers. */ notify_cpu_starting(cpu); smp_store_cpu_info(cpu); /* * OK, now it's safe to let the boot CPU continue. Wait for * the CPU migration code to notice that the CPU is online * before we continue. */ pr_info("CPU%u: Booted secondary processor [%08x]\n", cpu, read_cpuid_id()); set_cpu_online(cpu, true); complete(&cpu_running); local_irq_enable(); local_async_enable(); /* * OK, it's off to the idle thread for us */ cpu_startup_entry(CPUHP_ONLINE); }