static void stop_self(void *v) { int cpu = smp_processor_id(); load_cr3(swapper_pg_dir); set_cpu_online(cpu, false); HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL); BUG(); }
static void stop_self(void *v) { int cpu = smp_processor_id(); /* make sure we're not pinning something down */ load_cr3(swapper_pg_dir); /* should set up a minimal gdt */ set_cpu_online(cpu, false); HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL); BUG(); }
static void drop_mm_ref(struct mm_struct *mm) { if (current->active_mm == mm) { if (current->mm == mm) load_cr3(swapper_pg_dir); else leave_mm(smp_processor_id()); } if (!cpus_empty(mm->cpu_vm_mask)) xen_smp_call_function_mask(mm->cpu_vm_mask, drop_other_mm_ref, mm, 1); }
void efi_arch_leave(void) { pmap_t curpmap; curpmap = &curproc->p_vmspace->vm_pmap; if (pmap_pcid_enabled && !invpcid_works) PCPU_SET(curpmap, curpmap); load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ? curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0)); if (!pmap_pcid_enabled) invltlb(); }
/* * Activate a secondary processor. */ static void notrace start_secondary(void *unused) { /* * Don't put *anything* before cpu_init(), SMP booting is too * fragile that we want to limit the things done here to the * most necessary things. */ cpu_init(); x86_cpuinit.early_percpu_clock_init(); preempt_disable(); smp_callin(); enable_start_cpu0 = 0; #ifdef CONFIG_X86_32 /* switch away from the initial page table */ load_cr3(swapper_pg_dir); __flush_tlb_all(); #endif /* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier(); /* * Check TSC synchronization with the BP: */ check_tsc_sync_target(); /* * Lock vector_lock and initialize the vectors on this cpu * before setting the cpu online. We must set it online with * vector_lock held to prevent a concurrent setup/teardown * from seeing a half valid vector space. */ lock_vector_lock(); setup_vector_irq(smp_processor_id()); set_cpu_online(smp_processor_id(), true); unlock_vector_lock(); cpu_set_state_online(smp_processor_id()); x86_platform.nmi_init(); /* enable local interrupts */ local_irq_enable(); /* to prevent fake stack check failure in clock setup */ boot_init_stack_canary(); x86_cpuinit.setup_percpu_clockev(); wmb(); cpu_startup_entry(CPUHP_ONLINE); }
/* * Activate a secondary processor. */ static void notrace start_secondary(void *unused) { /* * Don't put *anything* before cpu_init(), SMP booting is too * fragile that we want to limit the things done here to the * most necessary things. */ cpu_init(); x86_cpuinit.early_percpu_clock_init(); preempt_disable(); smp_callin(); enable_start_cpu0 = 0; #ifdef CONFIG_X86_32 /* switch away from the initial page table */ load_cr3(swapper_pg_dir); __flush_tlb_all(); #endif /* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier(); /* * Check TSC synchronization with the BP: */ check_tsc_sync_target(); /* * We need to hold vector_lock so there the set of online cpus * does not change while we are assigning vectors to cpus. Holding * this lock ensures we don't half assign or remove an irq from a cpu. */ lock_vector_lock(); set_cpu_online(smp_processor_id(), true); unlock_vector_lock(); per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; x86_platform.nmi_init(); /* enable local interrupts */ local_irq_enable(); /* to prevent fake stack check failure in clock setup */ boot_init_stack_canary(); x86_cpuinit.setup_percpu_clockev(); wmb(); cpu_startup_entry(CPUHP_ONLINE); }
pgd_t * __init efi_call_phys_prolog(void) { struct desc_ptr gdt_descr; pgd_t *save_pgd; /* Current pgd is swapper_pg_dir, we'll restore it later: */ save_pgd = swapper_pg_dir; load_cr3(initial_page_table); __flush_tlb_all(); gdt_descr.address = __pa(get_cpu_gdt_table(0)); gdt_descr.size = GDT_SIZE - 1; load_gdt(&gdt_descr); return save_pgd; }
* 使用次数,若是共享的,则分配一个新的页目录,并将原页目录引用数减1。 */ BOOL pre_replace_user_zone(void) { free_user_pages(current->code_start, current->data_end-current->code_start, 1); if(1 < mem_map[M2MAP(current->tss.cr3)]) { unsigned long addr = current->tss.cr3; if(!copy_kernel_page_dir(&(current->tss.cr3))) { k_printf("free_user_zone: have no free-page!"); return FALSE; } load_cr3(current->tss.cr3); --mem_map[M2MAP(addr)]; d_printf("set cr3 finished, cr3 = %x.\n", current->tss.cr3); }
/* * We cannot call mmdrop() because we are in interrupt context, * instead update mm->cpu_vm_mask. */ void leave_mm(int cpu) { struct mm_struct *active_mm = this_cpu_read(cpu_tlbstate.active_mm); if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) BUG(); if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) { cpumask_clear_cpu(cpu, mm_cpumask(active_mm)); load_cr3(swapper_pg_dir); /* * This gets called in the idle path where RCU * functions differently. Tracing normally * uses RCU, so we have to call the tracepoint * specially here. */ trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); } }
void __init efi_call_phys_epilog(void) { /* * After the lock is released, the original page table is restored. */ int pgd; int n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); for (pgd = 0; pgd < n_pgds; pgd++) set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]); kfree(save_pgd); #ifdef CONFIG_PAX_PER_CPU_PGD load_cr3(get_cpu_pgd(smp_processor_id(), kernel)); #endif __flush_tlb_all(); local_irq_restore(efi_flags); early_code_mapping_set_exec(0); }
static int xen_suspend(void *data) { int *cancelled = data; int err; BUG_ON(!irqs_disabled()); load_cr3(swapper_pg_dir); err = device_power_down(PMSG_SUSPEND); if (err) { printk(KERN_ERR "xen_suspend: device_power_down failed: %d\n", err); return err; } xen_mm_pin_all(); gnttab_suspend(); xen_pre_suspend(); /* * This hypercall returns 1 if suspend was cancelled * or the domain was merely checkpointed, and 0 if it * is resuming in a new domain. */ *cancelled = HYPERVISOR_suspend(virt_to_mfn(xen_start_info)); xen_post_suspend(*cancelled); gnttab_resume(); xen_mm_unpin_all(); device_power_up(PMSG_RESUME); if (!*cancelled) { xen_irq_resume(); xen_console_resume(); xen_timer_resume(); } return 0; }
void do_wp_page(void *addr) { //printk("%s addr %08x current %08x\n", __func__, (unsigned long)addr, current); if((unsigned long) addr >= PAGE_OFFSET) { panic("%s invalid addr", __func__); } int npde = get_npd(addr); int npte = get_npt(addr); pde_t *page_dir = (pde_t *)current->cr3; pte_t *page_tbl = pa2va(PAGE_ALIGN(page_dir[npde])); unsigned long wp_pa_addr = PAGE_ALIGN(page_tbl[npte]); page_t *page = pa2page(wp_pa_addr); if(page->count > 0) { page->count --; unsigned long flags = PAGE_FLAGS(page_tbl[npte]); unsigned long wp_va_addr = (unsigned long) pa2va(wp_pa_addr); unsigned long newtbl = alloc_one_page(0); assert(newtbl != 0); memcpy((void *)newtbl, (void *)wp_va_addr, PAGE_SIZE); page_tbl[npte] = va2pa(newtbl) | flags; } page_tbl[npte] |= PAGE_WR; #if 0 page_tbl[npte] |= PAGE_US; page_dir[npde] |= PAGE_WR; page_dir[npde] |= PAGE_US; #endif load_cr3(current); }
void __noreturn machine_real_restart(unsigned int type) { local_irq_disable(); /* * Write zero to CMOS register number 0x0f, which the BIOS POST * routine will recognize as telling it to do a proper reboot. (Well * that's what this book in front of me says -- it may only apply to * the Phoenix BIOS though, it's not clear). At the same time, * disable NMIs by setting the top bit in the CMOS address register, * as we're about to do peculiar things to the CPU. I'm not sure if * `outb_p' is needed instead of just `outb'. Use it to be on the * safe side. (Yes, CMOS_WRITE does outb_p's. - Paul G.) */ spin_lock(&rtc_lock); CMOS_WRITE(0x00, 0x8f); spin_unlock(&rtc_lock); /* * Switch back to the initial page table. */ #ifdef CONFIG_X86_32 load_cr3(initial_page_table); #else write_cr3(real_mode_header->trampoline_pgd); #endif /* Jump to the identity-mapped low memory code */ #ifdef CONFIG_X86_32 asm volatile("jmpl *%0" : : "rm" (real_mode_header->machine_real_restart_asm), "a" (type)); #else asm volatile("ljmpl *%0" : : "m" (real_mode_header->machine_real_restart_asm), "D" (type)); #endif unreachable(); }
int acpi_wakeup_machdep(struct acpi_softc *sc, int state, int sleep_result, int intr_enabled) { if (sleep_result == -1) return (sleep_result); if (intr_enabled == 0) { /* Wakeup MD procedures in interrupt disabled context */ if (sleep_result == 1) { pmap_init_pat(); load_cr3(susppcbs[0]->pcb_cr3); initializecpu(); PCPU_SET(switchtime, 0); PCPU_SET(switchticks, ticks); #ifdef SMP if (!CPU_EMPTY(&suspcpus)) acpi_wakeup_cpus(sc, &suspcpus); #endif } #ifdef SMP if (!CPU_EMPTY(&suspcpus)) restart_cpus(suspcpus); #endif mca_resume(); intr_resume(); } else { /* Wakeup MD procedures in interrupt enabled context */ AcpiSetFirmwareWakingVector(0); if (sleep_result == 1 && mem_range_softc.mr_op != NULL && mem_range_softc.mr_op->reinit != NULL) mem_range_softc.mr_op->reinit(&mem_range_softc); } return (sleep_result); }
void machine_real_restart(unsigned int type) { void (*restart_lowmem)(unsigned int) = (void (*)(unsigned int)) real_mode_header->machine_real_restart_asm; local_irq_disable(); /* * Write zero to CMOS register number 0x0f, which the BIOS POST * routine will recognize as telling it to do a proper reboot. (Well * that's what this book in front of me says -- it may only apply to * the Phoenix BIOS though, it's not clear). At the same time, * disable NMIs by setting the top bit in the CMOS address register, * as we're about to do peculiar things to the CPU. I'm not sure if * `outb_p' is needed instead of just `outb'. Use it to be on the * safe side. (Yes, CMOS_WRITE does outb_p's. - Paul G.) */ spin_lock(&rtc_lock); CMOS_WRITE(0x00, 0x8f); spin_unlock(&rtc_lock); /* * Switch back to the initial page table. */ load_cr3(initial_page_table); /* * Write 0x1234 to absolute memory location 0x472. The BIOS reads * this on booting to tell it to "Bypass memory test (also warm * boot)". This seems like a fairly standard thing that gets set by * REBOOT.COM programs, and the previous reset routine did this * too. */ *((unsigned short *)0x472) = reboot_mode; /* Jump to the identity-mapped low memory code */ restart_lowmem(type); }
void __init efi_call_phys_prelog(void) { unsigned long vaddress; int pgd; int n_pgds; early_code_mapping_set_exec(1); local_irq_save(efi_flags); n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); save_pgd = kmalloc(n_pgds * sizeof(pgd_t), GFP_KERNEL); for (pgd = 0; pgd < n_pgds; pgd++) { save_pgd[pgd] = *pgd_offset_k(pgd * PGDIR_SIZE); vaddress = (unsigned long)__va(pgd * PGDIR_SIZE); set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress)); } #ifdef CONFIG_PAX_PER_CPU_PGD load_cr3(swapper_pg_dir); #endif __flush_tlb_all(); }
/* * paging_init() sets up the page tables - note that the first 8MB are * already mapped by head.S. * * This routines also unmaps the page at virtual kernel address 0, so * that we can trap those pesky NULL-reference errors in the kernel. */ void __init paging_init(void) { #ifdef CONFIG_X86_PAE set_nx(); if (nx_enabled) printk("NX (Execute Disable) protection: active\n"); #endif pagetable_init(); load_cr3(swapper_pg_dir); #ifdef CONFIG_X86_PAE /* * We will bail out later - printk doesn't work right now so * the user would just see a hanging kernel. */ if (cpu_has_pae) set_in_cr4(X86_CR4_PAE); #endif __flush_tlb_all(); kmap_init(); }
void __init efi_call_phys_prolog(void) { struct desc_ptr gdt_descr; #ifdef CONFIG_PAX_KERNEXEC struct desc_struct d; #endif local_irq_save(efi_rt_eflags); load_cr3(initial_page_table); __flush_tlb_all(); #ifdef CONFIG_PAX_KERNEXEC pack_descriptor(&d, 0, 0xFFFFF, 0x9B, 0xC); write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_CS, &d, DESCTYPE_S); pack_descriptor(&d, 0, 0xFFFFF, 0x93, 0xC); write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_DS, &d, DESCTYPE_S); #endif gdt_descr.address = __pa(get_cpu_gdt_table(0)); gdt_descr.size = GDT_SIZE - 1; load_gdt(&gdt_descr); }
/* * We cannot call mmdrop() because we are in interrupt context, * instead update mm->cpu_vm_mask. * * We need to reload %cr3 since the page tables may be going * away frm under us... */ static void inline leave_mm (unsigned long cpu) { BUG_ON(cpu_tlbstate[cpu].state == TLBSTATE_OK); clear_bit(cpu, &cpu_tlbstate[cpu].active_mm->cpu_vm_mask); load_cr3(swapper_pg_dir); }
static void drop_mm_ref(struct mm_struct *mm) { if (current->active_mm == mm) load_cr3(swapper_pg_dir); }
/* * First function called by the Xen PVH boot sequence. * * Set some Xen global variables and prepare the environment so it is * as similar as possible to what native FreeBSD init function expects. */ uint64_t hammer_time_xen(start_info_t *si, uint64_t xenstack) { uint64_t physfree; uint64_t *PT4 = (u_int64_t *)xenstack; uint64_t *PT3 = (u_int64_t *)(xenstack + PAGE_SIZE); uint64_t *PT2 = (u_int64_t *)(xenstack + 2 * PAGE_SIZE); int i; xen_domain_type = XEN_PV_DOMAIN; vm_guest = VM_GUEST_XEN; if ((si == NULL) || (xenstack == 0)) { xc_printf("ERROR: invalid start_info or xen stack, halting\n"); HYPERVISOR_shutdown(SHUTDOWN_crash); } xc_printf("FreeBSD PVH running on %s\n", si->magic); /* We use 3 pages of xen stack for the boot pagetables */ physfree = xenstack + 3 * PAGE_SIZE - KERNBASE; /* Setup Xen global variables */ HYPERVISOR_start_info = si; HYPERVISOR_shared_info = (shared_info_t *)(si->shared_info + KERNBASE); /* * Setup some misc global variables for Xen devices * * XXX: Devices that need these specific variables should * be rewritten to fetch this info by themselves from the * start_info page. */ xen_store = (struct xenstore_domain_interface *) (ptoa(si->store_mfn) + KERNBASE); console_page = (char *)(ptoa(si->console.domU.mfn) + KERNBASE); /* * Use the stack Xen gives us to build the page tables * as native FreeBSD expects to find them (created * by the boot trampoline). */ for (i = 0; i < (PAGE_SIZE / sizeof(uint64_t)); i++) { /* * Each slot of the level 4 pages points * to the same level 3 page */ PT4[i] = ((uint64_t)&PT3[0]) - KERNBASE; PT4[i] |= PG_V | PG_RW | PG_U; /* * Each slot of the level 3 pages points * to the same level 2 page */ PT3[i] = ((uint64_t)&PT2[0]) - KERNBASE; PT3[i] |= PG_V | PG_RW | PG_U; /* * The level 2 page slots are mapped with * 2MB pages for 1GB. */ PT2[i] = i * (2 * 1024 * 1024); PT2[i] |= PG_V | PG_RW | PG_PS | PG_U; } load_cr3(((uint64_t)&PT4[0]) - KERNBASE); /* Set the hooks for early functions that diverge from bare metal */ init_ops = xen_init_ops; apic_ops = xen_apic_ops; /* Now we can jump into the native init function */ return (hammer_time(0, physfree)); }
/* * We cannot call mmdrop() because we are in interrupt context, * instead update mm->cpu_vm_mask. * * We need to reload %cr3 since the page tables may be going * away from under us.. */ void leave_mm(int cpu) { BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK); cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask); load_cr3(swapper_pg_dir); }
int acpi_sleep_machdep(struct acpi_softc *sc, int state) { ACPI_STATUS status; struct pmap *pm; int ret; uint32_t cr3; u_long ef; ret = 0; if (sc->acpi_wakeaddr == 0) return (0); AcpiSetFirmwareWakingVector(sc->acpi_wakephys); ef = read_eflags(); /* * Temporarily switch to the kernel pmap because it provides an * identity mapping (setup at boot) for the low physical memory * region containing the wakeup code. */ pm = kernel_pmap; cr3 = rcr3(); #ifdef PAE load_cr3(vtophys(pm->pm_pdpt)); #else load_cr3(vtophys(pm->pm_pdir)); #endif ret_addr = 0; ACPI_DISABLE_IRQS(); if (acpi_savecpu()) { /* Execute Sleep */ intr_suspend(); p_gdt = (struct region_descriptor *) (sc->acpi_wakeaddr + physical_gdt); p_gdt->rd_limit = saved_gdt.rd_limit; p_gdt->rd_base = vtophys(saved_gdt.rd_base); WAKECODE_FIXUP(physical_esp, uint32_t, vtophys(r_esp)); WAKECODE_FIXUP(previous_cr0, uint32_t, r_cr0); WAKECODE_FIXUP(previous_cr2, uint32_t, r_cr2); WAKECODE_FIXUP(previous_cr3, uint32_t, r_cr3); WAKECODE_FIXUP(previous_cr4, uint32_t, r_cr4); WAKECODE_FIXUP(resume_beep, uint32_t, acpi_resume_beep); WAKECODE_FIXUP(reset_video, uint32_t, acpi_reset_video); WAKECODE_FIXUP(previous_tr, uint16_t, r_tr); WAKECODE_BCOPY(previous_gdt, struct region_descriptor, saved_gdt); WAKECODE_FIXUP(previous_ldt, uint16_t, saved_ldt); WAKECODE_BCOPY(previous_idt, struct region_descriptor, saved_idt); WAKECODE_FIXUP(where_to_recover, void *, acpi_restorecpu); WAKECODE_FIXUP(previous_ds, uint16_t, r_ds); WAKECODE_FIXUP(previous_es, uint16_t, r_es); WAKECODE_FIXUP(previous_fs, uint16_t, r_fs); WAKECODE_FIXUP(previous_gs, uint16_t, r_gs); WAKECODE_FIXUP(previous_ss, uint16_t, r_ss); if (bootverbose) acpi_printcpu(); /* Call ACPICA to enter the desired sleep state */ if (state == ACPI_STATE_S4 && sc->acpi_s4bios) status = AcpiEnterSleepStateS4bios(); else status = AcpiEnterSleepState(state); if (status != AE_OK) { device_printf(sc->acpi_dev, "AcpiEnterSleepState failed - %s\n", AcpiFormatException(status)); ret = -1; goto out; } for (;;) ; } else { /* Execute Wakeup */ intr_resume(); if (bootverbose) { acpi_savecpu(); acpi_printcpu(); } } out: load_cr3(cr3); write_eflags(ef); /* If we beeped, turn it off after a delay. */ if (acpi_resume_beep) timeout(acpi_stop_beep, NULL, 3 * hz); return (ret); }
int acpi_sleep_machdep(struct acpi_softc *sc, int state) { struct savefpu *stopfpu; #ifdef SMP cpumask_t wakeup_cpus; #endif register_t cr3, rf; ACPI_STATUS status; int ret; ret = -1; if (sc->acpi_wakeaddr == 0ul) return (ret); #ifdef SMP wakeup_cpus = PCPU_GET(other_cpus); #endif AcpiSetFirmwareWakingVector(WAKECODE_PADDR(sc)); rf = intr_disable(); intr_suspend(); /* * Temporarily switch to the kernel pmap because it provides * an identity mapping (setup at boot) for the low physical * memory region containing the wakeup code. */ cr3 = rcr3(); load_cr3(KPML4phys); stopfpu = &stopxpcbs[0].xpcb_pcb.pcb_save; if (acpi_savecpu(&stopxpcbs[0])) { fpugetregs(curthread, stopfpu); #ifdef SMP if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) { device_printf(sc->acpi_dev, "Failed to suspend APs: CPU mask = 0x%jx\n", (uintmax_t)(wakeup_cpus & ~stopped_cpus)); goto out; } #endif WAKECODE_FIXUP(resume_beep, uint8_t, (acpi_resume_beep != 0)); WAKECODE_FIXUP(reset_video, uint8_t, (acpi_reset_video != 0)); WAKECODE_FIXUP(wakeup_xpcb, struct xpcb *, &stopxpcbs[0]); WAKECODE_FIXUP(wakeup_gdt, uint16_t, stopxpcbs[0].xpcb_gdt.rd_limit); WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, stopxpcbs[0].xpcb_gdt.rd_base); WAKECODE_FIXUP(wakeup_cpu, int, 0); /* Call ACPICA to enter the desired sleep state */ if (state == ACPI_STATE_S4 && sc->acpi_s4bios) status = AcpiEnterSleepStateS4bios(); else status = AcpiEnterSleepState(state); if (status != AE_OK) { device_printf(sc->acpi_dev, "AcpiEnterSleepState failed - %s\n", AcpiFormatException(status)); goto out; } for (;;) ia32_pause(); } else {
static void schedule() { current_process = !current_process; load_cr3((uint32_t)upagedir[current_process]); }
/* * Activate a secondary processor. */ static void notrace start_secondary(void *unused) { /* * Don't put *anything* except direct CPU state initialization * before cpu_init(), SMP booting is too fragile that we want to * limit the things done here to the most necessary things. */ if (boot_cpu_has(X86_FEATURE_PCID)) __write_cr4(__read_cr4() | X86_CR4_PCIDE); #ifdef CONFIG_X86_32 /* switch away from the initial page table */ load_cr3(swapper_pg_dir); /* * Initialize the CR4 shadow before doing anything that could * try to read it. */ cr4_init_shadow(); __flush_tlb_all(); #endif load_current_idt(); cpu_init(); x86_cpuinit.early_percpu_clock_init(); preempt_disable(); smp_callin(); enable_start_cpu0 = 0; /* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier(); /* * Check TSC synchronization with the boot CPU: */ check_tsc_sync_target(); speculative_store_bypass_ht_init(); /* * Lock vector_lock, set CPU online and bring the vector * allocator online. Online must be set with vector_lock held * to prevent a concurrent irq setup/teardown from seeing a * half valid vector space. */ lock_vector_lock(); set_cpu_online(smp_processor_id(), true); lapic_online(); unlock_vector_lock(); cpu_set_state_online(smp_processor_id()); x86_platform.nmi_init(); /* enable local interrupts */ local_irq_enable(); /* to prevent fake stack check failure in clock setup */ boot_init_stack_canary(); x86_cpuinit.setup_percpu_clockev(); wmb(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); }
void __init kasan_init(void) { int i; void *shadow_cpu_entry_begin, *shadow_cpu_entry_end; #ifdef CONFIG_KASAN_INLINE register_die_notifier(&kasan_die_notifier); #endif memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt)); /* * We use the same shadow offset for 4- and 5-level paging to * facilitate boot-time switching between paging modes. * As result in 5-level paging mode KASAN_SHADOW_START and * KASAN_SHADOW_END are not aligned to PGD boundary. * * KASAN_SHADOW_START doesn't share PGD with anything else. * We claim whole PGD entry to make things easier. * * KASAN_SHADOW_END lands in the last PGD entry and it collides with * bunch of things like kernel code, modules, EFI mapping, etc. * We need to take extra steps to not overwrite them. */ if (pgtable_l5_enabled()) { void *ptr; ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END)); memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table)); set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)], __pgd(__pa(tmp_p4d_table) | _KERNPG_TABLE)); } load_cr3(early_top_pgt); __flush_tlb_all(); clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END); kasan_populate_early_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK), kasan_mem_to_shadow((void *)PAGE_OFFSET)); for (i = 0; i < E820_MAX_ENTRIES; i++) { if (pfn_mapped[i].end == 0) break; map_range(&pfn_mapped[i]); } shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE; shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); shadow_cpu_entry_begin = (void *)round_down( (unsigned long)shadow_cpu_entry_begin, PAGE_SIZE); shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE); shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); shadow_cpu_entry_end = (void *)round_up( (unsigned long)shadow_cpu_entry_end, PAGE_SIZE); kasan_populate_early_shadow( kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), shadow_cpu_entry_begin); kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, (unsigned long)shadow_cpu_entry_end, 0); kasan_populate_early_shadow(shadow_cpu_entry_end, kasan_mem_to_shadow((void *)__START_KERNEL_map)); kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext), (unsigned long)kasan_mem_to_shadow(_end), early_pfn_to_nid(__pa(_stext))); kasan_populate_early_shadow(kasan_mem_to_shadow((void *)MODULES_END), (void *)KASAN_SHADOW_END); load_cr3(init_top_pgt); __flush_tlb_all(); /* * kasan_early_shadow_page has been used as early shadow memory, thus * it may contain some garbage. Now we can clear and write protect it, * since after the TLB flush no one should write to it. */ memset(kasan_early_shadow_page, 0, PAGE_SIZE); for (i = 0; i < PTRS_PER_PTE; i++) { pte_t pte; pgprot_t prot; prot = __pgprot(__PAGE_KERNEL_RO | _PAGE_ENC); pgprot_val(prot) &= __default_kernel_pte_mask; pte = __pte(__pa(kasan_early_shadow_page) | pgprot_val(prot)); set_pte(&kasan_early_shadow_pte[i], pte); } /* Flush TLBs again to be sure that write protection applied. */ __flush_tlb_all(); init_task.kasan_depth = 0; pr_info("KernelAddressSanitizer initialized\n"); }
void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { unsigned cpu = smp_processor_id(); if (likely(prev != next)) { #ifdef CONFIG_SMP this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); this_cpu_write(cpu_tlbstate.active_mm, next); #endif cpumask_set_cpu(cpu, mm_cpumask(next)); /* * Re-load page tables. * * This logic has an ordering constraint: * * CPU 0: Write to a PTE for 'next' * CPU 0: load bit 1 in mm_cpumask. if nonzero, send IPI. * CPU 1: set bit 1 in next's mm_cpumask * CPU 1: load from the PTE that CPU 0 writes (implicit) * * We need to prevent an outcome in which CPU 1 observes * the new PTE value and CPU 0 observes bit 1 clear in * mm_cpumask. (If that occurs, then the IPI will never * be sent, and CPU 0's TLB will contain a stale entry.) * * The bad outcome can occur if either CPU's load is * reordered before that CPU's store, so both CPUs must * execute full barriers to prevent this from happening. * * Thus, switch_mm needs a full barrier between the * store to mm_cpumask and any operation that could load * from next->pgd. TLB fills are special and can happen * due to instruction fetches or for no reason at all, * and neither LOCK nor MFENCE orders them. * Fortunately, load_cr3() is serializing and gives the * ordering guarantee we need. * */ load_cr3(next->pgd); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); /* Stop flush ipis for the previous mm */ cpumask_clear_cpu(cpu, mm_cpumask(prev)); /* Load per-mm CR4 state */ load_mm_cr4(next); #ifdef CONFIG_MODIFY_LDT_SYSCALL /* * Load the LDT, if the LDT is different. * * It's possible that prev->context.ldt doesn't match * the LDT register. This can happen if leave_mm(prev) * was called and then modify_ldt changed * prev->context.ldt but suppressed an IPI to this CPU. * In this case, prev->context.ldt != NULL, because we * never set context.ldt to NULL while the mm still * exists. That means that next->context.ldt != * prev->context.ldt, because mms never share an LDT. */ if (unlikely(prev->context.ldt != next->context.ldt)) load_mm_ldt(next); #endif } #ifdef CONFIG_SMP else { this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next); if (!cpumask_test_cpu(cpu, mm_cpumask(next))) { /* * On established mms, the mm_cpumask is only changed * from irq context, from ptep_clear_flush() while in * lazy tlb mode, and here. Irqs are blocked during * schedule, protecting us from simultaneous changes. */ cpumask_set_cpu(cpu, mm_cpumask(next)); /* * We were in lazy tlb mode and leave_mm disabled * tlb flush IPI delivery. We must reload CR3 * to make sure to use no freed page tables. * * As above, load_cr3() is serializing and orders TLB * fills with respect to the mm_cpumask write. */ load_cr3(next->pgd); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); load_mm_cr4(next); load_mm_ldt(next); } } #endif }
void timer_handler() { uint64_t tmp = usec; uint32_t tmpSec = 0; uint32_t tmpMin = 0; uint32_t sec = 0; uint32_t min = 0; uint32_t hour = 0; if(tmp >= 100) { sec = tmp / 100; tmpSec = sec % 60; } if(sec >= 60) { min = sec / 60; tmpMin = min % 60; } if(min >= 60) { hour = min / 60; } tmp %= 100; tmp += (tmpSec*100); tmp += (tmpMin*10000); tmp += (hour*1000000); time_write(tmp); usec++; pic_sendEOI(32); offset = 0; colon = 0; //---------------------- scheduler -------------------------- uint64_t cur_rsp; task_struct *next_proc; task_struct *prev_proc = cur_proc; //when scheduler is on if(schedule_flag) { sleep_cnt_update(); if(!cur_proc) { next_proc = get_next_proc(); //printf("next_proc: %p\n", next_proc); //printf("next_proc->proc_name: %s\n", next_proc->proc_name); //printf("next_proc->rsp: %p\n", next_proc->rsp); //context_switch(next_proc); load_cr3(next_proc->mm_struct->pgd_t); //in_rsp(next_proc->rsp); if (next_proc->mode == USER) { //tss.rsp0 = (uint64_t)&next_proc->k_stack[511]; //the top addr of the stack //switch_to_ring3(); } } else { //store the rsp position before switch cur_rsp = out_rsp(); prev_proc->rsp = cur_rsp; add_proc(prev_proc); next_proc = get_next_proc(); //context_switch(next_proc); load_cr3(next_proc->mm_struct->pgd_t); //in_rsp(next_proc->rsp); if (next_proc->mode == USER) { //tss.rsp0 = (uint64_t)&next_proc->k_stack[511]; //the top addr of the stack //switch_to_ring3(); } } } }