/*
* Enter a 2MB PDE mapping for the supplied VA/PA into the resume-time pmap
*/
void
hibernate_enter_resume_2m_pde(vaddr_t va, paddr_t pa)
{
pt_entry_t *pde, npde;
if (va < NBPD_L4) {
if (va < NBPD_L3) {
/* First 512GB and 1GB are already mapped */
pde = (pt_entry_t *)(HIBERNATE_PD_LOW +
(pl2_pi(va) * sizeof(pt_entry_t)));
npde = (pa & PG_LGFRAME) |
PG_RW | PG_V | PG_M | PG_PS | PG_U;
*pde = npde;
} else {
/* Map the 1GB containing region */
pde = (pt_entry_t *)(HIBERNATE_PDPT_LOW +
(pl3_pi(va) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PD_LOW2) | PG_RW | PG_V;
*pde = npde;
/* Map 2MB page */
pde = (pt_entry_t *)(HIBERNATE_PD_LOW2 +
(pl2_pi(va) * sizeof(pt_entry_t)));
npde = (pa & PG_LGFRAME) |
PG_RW | PG_V | PG_M | PG_PS | PG_U;
*pde = npde;
}
} else {
/* First map the 512GB containing region */
pde = (pt_entry_t *)(HIBERNATE_PML4T +
(pl4_pi(va) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PDPT_HI) | PG_RW | PG_V;
*pde = npde;
/* Map the 1GB containing region */
pde = (pt_entry_t *)(HIBERNATE_PDPT_HI +
(pl3_pi(va) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PD_HI) | PG_RW | PG_V;
*pde = npde;
/* Map the 2MB page */
pde = (pt_entry_t *)(HIBERNATE_PD_HI +
(pl2_pi(va) * sizeof(pt_entry_t)));
npde = (pa & PG_LGFRAME) | PG_RW | PG_V | PG_PS;
*pde = npde;
}
}
/*
* Translate a kernel virtual address to a physical address.
*/
int
_kvm_kvatop(kvm_t *kd, u_long va, paddr_t *pa)
{
cpu_kcore_hdr_t *cpu_kh;
paddr_t pde_pa, pte_pa;
u_long page_off;
pd_entry_t pde;
pt_entry_t pte;
if (ISALIVE(kd)) {
_kvm_err(kd, 0, "vatop called in live kernel!");
return (0);
}
page_off = va & (kd->nbpg - 1);
if (va >= PMAP_DIRECT_BASE && va <= PMAP_DIRECT_END) {
*pa = va - PMAP_DIRECT_BASE;
return (int)(kd->nbpg - page_off);
}
cpu_kh = kd->cpu_data;
/*
* Find and read all entries to get to the pa.
*/
/*
* Level 4.
*/
pde_pa = cpu_kh->ptdpaddr + (pl4_pi(va) * sizeof(pd_entry_t));
if (pread(kd->pmfd, (void *)&pde, sizeof(pde),
_kvm_pa2off(kd, pde_pa)) != sizeof(pde)) {
_kvm_syserr(kd, 0, "could not read PT level 4 entry");
goto lose;
}
if ((pde & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid level 4 PDE)");
goto lose;
}
/*
* Level 3.
*/
pde_pa = (pde & PG_FRAME) + (pl3_pi(va) * sizeof(pd_entry_t));
if (pread(kd->pmfd, (void *)&pde, sizeof(pde),
_kvm_pa2off(kd, pde_pa)) != sizeof(pde)) {
_kvm_syserr(kd, 0, "could not read PT level 3 entry");
goto lose;
}
if ((pde & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid level 3 PDE)");
goto lose;
}
/*
* Level 2.
*/
pde_pa = (pde & PG_FRAME) + (pl2_pi(va) * sizeof(pd_entry_t));
if (pread(kd->pmfd, (void *)&pde, sizeof(pde),
_kvm_pa2off(kd, pde_pa)) != sizeof(pde)) {
_kvm_syserr(kd, 0, "could not read PT level 2 entry");
goto lose;
}
if ((pde & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid level 2 PDE)");
goto lose;
}
/*
* Might be a large page.
*/
if ((pde & PG_PS) != 0) {
page_off = va & (NBPD_L2 - 1);
*pa = (pde & PG_LGFRAME) | page_off;
return (int)(NBPD_L2 - page_off);
}
/*
* Level 1.
*/
pte_pa = (pde & PG_FRAME) + (pl1_pi(va) * sizeof(pt_entry_t));
if (pread(kd->pmfd, (void *) &pte, sizeof(pte),
_kvm_pa2off(kd, pte_pa)) != sizeof(pte)) {
_kvm_syserr(kd, 0, "could not read PTE");
goto lose;
}
/*
* Validate the PTE and return the physical address.
*/
if ((pte & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid PTE)");
goto lose;
}
*pa = (pte & PG_FRAME) + page_off;
return (int)(kd->nbpg - page_off);
lose:
*pa = (u_long)~0L;
return (0);
}
/*
* Create the resume-time page table. This table maps the image(pig) area,
* the kernel text area, and various utility pages for use during resume,
* since we cannot overwrite the resuming kernel's page table during inflate
* and expect things to work properly.
*/
void
hibernate_populate_resume_pt(union hibernate_info *hib_info,
paddr_t image_start, paddr_t image_end)
{
int phys_page_number, i;
paddr_t pa;
vaddr_t kern_start_2m_va, kern_end_2m_va, page;
vaddr_t piglet_start_va, piglet_end_va;
pt_entry_t *pde, npde;
/* Identity map MMU pages */
pmap_kenter_pa(HIBERNATE_PML4T, HIBERNATE_PML4T, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PDPT_LOW, HIBERNATE_PDPT_LOW, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PDPT_HI, HIBERNATE_PDPT_HI, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PD_LOW, HIBERNATE_PD_LOW, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PD_LOW2, HIBERNATE_PD_LOW2, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PD_HI, HIBERNATE_PD_HI, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PT_LOW, HIBERNATE_PT_LOW, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PT_LOW2, HIBERNATE_PT_LOW2, PROT_MASK);
pmap_kenter_pa(HIBERNATE_PT_HI, HIBERNATE_PT_HI, PROT_MASK);
/* Identity map 3 pages for stack */
pmap_kenter_pa(HIBERNATE_STACK_PAGE, HIBERNATE_STACK_PAGE, PROT_MASK);
pmap_kenter_pa(HIBERNATE_STACK_PAGE - PAGE_SIZE,
HIBERNATE_STACK_PAGE - PAGE_SIZE, PROT_MASK);
pmap_kenter_pa(HIBERNATE_STACK_PAGE - 2*PAGE_SIZE,
HIBERNATE_STACK_PAGE - 2*PAGE_SIZE, PROT_MASK);
pmap_activate(curproc);
bzero((caddr_t)HIBERNATE_PML4T, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PDPT_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PDPT_HI, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_LOW2, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_HI, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_LOW2, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_HI, PAGE_SIZE);
bzero((caddr_t)(HIBERNATE_STACK_PAGE - 3*PAGE_SIZE) , 3*PAGE_SIZE);
/* First 512GB PML4E */
pde = (pt_entry_t *)(HIBERNATE_PML4T +
(pl4_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PDPT_LOW) | PG_RW | PG_V;
*pde = npde;
/* First 1GB PDPTE */
pde = (pt_entry_t *)(HIBERNATE_PDPT_LOW +
(pl3_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PD_LOW) | PG_RW | PG_V;
*pde = npde;
/* PD for first 2MB */
pde = (pt_entry_t *)(HIBERNATE_PD_LOW +
(pl2_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PT_LOW) | PG_RW | PG_V;
*pde = npde;
/*
* Identity map low physical pages.
* See arch/amd64/include/hibernate_var.h for page ranges used here.
*/
for (i = ACPI_TRAMPOLINE; i <= HIBERNATE_HIBALLOC_PAGE; i += PAGE_SIZE)
hibernate_enter_resume_mapping(i, i, 0);
/*
* Map current kernel VA range using 2MB pages
*/
kern_start_2m_va = (paddr_t)&start & ~(PAGE_MASK_L2);
kern_end_2m_va = (paddr_t)&end & ~(PAGE_MASK_L2);
/* amd64 kernels load at 16MB phys (on the 8th 2mb page) */
phys_page_number = 8;
for (page = kern_start_2m_va; page <= kern_end_2m_va;
page += NBPD_L2, phys_page_number++) {
pa = (paddr_t)(phys_page_number * NBPD_L2);
hibernate_enter_resume_mapping(page, pa, 1);
}
/*
* Identity map the piglet using 2MB pages.
*/
phys_page_number = hib_info->piglet_pa / NBPD_L2;
/* VA == PA */
piglet_start_va = hib_info->piglet_pa;
piglet_end_va = piglet_start_va + HIBERNATE_CHUNK_SIZE * 4;
for (page = piglet_start_va; page <= piglet_end_va;
page += NBPD_L2, phys_page_number++) {
pa = (paddr_t)(phys_page_number * NBPD_L2);
hibernate_enter_resume_mapping(page, pa, 1);
}
/* Unmap MMU pages (stack remains mapped) */
pmap_kremove(HIBERNATE_PML4T, PAGE_SIZE);
pmap_kremove(HIBERNATE_PDPT_LOW, PAGE_SIZE);
pmap_kremove(HIBERNATE_PDPT_HI, PAGE_SIZE);
pmap_kremove(HIBERNATE_PD_LOW, PAGE_SIZE);
pmap_kremove(HIBERNATE_PD_LOW2, PAGE_SIZE);
pmap_kremove(HIBERNATE_PD_HI, PAGE_SIZE);
pmap_kremove(HIBERNATE_PT_LOW, PAGE_SIZE);
pmap_kremove(HIBERNATE_PT_LOW2, PAGE_SIZE);
pmap_kremove(HIBERNATE_PT_HI, PAGE_SIZE);
pmap_activate(curproc);
}
/*
* Create the resume-time page table. This table maps the image(pig) area,
* the kernel text area, and various utility pages for use during resume,
* since we cannot overwrite the resuming kernel's page table during inflate
* and expect things to work properly.
*/
void
hibernate_populate_resume_pt(union hibernate_info *hib_info,
paddr_t image_start, paddr_t image_end)
{
int phys_page_number, i;
paddr_t pa, piglet_start, piglet_end;
vaddr_t kern_start_2m_va, kern_end_2m_va, page;
pt_entry_t *pde, npde;
/* Identity map MMU pages */
pmap_kenter_pa(HIBERNATE_PML4T, HIBERNATE_PML4T, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PDPT_LOW, HIBERNATE_PDPT_LOW, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PDPT_HI, HIBERNATE_PDPT_HI, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PD_LOW, HIBERNATE_PD_LOW, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PD_LOW2, HIBERNATE_PD_LOW2, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PD_HI, HIBERNATE_PD_HI, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PT_LOW, HIBERNATE_PT_LOW, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PT_LOW2, HIBERNATE_PT_LOW2, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_PT_HI, HIBERNATE_PT_HI, VM_PROT_ALL);
/* Identity map 3 pages for stack */
pmap_kenter_pa(HIBERNATE_STACK_PAGE, HIBERNATE_STACK_PAGE, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_STACK_PAGE - PAGE_SIZE,
HIBERNATE_STACK_PAGE - PAGE_SIZE, VM_PROT_ALL);
pmap_kenter_pa(HIBERNATE_STACK_PAGE - 2*PAGE_SIZE,
HIBERNATE_STACK_PAGE - 2*PAGE_SIZE, VM_PROT_ALL);
pmap_activate(curproc);
bzero((caddr_t)HIBERNATE_PML4T, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PDPT_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PDPT_HI, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_LOW2, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PD_HI, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_LOW, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_LOW2, PAGE_SIZE);
bzero((caddr_t)HIBERNATE_PT_HI, PAGE_SIZE);
bzero((caddr_t)(HIBERNATE_STACK_PAGE - 3*PAGE_SIZE) , 3*PAGE_SIZE);
/* First 512GB PML4E */
pde = (pt_entry_t *)(HIBERNATE_PML4T +
(pl4_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PDPT_LOW) | PG_RW | PG_V;
*pde = npde;
/* First 1GB PDPTE */
pde = (pt_entry_t *)(HIBERNATE_PDPT_LOW +
(pl3_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PD_LOW) | PG_RW | PG_V;
*pde = npde;
/* PD for first 2MB */
pde = (pt_entry_t *)(HIBERNATE_PD_LOW +
(pl2_pi(0) * sizeof(pt_entry_t)));
npde = (HIBERNATE_PT_LOW) | PG_RW | PG_V;
*pde = npde;
/*
* Identity map first 640KB physical for tramps and special utility
* pages using 4KB mappings
*/
for (i = 0; i < 160; i ++) {
hibernate_enter_resume_mapping(i*PAGE_SIZE, i*PAGE_SIZE, 0);
}
/*
* Map current kernel VA range using 2MB pages
*/
kern_start_2m_va = (paddr_t)&start & ~(PAGE_MASK_2M);
kern_end_2m_va = (paddr_t)&end & ~(PAGE_MASK_2M);
/* amd64 kernels load at 16MB phys (on the 8th 2mb page) */
phys_page_number = 8;
for (page = kern_start_2m_va; page <= kern_end_2m_va;
page += NBPD_L2, phys_page_number++) {
pa = (paddr_t)(phys_page_number * NBPD_L2);
hibernate_enter_resume_mapping(page, pa, 1);
}
/*
* Map the piglet
*/
phys_page_number = hib_info->piglet_pa / NBPD_L2;
piglet_start = hib_info->piglet_va;
piglet_end = piglet_start + HIBERNATE_CHUNK_SIZE * 3;
piglet_start &= ~(PAGE_MASK_2M);
piglet_end &= ~(PAGE_MASK_2M);
for (page = piglet_start; page <= piglet_end ;
page += NBPD_L2, phys_page_number++) {
pa = (paddr_t)(phys_page_number * NBPD_L2);
hibernate_enter_resume_mapping(page, pa, 1);
}
}
/*
* Used to translate a virtual address to a physical address for systems
* running under PAE mode. Three levels of virtual memory pages are handled
* here: the per-CPU L3 page, the 4 L2 PDs and the PTs.
*/
int
_kvm_kvatop_i386pae(kvm_t *kd, vaddr_t va, paddr_t *pa)
{
cpu_kcore_hdr_t *cpu_kh;
u_long page_off;
pd_entry_t pde;
pt_entry_t pte;
paddr_t pde_pa, pte_pa;
cpu_kh = kd->cpu_data;
page_off = va & PGOFSET;
/*
* Find and read the PDE. Ignore the L3, as it is only a per-CPU
* page, not needed for kernel VA => PA translations.
* Remember that the 4 L2 pages are contiguous, so it is safe
* to increment pdppaddr to compute the address of the PDE.
* pdppaddr being PAGE_SIZE aligned, we mask the option bits.
*/
pde_pa = (cpu_kh->pdppaddr & PG_FRAME) + (pl2_pi(va) * sizeof(pde));
if (_kvm_pread(kd, kd->pmfd, (void *)&pde, sizeof(pde),
_kvm_pa2off(kd, pde_pa)) != sizeof(pde)) {
_kvm_syserr(kd, 0, "could not read PDE");
goto lose;
}
/*
* Find and read the page table entry.
*/
if ((pde & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid PDE)");
goto lose;
}
if ((pde & PG_PS) != 0) {
/*
* This is a 2MB page.
*/
page_off = va & ((vaddr_t)~PG_LGFRAME);
*pa = (pde & PG_LGFRAME) + page_off;
return (int)(NBPD_L2 - page_off);
}
pte_pa = (pde & PG_FRAME) + (pl1_pi(va) * sizeof(pt_entry_t));
if (_kvm_pread(kd, kd->pmfd, (void *) &pte, sizeof(pte),
_kvm_pa2off(kd, pte_pa)) != sizeof(pte)) {
_kvm_syserr(kd, 0, "could not read PTE");
goto lose;
}
/*
* Validate the PTE and return the physical address.
*/
if ((pte & PG_V) == 0) {
_kvm_err(kd, 0, "invalid translation (invalid PTE)");
goto lose;
}
*pa = (pte & PG_FRAME) + page_off;
return (int)(NBPG - page_off);
lose:
*pa = (paddr_t)~0L;
return 0;
}