/* Map a 4k page in a fixmap entry */ void set_fixmap(unsigned map, unsigned long mfn, unsigned attributes) { lpae_t pte = mfn_to_xen_entry(mfn, attributes); pte.pt.table = 1; /* 4k mappings always have this bit set */ pte.pt.xn = 1; write_pte(xen_fixmap + third_table_offset(FIXMAP_ADDR(map)), pte); flush_xen_data_tlb_range_va(FIXMAP_ADDR(map), PAGE_SIZE); }
void __kunmap_atomic(void *kv) { unsigned long kvaddr = (unsigned long)kv; if (kvaddr >= FIXMAP_BASE && kvaddr < (FIXMAP_BASE + FIXMAP_SIZE)) { /* * Because preemption is disabled, this vaddr can be associated * with the current allocated index. * But in case of multiple live kmap_atomic(), it still relies on * callers to unmap in right order. */ int cpu_idx = kmap_atomic_idx(); int idx = cpu_idx + KM_TYPE_NR * smp_processor_id(); WARN_ON(kvaddr != FIXMAP_ADDR(idx)); pte_clear(&init_mm, kvaddr, fixmap_page_table + idx); local_flush_tlb_kernel_range(kvaddr, kvaddr + PAGE_SIZE); kmap_atomic_idx_pop(); } pagefault_enable(); preempt_enable(); }
void *kmap_atomic(struct page *page) { int idx, cpu_idx; unsigned long vaddr; preempt_disable(); pagefault_disable(); if (!PageHighMem(page)) return page_address(page); cpu_idx = kmap_atomic_idx_push(); idx = cpu_idx + KM_TYPE_NR * smp_processor_id(); vaddr = FIXMAP_ADDR(idx); set_pte_at(&init_mm, vaddr, fixmap_page_table + idx, mk_pte(page, kmap_prot)); return (void *)vaddr; }
/* Checking VA memory layout alignment. */ static inline void check_memory_layout_alignment_constraints(void) { /* 2MB aligned regions */ BUILD_BUG_ON(XEN_VIRT_START & ~SECOND_MASK); BUILD_BUG_ON(FIXMAP_ADDR(0) & ~SECOND_MASK); BUILD_BUG_ON(BOOT_RELOC_VIRT_START & ~SECOND_MASK); /* 1GB aligned regions */ #ifdef CONFIG_ARM_32 BUILD_BUG_ON(XENHEAP_VIRT_START & ~FIRST_MASK); #else BUILD_BUG_ON(DIRECTMAP_VIRT_START & ~FIRST_MASK); #endif /* Page table structure constraints */ #ifdef CONFIG_ARM_64 BUILD_BUG_ON(zeroeth_table_offset(XEN_VIRT_START)); #endif BUILD_BUG_ON(first_table_offset(XEN_VIRT_START)); BUILD_BUG_ON(second_linear_offset(XEN_VIRT_START) >= LPAE_ENTRIES); #ifdef CONFIG_DOMAIN_PAGE BUILD_BUG_ON(DOMHEAP_VIRT_START & ~FIRST_MASK); #endif }
char *__acpi_map_table(paddr_t phys, unsigned long size) { unsigned long base, offset, mapped_size; int idx; offset = phys & (PAGE_SIZE - 1); mapped_size = PAGE_SIZE - offset; set_fixmap(FIXMAP_ACPI_BEGIN, phys >> PAGE_SHIFT, PAGE_HYPERVISOR); base = FIXMAP_ADDR(FIXMAP_ACPI_BEGIN); /* Most cases can be covered by the below. */ idx = FIXMAP_ACPI_BEGIN; while ( mapped_size < size ) { if ( ++idx > FIXMAP_ACPI_END ) return NULL; /* cannot handle this */ phys += PAGE_SIZE; set_fixmap(idx, phys >> PAGE_SHIFT, PAGE_HYPERVISOR); mapped_size += PAGE_SIZE; } return ((char *) base + offset); }
/* Remove a mapping from a fixmap entry */ void clear_fixmap(unsigned map) { lpae_t pte = {0}; write_pte(xen_fixmap + third_table_offset(FIXMAP_ADDR(map)), pte); flush_xen_data_tlb_range_va(FIXMAP_ADDR(map), PAGE_SIZE); }
/* Boot-time pagetable setup. * Changes here may need matching changes in head.S */ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr) { unsigned long dest_va; lpae_t pte, *p; int i; /* Map the destination in the boot misc area. */ dest_va = BOOT_MISC_VIRT_START; pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT); write_pte(xen_second + second_table_offset(dest_va), pte); flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE); /* Calculate virt-to-phys offset for the new location */ phys_offset = xen_paddr - (unsigned long) _start; /* Copy */ memcpy((void *) dest_va, _start, _end - _start); /* Beware! Any state we modify between now and the PT switch may be * discarded when we switch over to the copy. */ /* Update the copy of xen_pgtable to use the new paddrs */ p = (void *) xen_pgtable + dest_va - (unsigned long) _start; #ifdef CONFIG_ARM_64 p[0].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT; p = (void *) xen_first + dest_va - (unsigned long) _start; #endif for ( i = 0; i < 4; i++) p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT; p = (void *) xen_second + dest_va - (unsigned long) _start; if ( boot_phys_offset != 0 ) { /* Remove the old identity mapping of the boot paddr */ vaddr_t va = (vaddr_t)_start + boot_phys_offset; p[second_linear_offset(va)].bits = 0; } for ( i = 0; i < 4 * LPAE_ENTRIES; i++) if ( p[i].pt.valid ) p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT; /* Change pagetables to the copy in the relocated Xen */ boot_ttbr = (uintptr_t) xen_pgtable + phys_offset; flush_xen_dcache(boot_ttbr); flush_xen_dcache_va_range((void*)dest_va, _end - _start); flush_xen_text_tlb(); WRITE_SYSREG64(boot_ttbr, TTBR0_EL2); dsb(); /* Ensure visibility of HTTBR update */ flush_xen_text_tlb(); /* Undo the temporary map */ pte.bits = 0; write_pte(xen_second + second_table_offset(dest_va), pte); flush_xen_text_tlb(); /* Link in the fixmap pagetable */ pte = mfn_to_xen_entry((((unsigned long) xen_fixmap) + phys_offset) >> PAGE_SHIFT); pte.pt.table = 1; write_pte(xen_second + second_table_offset(FIXMAP_ADDR(0)), pte); /* * No flush required here. Individual flushes are done in * set_fixmap as entries are used. */ /* Break up the Xen mapping into 4k pages and protect them separately. */ for ( i = 0; i < LPAE_ENTRIES; i++ ) { unsigned long mfn = paddr_to_pfn(xen_paddr) + i; unsigned long va = XEN_VIRT_START + (i << PAGE_SHIFT); if ( !is_kernel(va) ) break; pte = mfn_to_xen_entry(mfn); pte.pt.table = 1; /* 4k mappings always have this bit set */ if ( is_kernel_text(va) || is_kernel_inittext(va) ) { pte.pt.xn = 0; pte.pt.ro = 1; } if ( is_kernel_rodata(va) ) pte.pt.ro = 1; write_pte(xen_xenmap + i, pte); /* No flush required here as page table is not hooked in yet. */ } pte = mfn_to_xen_entry((((unsigned long) xen_xenmap) + phys_offset) >> PAGE_SHIFT); pte.pt.table = 1; write_pte(xen_second + second_linear_offset(XEN_VIRT_START), pte); /* TLBFLUSH and ISB would be needed here, but wait until we set WXN */ /* From now on, no mapping may be both writable and executable. */ WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2); /* Flush everything after setting WXN bit. */ flush_xen_text_tlb(); }