unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
struct vm_unmapped_area_info info;
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
if (len & ~HPAGE_MASK)
return -EINVAL;
/* Handle MAP_FIXED */
if (flags & MAP_FIXED) {
if (prepare_hugepage_range(file, addr, len))
return -EINVAL;
return addr;
}
/* This code assumes that RGN_HPAGE != 0. */
if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
addr = HPAGE_REGION_BASE;
info.flags = 0;
info.length = len;
info.low_limit = addr;
info.high_limit = HPAGE_REGION_BASE + RGN_MAP_LIMIT;
info.align_mask = PAGE_MASK & (HPAGE_SIZE - 1);
info.align_offset = 0;
return vm_unmapped_area(&info);
}
unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
struct vm_area_struct *vmm;
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
if (len & ~HPAGE_MASK)
return -EINVAL;
/* Handle MAP_FIXED */
if (flags & MAP_FIXED) {
if (prepare_hugepage_range(file, addr, len))
return -EINVAL;
return addr;
}
/* This code assumes that RGN_HPAGE != 0. */
if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
addr = HPAGE_REGION_BASE;
else
addr = ALIGN(addr, HPAGE_SIZE);
for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
/* At this point: (!vmm || addr < vmm->vm_end). */
if (REGION_OFFSET(addr) + len > RGN_MAP_LIMIT)
return -ENOMEM;
if (!vmm || (addr + len) <= vmm->vm_start)
return addr;
addr = ALIGN(vmm->vm_end, HPAGE_SIZE);
}
}
unsigned long
arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
long map_shared = (flags & MAP_SHARED);
unsigned long start_addr, align_mask = PAGE_SIZE - 1;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
/* handle fixed mapping: prevent overlap with huge pages */
if (flags & MAP_FIXED) {
if (is_hugepage_only_range(mm, addr, len))
return -EINVAL;
return addr;
}
#ifdef CONFIG_HUGETLB_PAGE
if (REGION_NUMBER(addr) == RGN_HPAGE)
addr = 0;
#endif
if (!addr)
addr = mm->free_area_cache;
if (map_shared && (TASK_SIZE > 0xfffffffful))
/*
* For 64-bit tasks, align shared segments to 1MB to avoid potential
* performance penalty due to virtual aliasing (see ASDM). For 32-bit
* tasks, we prefer to avoid exhausting the address space too quickly by
* limiting alignment to a single page.
*/
align_mask = SHMLBA - 1;
full_search:
start_addr = addr = (addr + align_mask) & ~align_mask;
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (TASK_SIZE - len < addr || RGN_MAP_LIMIT - len < REGION_OFFSET(addr)) {
if (start_addr != TASK_UNMAPPED_BASE) {
/* Start a new search --- just in case we missed some holes. */
addr = TASK_UNMAPPED_BASE;
goto full_search;
}
return -ENOMEM;
}
if (!vma || addr + len <= vma->vm_start) {
/* Remember the address where we stopped this search: */
mm->free_area_cache = addr + len;
return addr;
}
addr = (vma->vm_end + align_mask) & ~align_mask;
}
}
int prepare_hugepage_range(struct file *file,
unsigned long addr, unsigned long len)
{
if (len & ~HPAGE_MASK)
return -EINVAL;
if (addr & ~HPAGE_MASK)
return -EINVAL;
if (REGION_NUMBER(addr) != RGN_HPAGE)
return -EINVAL;
return 0;
}
unsigned long
arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
long map_shared = (flags & MAP_SHARED);
unsigned long align_mask = 0;
struct mm_struct *mm = current->mm;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
/* handle fixed mapping: prevent overlap with huge pages */
if (flags & MAP_FIXED) {
if (is_hugepage_only_range(mm, addr, len))
return -EINVAL;
return addr;
}
#ifdef CONFIG_HUGETLB_PAGE
if (REGION_NUMBER(addr) == RGN_HPAGE)
addr = 0;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
addr = mm->free_area_cache;
else
#endif
if (!addr)
addr = TASK_UNMAPPED_BASE;
if (map_shared && (TASK_SIZE > 0xfffffffful))
/*
* For 64-bit tasks, align shared segments to 1MB to avoid potential
* performance penalty due to virtual aliasing (see ASDM). For 32-bit
* tasks, we prefer to avoid exhausting the address space too quickly by
* limiting alignment to a single page.
*/
align_mask = PAGE_MASK & (SHMLBA - 1);
info.flags = 0;
info.length = len;
info.low_limit = addr;
info.high_limit = TASK_SIZE;
info.align_mask = align_mask;
info.align_offset = 0;
info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int write)
{
struct page *page;
pte_t *ptep;
if (REGION_NUMBER(addr) != RGN_HPAGE)
return ERR_PTR(-EINVAL);
ptep = huge_pte_offset(mm, addr);
if (!ptep || pte_none(*ptep))
return NULL;
page = pte_page(*ptep);
page += ((addr & ~HPAGE_MASK) >> PAGE_SHIFT);
return page;
}
void hugetlb_free_pgd_range(struct mmu_gather *tlb,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
/*
* This is called to free hugetlb page tables.
*
* The offset of these addresses from the base of the hugetlb
* region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that
* the standard free_pgd_range will free the right page tables.
*
* If floor and ceiling are also in the hugetlb region, they
* must likewise be scaled down; but if outside, left unchanged.
*/
addr = htlbpage_to_page(addr);
end = htlbpage_to_page(end);
if (REGION_NUMBER(floor) == RGN_HPAGE)
floor = htlbpage_to_page(floor);
if (REGION_NUMBER(ceiling) == RGN_HPAGE)
ceiling = htlbpage_to_page(ceiling);
free_pgd_range(tlb, addr, end, floor, ceiling);
}
/* Translate virtual address to physical address. */
unsigned long
xencomm_vtop(unsigned long vaddr)
{
struct page *page;
struct vm_area_struct *vma;
if (vaddr == 0)
return 0UL;
if (REGION_NUMBER(vaddr) == 5) {
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep;
/* On ia64, TASK_SIZE refers to current. It is not initialized
during boot.
Furthermore the kernel is relocatable and __pa() doesn't
work on addresses. */
if (vaddr >= KERNEL_START
&& vaddr < (KERNEL_START + KERNEL_TR_PAGE_SIZE))
return vaddr - kernel_virtual_offset;
/* In kernel area -- virtually mapped. */
pgd = pgd_offset_k(vaddr);
if (pgd_none(*pgd) || pgd_bad(*pgd))
return ~0UL;
pud = pud_offset(pgd, vaddr);
if (pud_none(*pud) || pud_bad(*pud))
return ~0UL;
pmd = pmd_offset(pud, vaddr);
if (pmd_none(*pmd) || pmd_bad(*pmd))
return ~0UL;
ptep = pte_offset_kernel(pmd, vaddr);
if (!ptep)
return ~0UL;
return (pte_val(*ptep) & _PFN_MASK) | (vaddr & ~PAGE_MASK);
}
if (vaddr > TASK_SIZE) {
/* percpu variables */
if (REGION_NUMBER(vaddr) == 7 &&
REGION_OFFSET(vaddr) >= (1ULL << IA64_MAX_PHYS_BITS))
ia64_tpa(vaddr);
/* kernel address */
return __pa(vaddr);
}
vma = find_extend_vma(current->mm, vaddr);
if (!vma)
return ~0UL;
/* We assume the page is modified. */
page = follow_page(vma, vaddr, FOLL_WRITE | FOLL_TOUCH);
if (!page)
return ~0UL;
return (page_to_pfn(page) << PAGE_SHIFT) | (vaddr & ~PAGE_MASK);
}
void
iounmap (volatile void __iomem *addr)
{
if (REGION_NUMBER(addr) == RGN_GATE)
vunmap((void *) ((unsigned long) addr & PAGE_MASK));
}
