unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(current->mm, file, flags);
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
if (len & ~HPAGE_MASK)
return -EINVAL;
/* Handle MAP_FIXED */
if (flags & MAP_FIXED) {
if (prepare_hugepage_range(file, addr, len))
return -EINVAL;
return addr;
}
/* This code assumes that RGN_HPAGE != 0. */
if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
addr = HPAGE_REGION_BASE;
info.flags = 0;
info.length = len;
info.low_limit = addr;
info.high_limit = HPAGE_REGION_BASE + RGN_MAP_LIMIT;
info.align_mask = PAGE_MASK & (HPAGE_SIZE - 1);
info.align_offset = 0;
info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
unsigned long
arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
long map_shared = (flags & MAP_SHARED);
unsigned long align_mask = 0;
struct mm_struct *mm = current->mm;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
/* handle fixed mapping: prevent overlap with huge pages */
if (flags & MAP_FIXED) {
if (is_hugepage_only_range(mm, addr, len))
return -EINVAL;
return addr;
}
#ifdef CONFIG_HUGETLB_PAGE
if (REGION_NUMBER(addr) == RGN_HPAGE)
addr = 0;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
addr = mm->free_area_cache;
else
#endif
if (!addr)
addr = TASK_UNMAPPED_BASE;
if (map_shared && (TASK_SIZE > 0xfffffffful))
/*
* For 64-bit tasks, align shared segments to 1MB to avoid potential
* performance penalty due to virtual aliasing (see ASDM). For 32-bit
* tasks, we prefer to avoid exhausting the address space too quickly by
* limiting alignment to a single page.
*/
align_mask = PAGE_MASK & (SHMLBA - 1);
info.flags = 0;
info.length = len;
info.low_limit = addr;
info.high_limit = TASK_SIZE;
info.align_mask = align_mask;
info.align_offset = 0;
info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long task_size = TASK_SIZE;
unsigned long offset = gr_rand_threadstack_offset(mm, file, flags);
if (test_thread_flag(TIF_32BIT))
task_size = STACK_TOP32;
if (len & ~HPAGE_MASK)
return -EINVAL;
if (len > task_size)
return -ENOMEM;
if (flags & MAP_FIXED) {
if (prepare_hugepage_range(file, addr, len))
return -EINVAL;
return addr;
}
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
addr = ALIGN(addr, HPAGE_SIZE);
vma = find_vma(mm, addr);
if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
if (mm->get_unmapped_area == arch_get_unmapped_area)
return hugetlb_get_unmapped_area_bottomup(file, addr, len,
pgoff, flags, offset);
else
return hugetlb_get_unmapped_area_topdown(file, addr, len,
pgoff, flags, offset);
}
static unsigned long arch_get_unmapped_area_common(struct file *filp,
unsigned long addr0, unsigned long len, unsigned long pgoff,
unsigned long flags, enum mmap_allocation_direction dir)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long addr = addr0;
int do_color_align;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
if (unlikely(len > TASK_SIZE))
return -ENOMEM;
if (flags & MAP_FIXED) {
/* Even MAP_FIXED mappings must reside within TASK_SIZE */
if (TASK_SIZE - len < addr)
return -EINVAL;
/*
* We do not accept a shared mapping if it would violate
* cache aliasing constraints.
*/
if ((flags & MAP_SHARED) &&
((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
return -EINVAL;
return addr;
}
do_color_align = 0;
if (filp || (flags & MAP_SHARED))
do_color_align = 1;
/* requesting a specific address */
#ifdef CONFIG_PAX_RANDMMAP
if (!(current->mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
else
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.length = len;
info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
info.threadstack_offset = offset;
if (dir == DOWN) {
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
info.low_limit = PAGE_SIZE;
info.high_limit = mm->mmap_base;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
return addr;
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
}
info.flags = 0;
info.low_limit = mm->mmap_base;
info.high_limit = TASK_SIZE;
return vm_unmapped_area(&info);
}
unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long pax_task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
}
info.flags = 0;
info.length = len;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
info.threadstack_offset = offset;
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE)) {
info.low_limit = 0x00110000UL;
info.high_limit = mm->start_code;
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
info.low_limit += mm->delta_mmap & 0x03FFF000UL;
#endif
if (info.low_limit < info.high_limit) {
addr = vm_unmapped_area(&info);
if (!IS_ERR_VALUE(addr))
return addr;
}
} else
#endif
info.low_limit = mm->mmap_base;
info.high_limit = pax_task_size;
return vm_unmapped_area(&info);
}
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len, const unsigned long pgoff,
const unsigned long flags)
{
struct vm_area_struct *vma;
struct mm_struct *mm = current->mm;
unsigned long addr = addr0, pax_task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
/* requested length too big for entire address space */
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE))
goto bottomup;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
}
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
info.length = len;
info.low_limit = PAGE_SIZE;
info.high_limit = mm->mmap_base;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
return addr;
VM_BUG_ON(addr != -ENOMEM);
bottomup:
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
return arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
}
