unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct vm_area_struct * vmm;
int do_color_align;
if (len > TASK_SIZE)
return -ENOMEM;
if (flags & MAP_FIXED) {
/* Even MAP_FIXED mappings must reside within TASK_SIZE. */
if (TASK_SIZE - len < addr)
return -EINVAL;
/*
* We do not accept a shared mapping if it would violate
* cache aliasing constraints.
*/
if ((flags & MAP_SHARED) &&
((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
return -EINVAL;
return addr;
}
do_color_align = 0;
if (filp || (flags & MAP_SHARED))
do_color_align = 1;
#ifdef CONFIG_PAX_RANDMMAP
if (!(current->mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
else
addr = PAGE_ALIGN(addr);
vmm = find_vma(current->mm, addr);
if (TASK_SIZE - len >= addr && check_heap_stack_gap(vmm, addr, len))
return addr;
}
addr = current->mm->mmap_base;
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
else
addr = PAGE_ALIGN(addr);
for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
/* At this point: (!vmm || addr < vmm->vm_end). */
if (TASK_SIZE - len < addr)
return -ENOMEM;
if (check_heap_stack_gap(vmm, addr, len))
return addr;
addr = vmm->vm_end;
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
}
}
unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long task_size = TASK_SIZE;
if (test_thread_flag(TIF_32BIT))
task_size = STACK_TOP32;
if (len & ~HPAGE_MASK)
return -EINVAL;
if (len > task_size)
return -ENOMEM;
if (flags & MAP_FIXED) {
if (prepare_hugepage_range(file, addr, len))
return -EINVAL;
return addr;
}
if (addr) {
addr = ALIGN(addr, HPAGE_SIZE);
vma = find_vma(mm, addr);
if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len))
return addr;
}
if (mm->get_unmapped_area == arch_get_unmapped_area)
return hugetlb_get_unmapped_area_bottomup(file, addr, len,
pgoff, flags);
else
return hugetlb_get_unmapped_area_topdown(file, addr, len,
pgoff, flags);
}
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
unsigned long addr,
unsigned long len,
unsigned long pgoff,
unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct * vma;
unsigned long task_size = TASK_SIZE;
unsigned long start_addr;
if (test_thread_flag(TIF_32BIT))
task_size = STACK_TOP32;
if (unlikely(len >= VA_EXCLUDE_START))
return -ENOMEM;
if (len > mm->cached_hole_size) {
start_addr = addr = mm->free_area_cache;
} else {
start_addr = addr = TASK_UNMAPPED_BASE;
mm->cached_hole_size = 0;
}
task_size -= len;
full_search:
addr = ALIGN(addr, HPAGE_SIZE);
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (addr < VA_EXCLUDE_START &&
(addr + len) >= VA_EXCLUDE_START) {
addr = VA_EXCLUDE_END;
vma = find_vma(mm, VA_EXCLUDE_END);
}
if (unlikely(task_size < addr)) {
if (start_addr != TASK_UNMAPPED_BASE) {
start_addr = addr = TASK_UNMAPPED_BASE;
mm->cached_hole_size = 0;
goto full_search;
}
return -ENOMEM;
}
if (likely(check_heap_stack_gap(vma, addr, len))) {
/*
* Remember the place where we stopped the search:
*/
mm->free_area_cache = addr + len;
return addr;
}
if (addr + mm->cached_hole_size < vma->vm_start)
mm->cached_hole_size = vma->vm_start - addr;
addr = ALIGN(vma->vm_end, HPAGE_SIZE);
}
}
static unsigned long get_unshared_area(unsigned long addr, unsigned long len)
{
struct vm_area_struct *vma;
addr = PAGE_ALIGN(addr);
for (vma = find_vma(current->mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (TASK_SIZE - len < addr)
return -ENOMEM;
if (check_heap_stack_gap(vma, addr, len))
return addr;
addr = vma->vm_end;
}
}
static unsigned long get_shared_area(struct address_space *mapping,
unsigned long addr, unsigned long len, unsigned long pgoff)
{
struct vm_area_struct *vma;
int offset = mapping ? get_offset(mapping) : 0;
offset = (offset + (pgoff << PAGE_SHIFT)) & 0x3FF000;
addr = DCACHE_ALIGN(addr - offset) + offset;
for (vma = find_vma(current->mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (TASK_SIZE - len < addr)
return -ENOMEM;
if (check_heap_stack_gap(vma, addr, len))
return addr;
addr = DCACHE_ALIGN(vma->vm_end - offset) + offset;
if (addr < vma->vm_end) /* handle wraparound */
return -ENOMEM;
}
}
static unsigned long
hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len,
const unsigned long pgoff,
const unsigned long flags)
{
struct vm_area_struct *vma;
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
/* This should only ever run for 32-bit processes. */
BUG_ON(!test_thread_flag(TIF_32BIT));
/* check if free_area_cache is useful for us */
if (len <= mm->cached_hole_size) {
mm->cached_hole_size = 0;
mm->free_area_cache = mm->mmap_base;
}
/* either no address requested or can't fit in requested address hole */
addr = mm->free_area_cache & HPAGE_MASK;
/* make sure it can fit in the remaining address space */
if (likely(addr > len)) {
vma = find_vma(mm, addr-len);
if (check_heap_stack_gap(vma, addr - len, len)) {
/* remember the address as a hint for next time */
return (mm->free_area_cache = addr-len);
}
}
if (unlikely(mm->mmap_base < len))
goto bottomup;
addr = mm->mmap_base - len;
do {
addr &= HPAGE_MASK;
/*
* Lookup failure means no vma is above this address,
* else if new region fits below vma->vm_start,
* return with success:
*/
vma = find_vma(mm, addr);
if (likely(check_heap_stack_gap(vma, addr, len))) {
/* remember the address as a hint for next time */
return (mm->free_area_cache = addr);
}
/* remember the largest hole we saw so far */
if (addr + mm->cached_hole_size < vma->vm_start)
mm->cached_hole_size = vma->vm_start - addr;
/* try just below the current vma->vm_start */
addr = skip_heap_stack_gap(vma, len);
} while (!IS_ERR_VALUE(addr));
bottomup:
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
mm->cached_hole_size = ~0UL;
mm->free_area_cache = TASK_UNMAPPED_BASE;
addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
/*
* Restore the topdown base:
*/
mm->free_area_cache = mm->mmap_base;
mm->cached_hole_size = ~0UL;
return addr;
}
unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long start_addr, pax_task_size = TASK_SIZE;
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len))
return addr;
}
}
if (len > mm->cached_hole_size) {
start_addr = addr = mm->free_area_cache;
} else {
start_addr = addr = mm->mmap_base;
mm->cached_hole_size = 0;
}
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE) && start_addr >= mm->mmap_base) {
start_addr = 0x00110000UL;
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
start_addr += mm->delta_mmap & 0x03FFF000UL;
#endif
if (mm->start_brk <= start_addr && start_addr < mm->mmap_base)
start_addr = addr = mm->mmap_base;
else
addr = start_addr;
}
#endif
full_search:
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (pax_task_size - len < addr) {
/*
* Start a new search - just in case we missed
* some holes.
*/
if (start_addr != mm->mmap_base) {
start_addr = addr = mm->mmap_base;
mm->cached_hole_size = 0;
goto full_search;
}
return -ENOMEM;
}
if (check_heap_stack_gap(vma, addr, len))
break;
if (addr + mm->cached_hole_size < vma->vm_start)
mm->cached_hole_size = vma->vm_start - addr;
addr = vma->vm_end;
if (mm->start_brk <= addr && addr < mm->mmap_base) {
start_addr = addr = mm->mmap_base;
mm->cached_hole_size = 0;
goto full_search;
}
}
/*
* Remember the place where we stopped the search:
*/
mm->free_area_cache = addr + len;
return addr;
}
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len, const unsigned long pgoff,
const unsigned long flags)
{
struct vm_area_struct *vma;
struct mm_struct *mm = current->mm;
unsigned long base = mm->mmap_base, addr = addr0, pax_task_size = TASK_SIZE;
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
/* requested length too big for entire address space */
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE))
goto bottomup;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len))
return addr;
}
}
/* check if free_area_cache is useful for us */
if (len <= mm->cached_hole_size) {
mm->cached_hole_size = 0;
mm->free_area_cache = mm->mmap_base;
}
/* either no address requested or can't fit in requested address hole */
addr = mm->free_area_cache;
/* make sure it can fit in the remaining address space */
if (addr > len) {
vma = find_vma(mm, addr-len);
if (check_heap_stack_gap(vma, addr - len, len))
/* remember the address as a hint for next time */
return (mm->free_area_cache = addr-len);
}
if (mm->mmap_base < len)
goto bottomup;
addr = mm->mmap_base-len;
do {
/*
* Lookup failure means no vma is above this address,
* else if new region fits below vma->vm_start,
* return with success:
*/
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len))
/* remember the address as a hint for next time */
return (mm->free_area_cache = addr);
/* remember the largest hole we saw so far */
if (addr + mm->cached_hole_size < vma->vm_start)
mm->cached_hole_size = vma->vm_start - addr;
/* try just below the current vma->vm_start */
addr = skip_heap_stack_gap(vma, len);
} while (!IS_ERR_VALUE(addr));
bottomup:
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
mm->mmap_base = SEGMEXEC_TASK_UNMAPPED_BASE;
else
#endif
mm->mmap_base = TASK_UNMAPPED_BASE;
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
mm->mmap_base += mm->delta_mmap;
#endif
mm->free_area_cache = mm->mmap_base;
mm->cached_hole_size = ~0UL;
addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
/*
* Restore the topdown base:
*/
mm->mmap_base = base;
mm->free_area_cache = base;
mm->cached_hole_size = ~0UL;
return addr;
}
static unsigned long arch_get_unmapped_area_common(struct file *filp,
unsigned long addr0, unsigned long len, unsigned long pgoff,
unsigned long flags, enum mmap_allocation_direction dir)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long addr = addr0;
int do_color_align;
struct vm_unmapped_area_info info;
if (unlikely(len > TASK_SIZE))
return -ENOMEM;
if (flags & MAP_FIXED) {
/* Even MAP_FIXED mappings must reside within TASK_SIZE */
if (TASK_SIZE - len < addr)
return -EINVAL;
/*
* We do not accept a shared mapping if it would violate
* cache aliasing constraints.
*/
if ((flags & MAP_SHARED) &&
((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
return -EINVAL;
return addr;
}
do_color_align = 0;
if (filp || (flags & MAP_SHARED))
do_color_align = 1;
/* requesting a specific address */
#ifdef CONFIG_PAX_RANDMMAP
if (!(current->mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
else
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len))
return addr;
}
info.length = len;
info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
if (dir == DOWN) {
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
info.low_limit = PAGE_SIZE;
info.high_limit = mm->mmap_base;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
return addr;
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
}
info.flags = 0;
info.low_limit = mm->mmap_base;
info.high_limit = TASK_SIZE;
return vm_unmapped_area(&info);
}
unsigned long
arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
long map_shared = (flags & MAP_SHARED);
unsigned long start_addr, align_mask = PAGE_SIZE - 1;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
/* handle fixed mapping: prevent overlap with huge pages */
if (flags & MAP_FIXED) {
if (is_hugepage_only_range(mm, addr, len))
return -EINVAL;
return addr;
}
#ifdef CONFIG_HUGETLB_PAGE
if (REGION_NUMBER(addr) == RGN_HPAGE)
addr = 0;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
addr = mm->free_area_cache;
else
#endif
if (!addr)
addr = mm->free_area_cache;
if (map_shared && (TASK_SIZE > 0xfffffffful))
/*
* For 64-bit tasks, align shared segments to 1MB to avoid potential
* performance penalty due to virtual aliasing (see ASDM). For 32-bit
* tasks, we prefer to avoid exhausting the address space too quickly by
* limiting alignment to a single page.
*/
align_mask = SHMLBA - 1;
full_search:
start_addr = addr = (addr + align_mask) & ~align_mask;
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */
if (TASK_SIZE - len < addr || RGN_MAP_LIMIT - len < REGION_OFFSET(addr)) {
if (start_addr != mm->mmap_base) {
/* Start a new search --- just in case we missed some holes. */
addr = mm->mmap_base;
goto full_search;
}
return -ENOMEM;
}
if (check_heap_stack_gap(vma, addr, len)) {
/* Remember the address where we stopped this search: */
mm->free_area_cache = addr + len;
return addr;
}
addr = (vma->vm_end + align_mask) & ~align_mask;
}
}
unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long pax_task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
}
info.flags = 0;
info.length = len;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
info.threadstack_offset = offset;
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE)) {
info.low_limit = 0x00110000UL;
info.high_limit = mm->start_code;
#ifdef CONFIG_PAX_RANDMMAP
if (mm->pax_flags & MF_PAX_RANDMMAP)
info.low_limit += mm->delta_mmap & 0x03FFF000UL;
#endif
if (info.low_limit < info.high_limit) {
addr = vm_unmapped_area(&info);
if (!IS_ERR_VALUE(addr))
return addr;
}
} else
#endif
info.low_limit = mm->mmap_base;
info.high_limit = pax_task_size;
return vm_unmapped_area(&info);
}
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len, const unsigned long pgoff,
const unsigned long flags)
{
struct vm_area_struct *vma;
struct mm_struct *mm = current->mm;
unsigned long addr = addr0, pax_task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
#ifdef CONFIG_PAX_SEGMEXEC
if (mm->pax_flags & MF_PAX_SEGMEXEC)
pax_task_size = SEGMEXEC_TASK_SIZE;
#endif
pax_task_size -= PAGE_SIZE;
/* requested length too big for entire address space */
if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
#ifdef CONFIG_PAX_PAGEEXEC
if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE))
goto bottomup;
#endif
#ifdef CONFIG_PAX_RANDMMAP
if (!(mm->pax_flags & MF_PAX_RANDMMAP))
#endif
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
if (pax_task_size - len >= addr) {
vma = find_vma(mm, addr);
if (check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
}
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
info.length = len;
info.low_limit = PAGE_SIZE;
info.high_limit = mm->mmap_base;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
return addr;
VM_BUG_ON(addr != -ENOMEM);
bottomup:
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
return arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
}