unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct vm_unmapped_area_info info; unsigned long offset = gr_rand_threadstack_offset(current->mm, file, flags); if (len > RGN_MAP_LIMIT) return -ENOMEM; if (len & ~HPAGE_MASK) return -EINVAL; /* Handle MAP_FIXED */ if (flags & MAP_FIXED) { if (prepare_hugepage_range(file, addr, len)) return -EINVAL; return addr; } /* This code assumes that RGN_HPAGE != 0. */ if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1))) addr = HPAGE_REGION_BASE; info.flags = 0; info.length = len; info.low_limit = addr; info.high_limit = HPAGE_REGION_BASE + RGN_MAP_LIMIT; info.align_mask = PAGE_MASK & (HPAGE_SIZE - 1); info.align_offset = 0; info.threadstack_offset = offset; return vm_unmapped_area(&info); }
unsigned long arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { long map_shared = (flags & MAP_SHARED); unsigned long align_mask = 0; struct mm_struct *mm = current->mm; struct vm_unmapped_area_info info; unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags); if (len > RGN_MAP_LIMIT) return -ENOMEM; /* handle fixed mapping: prevent overlap with huge pages */ if (flags & MAP_FIXED) { if (is_hugepage_only_range(mm, addr, len)) return -EINVAL; return addr; } #ifdef CONFIG_HUGETLB_PAGE if (REGION_NUMBER(addr) == RGN_HPAGE) addr = 0; #endif #ifdef CONFIG_PAX_RANDMMAP if (mm->pax_flags & MF_PAX_RANDMMAP) addr = mm->free_area_cache; else #endif if (!addr) addr = TASK_UNMAPPED_BASE; if (map_shared && (TASK_SIZE > 0xfffffffful)) /* * For 64-bit tasks, align shared segments to 1MB to avoid potential * performance penalty due to virtual aliasing (see ASDM). For 32-bit * tasks, we prefer to avoid exhausting the address space too quickly by * limiting alignment to a single page. */ align_mask = PAGE_MASK & (SHMLBA - 1); info.flags = 0; info.length = len; info.low_limit = addr; info.high_limit = TASK_SIZE; info.align_mask = align_mask; info.align_offset = 0; info.threadstack_offset = offset; return vm_unmapped_area(&info); }
unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long task_size = TASK_SIZE; unsigned long offset = gr_rand_threadstack_offset(mm, file, flags); if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; if (len & ~HPAGE_MASK) return -EINVAL; if (len > task_size) return -ENOMEM; if (flags & MAP_FIXED) { if (prepare_hugepage_range(file, addr, len)) return -EINVAL; return addr; } #ifdef CONFIG_PAX_RANDMMAP if (!(mm->pax_flags & MF_PAX_RANDMMAP)) #endif if (addr) { addr = ALIGN(addr, HPAGE_SIZE); vma = find_vma(mm, addr); if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset)) return addr; } if (mm->get_unmapped_area == arch_get_unmapped_area) return hugetlb_get_unmapped_area_bottomup(file, addr, len, pgoff, flags, offset); else return hugetlb_get_unmapped_area_topdown(file, addr, len, pgoff, flags, offset); }
static unsigned long arch_get_unmapped_area_common(struct file *filp, unsigned long addr0, unsigned long len, unsigned long pgoff, unsigned long flags, enum mmap_allocation_direction dir) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long addr = addr0; int do_color_align; unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags); struct vm_unmapped_area_info info; if (unlikely(len > TASK_SIZE)) return -ENOMEM; if (flags & MAP_FIXED) { /* Even MAP_FIXED mappings must reside within TASK_SIZE */ if (TASK_SIZE - len < addr) return -EINVAL; /* * We do not accept a shared mapping if it would violate * cache aliasing constraints. */ if ((flags & MAP_SHARED) && ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask)) return -EINVAL; return addr; } do_color_align = 0; if (filp || (flags & MAP_SHARED)) do_color_align = 1; /* requesting a specific address */ #ifdef CONFIG_PAX_RANDMMAP if (!(current->mm->pax_flags & MF_PAX_RANDMMAP)) #endif if (addr) { if (do_color_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset)) return addr; } info.length = len; info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0; info.align_offset = pgoff << PAGE_SHIFT; info.threadstack_offset = offset; if (dir == DOWN) { info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) return addr; /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ } info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; return vm_unmapped_area(&info); }
unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long pax_task_size = TASK_SIZE; struct vm_unmapped_area_info info; unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags); #ifdef CONFIG_PAX_SEGMEXEC if (mm->pax_flags & MF_PAX_SEGMEXEC) pax_task_size = SEGMEXEC_TASK_SIZE; #endif pax_task_size -= PAGE_SIZE; if (len > pax_task_size) return -ENOMEM; if (flags & MAP_FIXED) return addr; #ifdef CONFIG_PAX_RANDMMAP if (!(mm->pax_flags & MF_PAX_RANDMMAP)) #endif if (addr) { addr = PAGE_ALIGN(addr); if (pax_task_size - len >= addr) { vma = find_vma(mm, addr); if (check_heap_stack_gap(vma, addr, len, offset)) return addr; } } info.flags = 0; info.length = len; info.align_mask = filp ? get_align_mask() : 0; info.align_offset = pgoff << PAGE_SHIFT; info.threadstack_offset = offset; #ifdef CONFIG_PAX_PAGEEXEC if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE)) { info.low_limit = 0x00110000UL; info.high_limit = mm->start_code; #ifdef CONFIG_PAX_RANDMMAP if (mm->pax_flags & MF_PAX_RANDMMAP) info.low_limit += mm->delta_mmap & 0x03FFF000UL; #endif if (info.low_limit < info.high_limit) { addr = vm_unmapped_area(&info); if (!IS_ERR_VALUE(addr)) return addr; } } else #endif info.low_limit = mm->mmap_base; info.high_limit = pax_task_size; return vm_unmapped_area(&info); }
unsigned long arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long len, const unsigned long pgoff, const unsigned long flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0, pax_task_size = TASK_SIZE; struct vm_unmapped_area_info info; unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags); #ifdef CONFIG_PAX_SEGMEXEC if (mm->pax_flags & MF_PAX_SEGMEXEC) pax_task_size = SEGMEXEC_TASK_SIZE; #endif pax_task_size -= PAGE_SIZE; /* requested length too big for entire address space */ if (len > pax_task_size) return -ENOMEM; if (flags & MAP_FIXED) return addr; #ifdef CONFIG_PAX_PAGEEXEC if (!(__supported_pte_mask & _PAGE_NX) && (mm->pax_flags & MF_PAX_PAGEEXEC) && (flags & MAP_EXECUTABLE)) goto bottomup; #endif #ifdef CONFIG_PAX_RANDMMAP if (!(mm->pax_flags & MF_PAX_RANDMMAP)) #endif /* requesting a specific address */ if (addr) { addr = PAGE_ALIGN(addr); if (pax_task_size - len >= addr) { vma = find_vma(mm, addr); if (check_heap_stack_gap(vma, addr, len, offset)) return addr; } } info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; info.align_mask = filp ? get_align_mask() : 0; info.align_offset = pgoff << PAGE_SHIFT; info.threadstack_offset = offset; addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) return addr; VM_BUG_ON(addr != -ENOMEM); bottomup: /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ return arch_get_unmapped_area(filp, addr0, len, pgoff, flags); }