asmlinkage unsigned long
ia64_brk (unsigned long brk, long arg1, long arg2, long arg3,
long arg4, long arg5, long arg6, long arg7, long stack)
{
extern int vm_enough_memory (long pages);
struct pt_regs *regs = (struct pt_regs *) &stack;
unsigned long rlim, retval, newbrk, oldbrk;
struct mm_struct *mm = current->mm;
/*
* Most of this replicates the code in sys_brk() except for an additional safety
* check and the clearing of r8. However, we can't call sys_brk() because we need
* to acquire the mmap_sem before we can do the test...
*/
down_write(&mm->mmap_sem);
if (brk < mm->end_code)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk)
goto set_brk;
/* Always allow shrinking brk. */
if (brk <= mm->brk) {
if (!do_munmap(mm, newbrk, oldbrk-newbrk,1))
goto set_brk;
goto out;
}
/* Check against unimplemented/unmapped addresses: */
if ((newbrk - oldbrk) > RGN_MAP_LIMIT || rgn_offset(newbrk) > RGN_MAP_LIMIT)
goto out;
/* Check against rlimit.. */
rlim = current->rlim[RLIMIT_DATA].rlim_cur;
if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim)
goto out;
/* Check against existing mmap mappings. */
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
/* Ok, looks good - let it rip. */
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
regs->r8 = 0; /* ensure large retval isn't mistaken as error code */
return retval;
}
asmlinkage int sunos_brk(u32 baddr)
{
int freepages, retval = -ENOMEM;
unsigned long rlim;
unsigned long newbrk, oldbrk, brk = (unsigned long) baddr;
down_write(¤t->mm->mmap_sem);
if (brk < current->mm->end_code)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(current->mm->brk);
retval = 0;
if (oldbrk == newbrk) {
current->mm->brk = brk;
goto out;
}
/* Always allow shrinking brk. */
if (brk <= current->mm->brk) {
current->mm->brk = brk;
do_munmap(current->mm, newbrk, oldbrk-newbrk);
goto out;
}
/* Check against rlimit and stack.. */
retval = -ENOMEM;
rlim = current->rlim[RLIMIT_DATA].rlim_cur;
if (rlim >= RLIM_INFINITY)
rlim = ~0;
if (brk - current->mm->end_code > rlim)
goto out;
/* Check against existing mmap mappings. */
if (find_vma_intersection(current->mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
/* stupid algorithm to decide if we have enough memory: while
* simple, it hopefully works in most obvious cases.. Easy to
* fool it, but this should catch most mistakes.
*/
freepages = get_page_cache_size();
freepages >>= 1;
freepages += nr_free_pages();
freepages += nr_swap_pages;
freepages -= num_physpages >> 4;
freepages -= (newbrk-oldbrk) >> PAGE_SHIFT;
if (freepages < 0)
goto out;
/* Ok, we have probably got enough memory - let it rip. */
current->mm->brk = brk;
do_brk(oldbrk, newbrk-oldbrk);
retval = 0;
out:
up_write(¤t->mm->mmap_sem);
return retval;
}
asmlinkage unsigned long
ia64_brk (unsigned long brk)
{
unsigned long rlim, retval, newbrk, oldbrk;
struct mm_struct *mm = current->mm;
/*
* Most of this replicates the code in sys_brk() except for an additional safety
* check and the clearing of r8. However, we can't call sys_brk() because we need
* to acquire the mmap_sem before we can do the test...
*/
down_write(&mm->mmap_sem);
if (brk < mm->end_code)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk)
goto set_brk;
/* Always allow shrinking brk. */
if (brk <= mm->brk) {
if (!do_munmap(mm, newbrk, oldbrk-newbrk))
goto set_brk;
goto out;
}
/* Check against unimplemented/unmapped addresses: */
if ((newbrk - oldbrk) > RGN_MAP_LIMIT || REGION_OFFSET(newbrk) > RGN_MAP_LIMIT)
goto out;
/* Check against rlimit.. */
rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim)
goto out;
/* Check against existing mmap mappings. */
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
/* Ok, looks good - let it rip. */
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
force_successful_syscall_return();
return retval;
}
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
unsigned long rlim, retval;
unsigned long newbrk, oldbrk;
struct mm_struct *mm = current->mm;
unsigned long min_brk;
down_write(&mm->mmap_sem);
#ifdef CONFIG_COMPAT_BRK
if (current->brk_randomized)
min_brk = mm->start_brk;
else
min_brk = mm->end_data;
#else
min_brk = mm->start_brk;
#endif
if (brk < min_brk)
goto out;
rlim = rlimit(RLIMIT_DATA);
if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
(mm->end_data - mm->start_data) > rlim)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk)
goto set_brk;
if (brk <= mm->brk) {
if (!do_munmap(mm, newbrk, oldbrk-newbrk))
goto set_brk;
goto out;
}
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
return retval;
}
// do_brk - adjust(increase/decrease) the size of process heap, align with page size
// NOTE: will change the process vma
int
do_brk(uintptr_t *brk_store) {
struct mm_struct *mm = current->mm;
if (mm == NULL) {
panic("kernel thread call sys_brk!!.\n");
}
if (brk_store == NULL) {
return -E_INVAL;
}
uintptr_t brk;
lock_mm(mm);
if (!copy_from_user(mm, &brk, brk_store, sizeof(uintptr_t), 1)) {
unlock_mm(mm);
return -E_INVAL;
}
if (brk < mm->brk_start) {
goto out_unlock;
}
uintptr_t newbrk = ROUNDUP(brk, PGSIZE), oldbrk = mm->brk;
assert(oldbrk % PGSIZE == 0);
if (newbrk == oldbrk) {
goto out_unlock;
}
if (newbrk < oldbrk) {
if (mm_unmap(mm, newbrk, oldbrk - newbrk) != 0) {
goto out_unlock;
}
}
else {
if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE) != NULL) {
goto out_unlock;
}
if (mm_brk(mm, oldbrk, newbrk - oldbrk) != 0) {
goto out_unlock;
}
}
mm->brk = newbrk;
out_unlock:
copy_to_user (mm, brk_store, &mm->brk, sizeof (uintptr_t));
unlock_mm(mm);
return 0;
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_memory_slot *memslot, unsigned long hva,
unsigned long fault_status)
{
int ret;
bool write_fault, writable, hugetlb = false, force_pte = false;
unsigned long mmu_seq;
gfn_t gfn = fault_ipa >> PAGE_SHIFT;
struct kvm *kvm = vcpu->kvm;
struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
struct vm_area_struct *vma;
pfn_t pfn;
pgprot_t mem_type = PAGE_S2;
bool fault_ipa_uncached;
write_fault = kvm_is_write_fault(vcpu);
if (fault_status == FSC_PERM && !write_fault) {
kvm_err("Unexpected L2 read permission error\n");
return -EFAULT;
}
/* Let's check if we will get back a huge page backed by hugetlbfs */
down_read(¤t->mm->mmap_sem);
vma = find_vma_intersection(current->mm, hva, hva + 1);
if (unlikely(!vma)) {
kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
up_read(¤t->mm->mmap_sem);
return -EFAULT;
}
if (is_vm_hugetlb_page(vma)) {
hugetlb = true;
gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
} else {
/*
* Pages belonging to memslots that don't have the same
* alignment for userspace and IPA cannot be mapped using
* block descriptors even if the pages belong to a THP for
* the process, because the stage-2 block descriptor will
* cover more than a single THP and we loose atomicity for
* unmapping, updates, and splits of the THP or other pages
* in the stage-2 block range.
*/
if ((memslot->userspace_addr & ~PMD_MASK) !=
/* poring from linux */
int do_linux_brk(uintptr_t brk)
{
uint32_t newbrk, oldbrk, retval;
struct mm_struct *mm = current->mm;
uint32_t min_brk;
if (!mm) {
panic("kernel thread call sys_brk!!.\n");
}
lock_mm(mm);
min_brk = mm->brk_start;
if (brk < min_brk)
goto out_unlock;
newbrk = ROUNDUP(brk, PGSIZE);
oldbrk = ROUNDUP(mm->brk, PGSIZE);
if (oldbrk == newbrk)
goto set_brk;
if (brk <= mm->brk) {
if (!mm_unmap(mm, newbrk, oldbrk - newbrk))
goto set_brk;
goto out_unlock;
}
if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE))
goto out_unlock;
/* set the brk */
if (mm_brk(mm, oldbrk, newbrk - oldbrk))
goto out_unlock;
set_brk:
mm->brk = brk;
out_unlock:
retval = mm->brk;
unlock_mm(mm);
return retval;
}
/*
* sys_brk() for the most part doesn't need the global kernel
* lock, except when an application is doing something nasty
* like trying to un-brk an area that has already been mapped
* to a regular file. in this case, the unmapping will need
* to invoke file system routines that need the global lock.
*/
asmlinkage unsigned long sys_brk(unsigned long brk)
{
unsigned long rlim, retval;
unsigned long newbrk, oldbrk;
struct mm_struct *mm = current->mm;
down_write(&mm->mmap_sem);
if (brk < mm->end_code)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk)
goto set_brk;
/* Always allow shrinking brk. */
if (brk <= mm->brk) {
if (!do_munmap(mm, newbrk, oldbrk-newbrk))
goto set_brk;
goto out;
}
/* Check against rlimit.. */
rlim = current->rlim[RLIMIT_DATA].rlim_cur;
if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim)
goto out;
/* Check against existing mmap mappings. */
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
/* Ok, looks good - let it rip. */
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
return retval;
}
/* from x86 bionic porting */
int
do_linux_brk(uintptr_t brk) {
struct mm_struct *mm = current->mm;
if (mm == NULL) {
panic("kernel thread call sys_brk!!.\n");
}
if (brk == 0) {
return mm->brk_start;
}
lock_mm(mm);
if (brk < mm->brk_start) {
goto out_unlock;
}
uintptr_t newbrk = ROUNDUP(brk, PGSIZE), oldbrk = mm->brk;
assert(oldbrk % PGSIZE == 0);
if (newbrk == oldbrk) {
goto out_unlock;
}
if (newbrk < oldbrk) {
if (mm_unmap(mm, newbrk, oldbrk - newbrk) != 0) {
goto out_unlock;
}
}
else {
if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE) != NULL) {
goto out_unlock;
}
if (mm_brk(mm, oldbrk, newbrk - oldbrk) != 0) {
goto out_unlock;
}
}
mm->brk = newbrk;
out_unlock:
unlock_mm(mm);
return newbrk;
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_memory_slot *memslot, unsigned long hva,
unsigned long fault_status)
{
int ret;
bool write_fault, writable, hugetlb = false, force_pte = false;
unsigned long mmu_seq;
gfn_t gfn = fault_ipa >> PAGE_SHIFT;
struct kvm *kvm = vcpu->kvm;
struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
struct vm_area_struct *vma;
pfn_t pfn;
pgprot_t mem_type = PAGE_S2;
bool fault_ipa_uncached;
bool logging_active = memslot_is_logging(memslot);
unsigned long flags = 0;
write_fault = kvm_is_write_fault(vcpu);
if (fault_status == FSC_PERM && !write_fault) {
kvm_err("Unexpected L2 read permission error\n");
return -EFAULT;
}
/* Let's check if we will get back a huge page backed by hugetlbfs */
down_read(¤t->mm->mmap_sem);
vma = find_vma_intersection(current->mm, hva, hva + 1);
if (unlikely(!vma)) {
kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
up_read(¤t->mm->mmap_sem);
return -EFAULT;
}
if (is_vm_hugetlb_page(vma) && !logging_active) {
hugetlb = true;
gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
} else {
unsigned long sys_brk(unsigned long brk)
{
unsigned long rlim;
unsigned long newbrk, oldbrk;
if (brk < current->mm->end_code)
return current->mm->brk;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(current->mm->brk);
if (oldbrk == newbrk)
return current->mm->brk = brk;
/*
* Always allow shrinking brk
*/
if (brk <= current->mm->brk) {
current->mm->brk = brk;
do_munmap(newbrk, oldbrk - newbrk);
return brk;
}
/*
* Check against rlimit
*/
rlim = current->rlim[RLIMIT_DATA].rlim_cur;
if (brk - current->mm->end_code > rlim)
return current->mm->brk;
/*
* Check against existing mmap mappings
*/
if (find_vma_intersection(current->mm, oldbrk, newbrk + PAGE_SIZE))
return current->mm->brk;
current->mm->brk = brk;
do_mmap(oldbrk, newbrk - oldbrk, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return brk;
}
/**
* get_vaddr_frames() - map virtual addresses to pfns
* @start: starting user address
* @nr_frames: number of pages / pfns from start to map
* @write: whether pages will be written to by the caller
* @force: whether to force write access even if user mapping is
* readonly. See description of the same argument of
get_user_pages().
* @vec: structure which receives pages / pfns of the addresses mapped.
* It should have space for at least nr_frames entries.
*
* This function maps virtual addresses from @start and fills @vec structure
* with page frame numbers or page pointers to corresponding pages (choice
* depends on the type of the vma underlying the virtual address). If @start
* belongs to a normal vma, the function grabs reference to each of the pages
* to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't
* touch page structures and the caller must make sure pfns aren't reused for
* anything else while he is using them.
*
* The function returns number of pages mapped which may be less than
* @nr_frames. In particular we stop mapping if there are more vmas of
* different type underlying the specified range of virtual addresses.
* When the function isn't able to map a single page, it returns error.
*
* This function takes care of grabbing mmap_sem as necessary.
*/
int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
bool write, bool force, struct frame_vector *vec)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int ret = 0;
int err;
int locked;
if (nr_frames == 0)
return 0;
if (WARN_ON_ONCE(nr_frames > vec->nr_allocated))
nr_frames = vec->nr_allocated;
down_read(&mm->mmap_sem);
locked = 1;
vma = find_vma_intersection(mm, start, start + 1);
if (!vma) {
ret = -EFAULT;
goto out;
}
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
vec->got_ref = true;
vec->is_pfns = false;
ret = get_user_pages_locked(current, mm, start, nr_frames,
write, force, (struct page **)(vec->ptrs), &locked);
goto out;
}
vec->got_ref = false;
vec->is_pfns = true;
do {
unsigned long *nums = frame_vector_pfns(vec);
while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) {
err = follow_pfn(vma, start, &nums[ret]);
if (err) {
if (ret == 0)
ret = err;
goto out;
}
start += PAGE_SIZE;
ret++;
}
/*
* We stop if we have enough pages or if VMA doesn't completely
* cover the tail page.
*/
if (ret >= nr_frames || start < vma->vm_end)
break;
vma = find_vma_intersection(mm, start, start + 1);
} while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP));
out:
if (locked)
up_read(&mm->mmap_sem);
if (!ret)
ret = -EFAULT;
if (ret > 0)
vec->nr_frames = ret;
return ret;
}
/**
* get_vaddr_frames() - map virtual addresses to pfns
* @start: starting user address
* @nr_frames: number of pages / pfns from start to map
* @gup_flags: flags modifying lookup behaviour
* @vec: structure which receives pages / pfns of the addresses mapped.
* It should have space for at least nr_frames entries.
*
* This function maps virtual addresses from @start and fills @vec structure
* with page frame numbers or page pointers to corresponding pages (choice
* depends on the type of the vma underlying the virtual address). If @start
* belongs to a normal vma, the function grabs reference to each of the pages
* to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't
* touch page structures and the caller must make sure pfns aren't reused for
* anything else while he is using them.
*
* The function returns number of pages mapped which may be less than
* @nr_frames. In particular we stop mapping if there are more vmas of
* different type underlying the specified range of virtual addresses.
* When the function isn't able to map a single page, it returns error.
*
* This function takes care of grabbing mmap_sem as necessary.
*/
int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
unsigned int gup_flags, struct frame_vector *vec)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int ret = 0;
int err;
int locked;
if (nr_frames == 0)
return 0;
if (WARN_ON_ONCE(nr_frames > vec->nr_allocated))
nr_frames = vec->nr_allocated;
down_read(&mm->mmap_sem);
locked = 1;
vma = find_vma_intersection(mm, start, start + 1);
if (!vma) {
ret = -EFAULT;
goto out;
}
/*
* While get_vaddr_frames() could be used for transient (kernel
* controlled lifetime) pinning of memory pages all current
* users establish long term (userspace controlled lifetime)
* page pinning. Treat get_vaddr_frames() like
* get_user_pages_longterm() and disallow it for filesystem-dax
* mappings.
*/
if (vma_is_fsdax(vma))
return -EOPNOTSUPP;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
vec->got_ref = true;
vec->is_pfns = false;
ret = get_user_pages_locked(start, nr_frames,
gup_flags, (struct page **)(vec->ptrs), &locked);
goto out;
}
vec->got_ref = false;
vec->is_pfns = true;
do {
unsigned long *nums = frame_vector_pfns(vec);
while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) {
err = follow_pfn(vma, start, &nums[ret]);
if (err) {
if (ret == 0)
ret = err;
goto out;
}
start += PAGE_SIZE;
ret++;
}
/*
* We stop if we have enough pages or if VMA doesn't completely
* cover the tail page.
*/
if (ret >= nr_frames || start < vma->vm_end)
break;
vma = find_vma_intersection(mm, start, start + 1);
} while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP));
out:
if (locked)
up_read(&mm->mmap_sem);
if (!ret)
ret = -EFAULT;
if (ret > 0)
vec->nr_frames = ret;
return ret;
}
/*
* Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
*/
asmlinkage int sys_shmat (int shmid, char *shmaddr, int shmflg, ulong *raddr)
{
struct shmid_kernel *shp;
struct vm_area_struct *shmd;
int err = -EINVAL;
unsigned int id;
unsigned long addr;
unsigned long len;
down(¤t->mm->mmap_sem);
lock_kernel();
if (shmid < 0) {
/* printk("shmat() -> EINVAL because shmid = %d < 0\n",shmid); */
goto out;
}
shp = shm_segs[id = (unsigned int) shmid % SHMMNI];
if (shp == IPC_UNUSED || shp == IPC_NOID) {
/* printk("shmat() -> EINVAL because shmid = %d is invalid\n",shmid); */
goto out;
}
if (!(addr = (ulong) shmaddr)) {
if (shmflg & SHM_REMAP)
goto out;
err = -ENOMEM;
addr = 0;
again:
if (!(addr = get_unmapped_area(addr, shp->u.shm_segsz)))
goto out;
if(addr & (SHMLBA - 1)) {
addr = (addr + (SHMLBA - 1)) & ~(SHMLBA - 1);
goto again;
}
} else if (addr & (SHMLBA-1)) {
if (shmflg & SHM_RND)
addr &= ~(SHMLBA-1); /* round down */
else
goto out;
}
/*
* Check if addr exceeds TASK_SIZE (from do_mmap)
*/
len = PAGE_SIZE*shp->shm_npages;
err = -EINVAL;
if (addr >= TASK_SIZE || len > TASK_SIZE || addr > TASK_SIZE - len)
goto out;
/*
* If shm segment goes below stack, make sure there is some
* space left for the stack to grow (presently 4 pages).
*/
if (addr < current->mm->start_stack &&
addr > current->mm->start_stack - PAGE_SIZE*(shp->shm_npages + 4))
{
/* printk("shmat() -> EINVAL because segment intersects stack\n"); */
goto out;
}
if (!(shmflg & SHM_REMAP))
if ((shmd = find_vma_intersection(current->mm, addr, addr + shp->u.shm_segsz))) {
/* printk("shmat() -> EINVAL because the interval [0x%lx,0x%lx) intersects an already mapped interval [0x%lx,0x%lx).\n",
addr, addr + shp->shm_segsz, shmd->vm_start, shmd->vm_end); */
goto out;
}
err = -EACCES;
if (ipcperms(&shp->u.shm_perm, shmflg & SHM_RDONLY ? S_IRUGO : S_IRUGO|S_IWUGO))
goto out;
err = -EIDRM;
if (shp->u.shm_perm.seq != (unsigned int) shmid / SHMMNI)
goto out;
err = -ENOMEM;
shmd = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
if (!shmd)
goto out;
if ((shp != shm_segs[id]) || (shp->u.shm_perm.seq != (unsigned int) shmid / SHMMNI)) {
kmem_cache_free(vm_area_cachep, shmd);
err = -EIDRM;
goto out;
}
shmd->vm_pte = SWP_ENTRY(SHM_SWP_TYPE, id);
shmd->vm_start = addr;
shmd->vm_end = addr + shp->shm_npages * PAGE_SIZE;
shmd->vm_mm = current->mm;
shmd->vm_page_prot = (shmflg & SHM_RDONLY) ? PAGE_READONLY : PAGE_SHARED;
shmd->vm_flags = VM_SHM | VM_MAYSHARE | VM_SHARED
| VM_MAYREAD | VM_MAYEXEC | VM_READ | VM_EXEC
| ((shmflg & SHM_RDONLY) ? 0 : VM_MAYWRITE | VM_WRITE);
shmd->vm_file = NULL;
shmd->vm_offset = 0;
shmd->vm_ops = &shm_vm_ops;
shp->u.shm_nattch++; /* prevent destruction */
if (shp->u.shm_nattch > 0xffff - NR_TASKS || (err = shm_map (shmd))) {
if (--shp->u.shm_nattch <= 0 && shp->u.shm_perm.mode & SHM_DEST)
killseg(id);
kmem_cache_free(vm_area_cachep, shmd);
goto out;
}
insert_attach(shp,shmd); /* insert shmd into shp->attaches */
shp->u.shm_lpid = current->pid;
shp->u.shm_atime = CURRENT_TIME;
*raddr = addr;
err = 0;
out:
unlock_kernel();
up(¤t->mm->mmap_sem);
return err;
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_memory_slot *memslot,
unsigned long fault_status)
{
int ret;
bool write_fault, writable, hugetlb = false;
unsigned long mmu_seq;
gfn_t gfn = fault_ipa >> PAGE_SHIFT;
unsigned long hva = gfn_to_hva(vcpu->kvm, gfn);
struct kvm *kvm = vcpu->kvm;
struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
struct vm_area_struct *vma;
pfn_t pfn;
write_fault = kvm_is_write_fault(kvm_vcpu_get_hsr(vcpu));
if (fault_status == FSC_PERM && !write_fault) {
kvm_err("Unexpected L2 read permission error\n");
return -EFAULT;
}
/* Let's check if we will get back a huge page backed by hugetlbfs */
down_read(¤t->mm->mmap_sem);
vma = find_vma_intersection(current->mm, hva, hva + 1);
if (is_vm_hugetlb_page(vma)) {
hugetlb = true;
gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
}
up_read(¤t->mm->mmap_sem);
/* We need minimum second+third level pages */
ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS);
if (ret)
return ret;
mmu_seq = vcpu->kvm->mmu_notifier_seq;
/*
* Ensure the read of mmu_notifier_seq happens before we call
* gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
* the page we just got a reference to gets unmapped before we have a
* chance to grab the mmu_lock, which ensure that if the page gets
* unmapped afterwards, the call to kvm_unmap_hva will take it away
* from us again properly. This smp_rmb() interacts with the smp_wmb()
* in kvm_mmu_notifier_invalidate_<page|range_end>.
*/
smp_rmb();
pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
if (is_error_pfn(pfn))
return -EFAULT;
spin_lock(&kvm->mmu_lock);
if (mmu_notifier_retry(kvm, mmu_seq))
goto out_unlock;
if (hugetlb) {
pmd_t new_pmd = pfn_pmd(pfn, PAGE_S2);
new_pmd = pmd_mkhuge(new_pmd);
if (writable) {
kvm_set_s2pmd_writable(&new_pmd);
kvm_set_pfn_dirty(pfn);
}
coherent_icache_guest_page(kvm, hva & PMD_MASK, PMD_SIZE);
ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
} else {
pte_t new_pte = pfn_pte(pfn, PAGE_S2);
if (writable) {
kvm_set_s2pte_writable(&new_pte);
kvm_set_pfn_dirty(pfn);
}
coherent_icache_guest_page(kvm, hva, PAGE_SIZE);
ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, false);
}
out_unlock:
spin_unlock(&kvm->mmu_lock);
kvm_release_pfn_clean(pfn);
return ret;
}
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
unsigned long rlim, retval;
unsigned long newbrk, oldbrk;
struct mm_struct *mm = current->mm;
unsigned long min_brk;
down_write(&mm->mmap_sem);
#ifdef CONFIG_COMPAT_BRK
/*
* CONFIG_COMPAT_BRK can still be overridden by setting
* randomize_va_space to 2, which will still cause mm->start_brk
* to be arbitrarily shifted
*/
if (current->brk_randomized)
min_brk = mm->start_brk;
else
min_brk = mm->end_data;
#else
min_brk = mm->start_brk;
#endif
if (brk < min_brk)
goto out;
/*
* Check against rlimit here. If this check is done later after the test
* of oldbrk with newbrk then it can escape the test and let the data
* segment grow beyond its set limit the in case where the limit is
* not page aligned -Ram Gupta
*/
rlim = rlimit(RLIMIT_DATA);
if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
(mm->end_data - mm->start_data) > rlim)
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk)
goto set_brk;
/* Always allow shrinking brk. */
if (brk <= mm->brk) {
if (!do_munmap(mm, newbrk, oldbrk-newbrk))
goto set_brk;
goto out;
}
/* Check against existing mmap mappings. */
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
goto out;
/* Ok, looks good - let it rip. */
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
return retval;
}
/* SunOS is completely broken... it returns 0 on success, otherwise
* ENOMEM. For sys_sbrk() it wants the new brk value as a return
* on success and ENOMEM as before on failure.
*/
asmlinkage int sunos_brk(unsigned long brk)
{
int freepages;
unsigned long rlim;
unsigned long newbrk, oldbrk;
if (brk < current->mm->end_code)
return -ENOMEM;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(current->mm->brk);
if (oldbrk == newbrk) {
current->mm->brk = brk;
return 0;
}
/*
* Always allow shrinking brk
*/
if (brk <= current->mm->brk) {
current->mm->brk = brk;
do_munmap(newbrk, oldbrk-newbrk);
return 0;
}
/*
* Check against rlimit and stack..
*/
rlim = current->rlim[RLIMIT_DATA].rlim_cur;
if (rlim >= RLIM_INFINITY)
rlim = ~0;
if (brk - current->mm->end_code > rlim)
return -ENOMEM;
/*
* Check against existing mmap mappings.
*/
if (find_vma_intersection(current, oldbrk, newbrk+PAGE_SIZE))
return -ENOMEM;
/*
* stupid algorithm to decide if we have enough memory: while
* simple, it hopefully works in most obvious cases.. Easy to
* fool it, but this should catch most mistakes.
*/
freepages = buffermem >> PAGE_SHIFT;
freepages += page_cache_size;
freepages >>= 1;
freepages += nr_free_pages;
freepages += nr_swap_pages;
freepages -= MAP_NR(high_memory) >> 4;
freepages -= (newbrk-oldbrk) >> PAGE_SHIFT;
if (freepages < 0)
return -ENOMEM;
/*
* Ok, we have probably got enough memory - let it rip.
*/
current->mm->brk = brk;
do_mmap(NULL, oldbrk, newbrk-oldbrk,
PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_FIXED|MAP_PRIVATE, 0);
return 0;
}
