asmlinkage unsigned long ia64_brk (unsigned long brk, long arg1, long arg2, long arg3, long arg4, long arg5, long arg6, long arg7, long stack) { extern int vm_enough_memory (long pages); struct pt_regs *regs = (struct pt_regs *) &stack; unsigned long rlim, retval, newbrk, oldbrk; struct mm_struct *mm = current->mm; /* * Most of this replicates the code in sys_brk() except for an additional safety * check and the clearing of r8. However, we can't call sys_brk() because we need * to acquire the mmap_sem before we can do the test... */ down_write(&mm->mmap_sem); if (brk < mm->end_code) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) goto set_brk; /* Always allow shrinking brk. */ if (brk <= mm->brk) { if (!do_munmap(mm, newbrk, oldbrk-newbrk,1)) goto set_brk; goto out; } /* Check against unimplemented/unmapped addresses: */ if ((newbrk - oldbrk) > RGN_MAP_LIMIT || rgn_offset(newbrk) > RGN_MAP_LIMIT) goto out; /* Check against rlimit.. */ rlim = current->rlim[RLIMIT_DATA].rlim_cur; if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim) goto out; /* Check against existing mmap mappings. */ if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; /* Ok, looks good - let it rip. */ if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) goto out; set_brk: mm->brk = brk; out: retval = mm->brk; up_write(&mm->mmap_sem); regs->r8 = 0; /* ensure large retval isn't mistaken as error code */ return retval; }
asmlinkage int sunos_brk(u32 baddr) { int freepages, retval = -ENOMEM; unsigned long rlim; unsigned long newbrk, oldbrk, brk = (unsigned long) baddr; down_write(¤t->mm->mmap_sem); if (brk < current->mm->end_code) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(current->mm->brk); retval = 0; if (oldbrk == newbrk) { current->mm->brk = brk; goto out; } /* Always allow shrinking brk. */ if (brk <= current->mm->brk) { current->mm->brk = brk; do_munmap(current->mm, newbrk, oldbrk-newbrk); goto out; } /* Check against rlimit and stack.. */ retval = -ENOMEM; rlim = current->rlim[RLIMIT_DATA].rlim_cur; if (rlim >= RLIM_INFINITY) rlim = ~0; if (brk - current->mm->end_code > rlim) goto out; /* Check against existing mmap mappings. */ if (find_vma_intersection(current->mm, oldbrk, newbrk+PAGE_SIZE)) goto out; /* stupid algorithm to decide if we have enough memory: while * simple, it hopefully works in most obvious cases.. Easy to * fool it, but this should catch most mistakes. */ freepages = get_page_cache_size(); freepages >>= 1; freepages += nr_free_pages(); freepages += nr_swap_pages; freepages -= num_physpages >> 4; freepages -= (newbrk-oldbrk) >> PAGE_SHIFT; if (freepages < 0) goto out; /* Ok, we have probably got enough memory - let it rip. */ current->mm->brk = brk; do_brk(oldbrk, newbrk-oldbrk); retval = 0; out: up_write(¤t->mm->mmap_sem); return retval; }
asmlinkage unsigned long ia64_brk (unsigned long brk) { unsigned long rlim, retval, newbrk, oldbrk; struct mm_struct *mm = current->mm; /* * Most of this replicates the code in sys_brk() except for an additional safety * check and the clearing of r8. However, we can't call sys_brk() because we need * to acquire the mmap_sem before we can do the test... */ down_write(&mm->mmap_sem); if (brk < mm->end_code) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) goto set_brk; /* Always allow shrinking brk. */ if (brk <= mm->brk) { if (!do_munmap(mm, newbrk, oldbrk-newbrk)) goto set_brk; goto out; } /* Check against unimplemented/unmapped addresses: */ if ((newbrk - oldbrk) > RGN_MAP_LIMIT || REGION_OFFSET(newbrk) > RGN_MAP_LIMIT) goto out; /* Check against rlimit.. */ rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim) goto out; /* Check against existing mmap mappings. */ if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; /* Ok, looks good - let it rip. */ if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) goto out; set_brk: mm->brk = brk; out: retval = mm->brk; up_write(&mm->mmap_sem); force_successful_syscall_return(); return retval; }
SYSCALL_DEFINE1(brk, unsigned long, brk) { unsigned long rlim, retval; unsigned long newbrk, oldbrk; struct mm_struct *mm = current->mm; unsigned long min_brk; down_write(&mm->mmap_sem); #ifdef CONFIG_COMPAT_BRK if (current->brk_randomized) min_brk = mm->start_brk; else min_brk = mm->end_data; #else min_brk = mm->start_brk; #endif if (brk < min_brk) goto out; rlim = rlimit(RLIMIT_DATA); if (rlim < RLIM_INFINITY && (brk - mm->start_brk) + (mm->end_data - mm->start_data) > rlim) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) goto set_brk; if (brk <= mm->brk) { if (!do_munmap(mm, newbrk, oldbrk-newbrk)) goto set_brk; goto out; } if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) goto out; set_brk: mm->brk = brk; out: retval = mm->brk; up_write(&mm->mmap_sem); return retval; }
// do_brk - adjust(increase/decrease) the size of process heap, align with page size // NOTE: will change the process vma int do_brk(uintptr_t *brk_store) { struct mm_struct *mm = current->mm; if (mm == NULL) { panic("kernel thread call sys_brk!!.\n"); } if (brk_store == NULL) { return -E_INVAL; } uintptr_t brk; lock_mm(mm); if (!copy_from_user(mm, &brk, brk_store, sizeof(uintptr_t), 1)) { unlock_mm(mm); return -E_INVAL; } if (brk < mm->brk_start) { goto out_unlock; } uintptr_t newbrk = ROUNDUP(brk, PGSIZE), oldbrk = mm->brk; assert(oldbrk % PGSIZE == 0); if (newbrk == oldbrk) { goto out_unlock; } if (newbrk < oldbrk) { if (mm_unmap(mm, newbrk, oldbrk - newbrk) != 0) { goto out_unlock; } } else { if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE) != NULL) { goto out_unlock; } if (mm_brk(mm, oldbrk, newbrk - oldbrk) != 0) { goto out_unlock; } } mm->brk = newbrk; out_unlock: copy_to_user (mm, brk_store, &mm->brk, sizeof (uintptr_t)); unlock_mm(mm); return 0; }
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) { int ret; bool write_fault, writable, hugetlb = false, force_pte = false; unsigned long mmu_seq; gfn_t gfn = fault_ipa >> PAGE_SHIFT; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; struct vm_area_struct *vma; pfn_t pfn; pgprot_t mem_type = PAGE_S2; bool fault_ipa_uncached; write_fault = kvm_is_write_fault(vcpu); if (fault_status == FSC_PERM && !write_fault) { kvm_err("Unexpected L2 read permission error\n"); return -EFAULT; } /* Let's check if we will get back a huge page backed by hugetlbfs */ down_read(¤t->mm->mmap_sem); vma = find_vma_intersection(current->mm, hva, hva + 1); if (unlikely(!vma)) { kvm_err("Failed to find VMA for hva 0x%lx\n", hva); up_read(¤t->mm->mmap_sem); return -EFAULT; } if (is_vm_hugetlb_page(vma)) { hugetlb = true; gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT; } else { /* * Pages belonging to memslots that don't have the same * alignment for userspace and IPA cannot be mapped using * block descriptors even if the pages belong to a THP for * the process, because the stage-2 block descriptor will * cover more than a single THP and we loose atomicity for * unmapping, updates, and splits of the THP or other pages * in the stage-2 block range. */ if ((memslot->userspace_addr & ~PMD_MASK) !=
/* poring from linux */ int do_linux_brk(uintptr_t brk) { uint32_t newbrk, oldbrk, retval; struct mm_struct *mm = current->mm; uint32_t min_brk; if (!mm) { panic("kernel thread call sys_brk!!.\n"); } lock_mm(mm); min_brk = mm->brk_start; if (brk < min_brk) goto out_unlock; newbrk = ROUNDUP(brk, PGSIZE); oldbrk = ROUNDUP(mm->brk, PGSIZE); if (oldbrk == newbrk) goto set_brk; if (brk <= mm->brk) { if (!mm_unmap(mm, newbrk, oldbrk - newbrk)) goto set_brk; goto out_unlock; } if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE)) goto out_unlock; /* set the brk */ if (mm_brk(mm, oldbrk, newbrk - oldbrk)) goto out_unlock; set_brk: mm->brk = brk; out_unlock: retval = mm->brk; unlock_mm(mm); return retval; }
/* * sys_brk() for the most part doesn't need the global kernel * lock, except when an application is doing something nasty * like trying to un-brk an area that has already been mapped * to a regular file. in this case, the unmapping will need * to invoke file system routines that need the global lock. */ asmlinkage unsigned long sys_brk(unsigned long brk) { unsigned long rlim, retval; unsigned long newbrk, oldbrk; struct mm_struct *mm = current->mm; down_write(&mm->mmap_sem); if (brk < mm->end_code) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) goto set_brk; /* Always allow shrinking brk. */ if (brk <= mm->brk) { if (!do_munmap(mm, newbrk, oldbrk-newbrk)) goto set_brk; goto out; } /* Check against rlimit.. */ rlim = current->rlim[RLIMIT_DATA].rlim_cur; if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim) goto out; /* Check against existing mmap mappings. */ if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; /* Ok, looks good - let it rip. */ if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) goto out; set_brk: mm->brk = brk; out: retval = mm->brk; up_write(&mm->mmap_sem); return retval; }
/* from x86 bionic porting */ int do_linux_brk(uintptr_t brk) { struct mm_struct *mm = current->mm; if (mm == NULL) { panic("kernel thread call sys_brk!!.\n"); } if (brk == 0) { return mm->brk_start; } lock_mm(mm); if (brk < mm->brk_start) { goto out_unlock; } uintptr_t newbrk = ROUNDUP(brk, PGSIZE), oldbrk = mm->brk; assert(oldbrk % PGSIZE == 0); if (newbrk == oldbrk) { goto out_unlock; } if (newbrk < oldbrk) { if (mm_unmap(mm, newbrk, oldbrk - newbrk) != 0) { goto out_unlock; } } else { if (find_vma_intersection(mm, oldbrk, newbrk + PGSIZE) != NULL) { goto out_unlock; } if (mm_brk(mm, oldbrk, newbrk - oldbrk) != 0) { goto out_unlock; } } mm->brk = newbrk; out_unlock: unlock_mm(mm); return newbrk; }
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) { int ret; bool write_fault, writable, hugetlb = false, force_pte = false; unsigned long mmu_seq; gfn_t gfn = fault_ipa >> PAGE_SHIFT; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; struct vm_area_struct *vma; pfn_t pfn; pgprot_t mem_type = PAGE_S2; bool fault_ipa_uncached; bool logging_active = memslot_is_logging(memslot); unsigned long flags = 0; write_fault = kvm_is_write_fault(vcpu); if (fault_status == FSC_PERM && !write_fault) { kvm_err("Unexpected L2 read permission error\n"); return -EFAULT; } /* Let's check if we will get back a huge page backed by hugetlbfs */ down_read(¤t->mm->mmap_sem); vma = find_vma_intersection(current->mm, hva, hva + 1); if (unlikely(!vma)) { kvm_err("Failed to find VMA for hva 0x%lx\n", hva); up_read(¤t->mm->mmap_sem); return -EFAULT; } if (is_vm_hugetlb_page(vma) && !logging_active) { hugetlb = true; gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT; } else {
unsigned long sys_brk(unsigned long brk) { unsigned long rlim; unsigned long newbrk, oldbrk; if (brk < current->mm->end_code) return current->mm->brk; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(current->mm->brk); if (oldbrk == newbrk) return current->mm->brk = brk; /* * Always allow shrinking brk */ if (brk <= current->mm->brk) { current->mm->brk = brk; do_munmap(newbrk, oldbrk - newbrk); return brk; } /* * Check against rlimit */ rlim = current->rlim[RLIMIT_DATA].rlim_cur; if (brk - current->mm->end_code > rlim) return current->mm->brk; /* * Check against existing mmap mappings */ if (find_vma_intersection(current->mm, oldbrk, newbrk + PAGE_SIZE)) return current->mm->brk; current->mm->brk = brk; do_mmap(oldbrk, newbrk - oldbrk, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); return brk; }
/** * get_vaddr_frames() - map virtual addresses to pfns * @start: starting user address * @nr_frames: number of pages / pfns from start to map * @write: whether pages will be written to by the caller * @force: whether to force write access even if user mapping is * readonly. See description of the same argument of get_user_pages(). * @vec: structure which receives pages / pfns of the addresses mapped. * It should have space for at least nr_frames entries. * * This function maps virtual addresses from @start and fills @vec structure * with page frame numbers or page pointers to corresponding pages (choice * depends on the type of the vma underlying the virtual address). If @start * belongs to a normal vma, the function grabs reference to each of the pages * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't * touch page structures and the caller must make sure pfns aren't reused for * anything else while he is using them. * * The function returns number of pages mapped which may be less than * @nr_frames. In particular we stop mapping if there are more vmas of * different type underlying the specified range of virtual addresses. * When the function isn't able to map a single page, it returns error. * * This function takes care of grabbing mmap_sem as necessary. */ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, bool write, bool force, struct frame_vector *vec) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int ret = 0; int err; int locked; if (nr_frames == 0) return 0; if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) nr_frames = vec->nr_allocated; down_read(&mm->mmap_sem); locked = 1; vma = find_vma_intersection(mm, start, start + 1); if (!vma) { ret = -EFAULT; goto out; } if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { vec->got_ref = true; vec->is_pfns = false; ret = get_user_pages_locked(current, mm, start, nr_frames, write, force, (struct page **)(vec->ptrs), &locked); goto out; } vec->got_ref = false; vec->is_pfns = true; do { unsigned long *nums = frame_vector_pfns(vec); while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { err = follow_pfn(vma, start, &nums[ret]); if (err) { if (ret == 0) ret = err; goto out; } start += PAGE_SIZE; ret++; } /* * We stop if we have enough pages or if VMA doesn't completely * cover the tail page. */ if (ret >= nr_frames || start < vma->vm_end) break; vma = find_vma_intersection(mm, start, start + 1); } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); out: if (locked) up_read(&mm->mmap_sem); if (!ret) ret = -EFAULT; if (ret > 0) vec->nr_frames = ret; return ret; }
/** * get_vaddr_frames() - map virtual addresses to pfns * @start: starting user address * @nr_frames: number of pages / pfns from start to map * @gup_flags: flags modifying lookup behaviour * @vec: structure which receives pages / pfns of the addresses mapped. * It should have space for at least nr_frames entries. * * This function maps virtual addresses from @start and fills @vec structure * with page frame numbers or page pointers to corresponding pages (choice * depends on the type of the vma underlying the virtual address). If @start * belongs to a normal vma, the function grabs reference to each of the pages * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't * touch page structures and the caller must make sure pfns aren't reused for * anything else while he is using them. * * The function returns number of pages mapped which may be less than * @nr_frames. In particular we stop mapping if there are more vmas of * different type underlying the specified range of virtual addresses. * When the function isn't able to map a single page, it returns error. * * This function takes care of grabbing mmap_sem as necessary. */ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, unsigned int gup_flags, struct frame_vector *vec) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int ret = 0; int err; int locked; if (nr_frames == 0) return 0; if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) nr_frames = vec->nr_allocated; down_read(&mm->mmap_sem); locked = 1; vma = find_vma_intersection(mm, start, start + 1); if (!vma) { ret = -EFAULT; goto out; } /* * While get_vaddr_frames() could be used for transient (kernel * controlled lifetime) pinning of memory pages all current * users establish long term (userspace controlled lifetime) * page pinning. Treat get_vaddr_frames() like * get_user_pages_longterm() and disallow it for filesystem-dax * mappings. */ if (vma_is_fsdax(vma)) return -EOPNOTSUPP; if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { vec->got_ref = true; vec->is_pfns = false; ret = get_user_pages_locked(start, nr_frames, gup_flags, (struct page **)(vec->ptrs), &locked); goto out; } vec->got_ref = false; vec->is_pfns = true; do { unsigned long *nums = frame_vector_pfns(vec); while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { err = follow_pfn(vma, start, &nums[ret]); if (err) { if (ret == 0) ret = err; goto out; } start += PAGE_SIZE; ret++; } /* * We stop if we have enough pages or if VMA doesn't completely * cover the tail page. */ if (ret >= nr_frames || start < vma->vm_end) break; vma = find_vma_intersection(mm, start, start + 1); } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); out: if (locked) up_read(&mm->mmap_sem); if (!ret) ret = -EFAULT; if (ret > 0) vec->nr_frames = ret; return ret; }
/* * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists. */ asmlinkage int sys_shmat (int shmid, char *shmaddr, int shmflg, ulong *raddr) { struct shmid_kernel *shp; struct vm_area_struct *shmd; int err = -EINVAL; unsigned int id; unsigned long addr; unsigned long len; down(¤t->mm->mmap_sem); lock_kernel(); if (shmid < 0) { /* printk("shmat() -> EINVAL because shmid = %d < 0\n",shmid); */ goto out; } shp = shm_segs[id = (unsigned int) shmid % SHMMNI]; if (shp == IPC_UNUSED || shp == IPC_NOID) { /* printk("shmat() -> EINVAL because shmid = %d is invalid\n",shmid); */ goto out; } if (!(addr = (ulong) shmaddr)) { if (shmflg & SHM_REMAP) goto out; err = -ENOMEM; addr = 0; again: if (!(addr = get_unmapped_area(addr, shp->u.shm_segsz))) goto out; if(addr & (SHMLBA - 1)) { addr = (addr + (SHMLBA - 1)) & ~(SHMLBA - 1); goto again; } } else if (addr & (SHMLBA-1)) { if (shmflg & SHM_RND) addr &= ~(SHMLBA-1); /* round down */ else goto out; } /* * Check if addr exceeds TASK_SIZE (from do_mmap) */ len = PAGE_SIZE*shp->shm_npages; err = -EINVAL; if (addr >= TASK_SIZE || len > TASK_SIZE || addr > TASK_SIZE - len) goto out; /* * If shm segment goes below stack, make sure there is some * space left for the stack to grow (presently 4 pages). */ if (addr < current->mm->start_stack && addr > current->mm->start_stack - PAGE_SIZE*(shp->shm_npages + 4)) { /* printk("shmat() -> EINVAL because segment intersects stack\n"); */ goto out; } if (!(shmflg & SHM_REMAP)) if ((shmd = find_vma_intersection(current->mm, addr, addr + shp->u.shm_segsz))) { /* printk("shmat() -> EINVAL because the interval [0x%lx,0x%lx) intersects an already mapped interval [0x%lx,0x%lx).\n", addr, addr + shp->shm_segsz, shmd->vm_start, shmd->vm_end); */ goto out; } err = -EACCES; if (ipcperms(&shp->u.shm_perm, shmflg & SHM_RDONLY ? S_IRUGO : S_IRUGO|S_IWUGO)) goto out; err = -EIDRM; if (shp->u.shm_perm.seq != (unsigned int) shmid / SHMMNI) goto out; err = -ENOMEM; shmd = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); if (!shmd) goto out; if ((shp != shm_segs[id]) || (shp->u.shm_perm.seq != (unsigned int) shmid / SHMMNI)) { kmem_cache_free(vm_area_cachep, shmd); err = -EIDRM; goto out; } shmd->vm_pte = SWP_ENTRY(SHM_SWP_TYPE, id); shmd->vm_start = addr; shmd->vm_end = addr + shp->shm_npages * PAGE_SIZE; shmd->vm_mm = current->mm; shmd->vm_page_prot = (shmflg & SHM_RDONLY) ? PAGE_READONLY : PAGE_SHARED; shmd->vm_flags = VM_SHM | VM_MAYSHARE | VM_SHARED | VM_MAYREAD | VM_MAYEXEC | VM_READ | VM_EXEC | ((shmflg & SHM_RDONLY) ? 0 : VM_MAYWRITE | VM_WRITE); shmd->vm_file = NULL; shmd->vm_offset = 0; shmd->vm_ops = &shm_vm_ops; shp->u.shm_nattch++; /* prevent destruction */ if (shp->u.shm_nattch > 0xffff - NR_TASKS || (err = shm_map (shmd))) { if (--shp->u.shm_nattch <= 0 && shp->u.shm_perm.mode & SHM_DEST) killseg(id); kmem_cache_free(vm_area_cachep, shmd); goto out; } insert_attach(shp,shmd); /* insert shmd into shp->attaches */ shp->u.shm_lpid = current->pid; shp->u.shm_atime = CURRENT_TIME; *raddr = addr; err = 0; out: unlock_kernel(); up(¤t->mm->mmap_sem); return err; }
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long fault_status) { int ret; bool write_fault, writable, hugetlb = false; unsigned long mmu_seq; gfn_t gfn = fault_ipa >> PAGE_SHIFT; unsigned long hva = gfn_to_hva(vcpu->kvm, gfn); struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; struct vm_area_struct *vma; pfn_t pfn; write_fault = kvm_is_write_fault(kvm_vcpu_get_hsr(vcpu)); if (fault_status == FSC_PERM && !write_fault) { kvm_err("Unexpected L2 read permission error\n"); return -EFAULT; } /* Let's check if we will get back a huge page backed by hugetlbfs */ down_read(¤t->mm->mmap_sem); vma = find_vma_intersection(current->mm, hva, hva + 1); if (is_vm_hugetlb_page(vma)) { hugetlb = true; gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT; } up_read(¤t->mm->mmap_sem); /* We need minimum second+third level pages */ ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS); if (ret) return ret; mmu_seq = vcpu->kvm->mmu_notifier_seq; /* * Ensure the read of mmu_notifier_seq happens before we call * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk * the page we just got a reference to gets unmapped before we have a * chance to grab the mmu_lock, which ensure that if the page gets * unmapped afterwards, the call to kvm_unmap_hva will take it away * from us again properly. This smp_rmb() interacts with the smp_wmb() * in kvm_mmu_notifier_invalidate_<page|range_end>. */ smp_rmb(); pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); if (is_error_pfn(pfn)) return -EFAULT; spin_lock(&kvm->mmu_lock); if (mmu_notifier_retry(kvm, mmu_seq)) goto out_unlock; if (hugetlb) { pmd_t new_pmd = pfn_pmd(pfn, PAGE_S2); new_pmd = pmd_mkhuge(new_pmd); if (writable) { kvm_set_s2pmd_writable(&new_pmd); kvm_set_pfn_dirty(pfn); } coherent_icache_guest_page(kvm, hva & PMD_MASK, PMD_SIZE); ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd); } else { pte_t new_pte = pfn_pte(pfn, PAGE_S2); if (writable) { kvm_set_s2pte_writable(&new_pte); kvm_set_pfn_dirty(pfn); } coherent_icache_guest_page(kvm, hva, PAGE_SIZE); ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, false); } out_unlock: spin_unlock(&kvm->mmu_lock); kvm_release_pfn_clean(pfn); return ret; }
SYSCALL_DEFINE1(brk, unsigned long, brk) { unsigned long rlim, retval; unsigned long newbrk, oldbrk; struct mm_struct *mm = current->mm; unsigned long min_brk; down_write(&mm->mmap_sem); #ifdef CONFIG_COMPAT_BRK /* * CONFIG_COMPAT_BRK can still be overridden by setting * randomize_va_space to 2, which will still cause mm->start_brk * to be arbitrarily shifted */ if (current->brk_randomized) min_brk = mm->start_brk; else min_brk = mm->end_data; #else min_brk = mm->start_brk; #endif if (brk < min_brk) goto out; /* * Check against rlimit here. If this check is done later after the test * of oldbrk with newbrk then it can escape the test and let the data * segment grow beyond its set limit the in case where the limit is * not page aligned -Ram Gupta */ rlim = rlimit(RLIMIT_DATA); if (rlim < RLIM_INFINITY && (brk - mm->start_brk) + (mm->end_data - mm->start_data) > rlim) goto out; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) goto set_brk; /* Always allow shrinking brk. */ if (brk <= mm->brk) { if (!do_munmap(mm, newbrk, oldbrk-newbrk)) goto set_brk; goto out; } /* Check against existing mmap mappings. */ if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; /* Ok, looks good - let it rip. */ if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) goto out; set_brk: mm->brk = brk; out: retval = mm->brk; up_write(&mm->mmap_sem); return retval; }
/* SunOS is completely broken... it returns 0 on success, otherwise * ENOMEM. For sys_sbrk() it wants the new brk value as a return * on success and ENOMEM as before on failure. */ asmlinkage int sunos_brk(unsigned long brk) { int freepages; unsigned long rlim; unsigned long newbrk, oldbrk; if (brk < current->mm->end_code) return -ENOMEM; newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(current->mm->brk); if (oldbrk == newbrk) { current->mm->brk = brk; return 0; } /* * Always allow shrinking brk */ if (brk <= current->mm->brk) { current->mm->brk = brk; do_munmap(newbrk, oldbrk-newbrk); return 0; } /* * Check against rlimit and stack.. */ rlim = current->rlim[RLIMIT_DATA].rlim_cur; if (rlim >= RLIM_INFINITY) rlim = ~0; if (brk - current->mm->end_code > rlim) return -ENOMEM; /* * Check against existing mmap mappings. */ if (find_vma_intersection(current, oldbrk, newbrk+PAGE_SIZE)) return -ENOMEM; /* * stupid algorithm to decide if we have enough memory: while * simple, it hopefully works in most obvious cases.. Easy to * fool it, but this should catch most mistakes. */ freepages = buffermem >> PAGE_SHIFT; freepages += page_cache_size; freepages >>= 1; freepages += nr_free_pages; freepages += nr_swap_pages; freepages -= MAP_NR(high_memory) >> 4; freepages -= (newbrk-oldbrk) >> PAGE_SHIFT; if (freepages < 0) return -ENOMEM; /* * Ok, we have probably got enough memory - let it rip. */ current->mm->brk = brk; do_mmap(NULL, oldbrk, newbrk-oldbrk, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_FIXED|MAP_PRIVATE, 0); return 0; }