/* * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will * occur here if the calling user does not have access to the submitted address. */ static int via_lock_all_dma_pages(drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer) { unsigned long first_pfn = VIA_PFN(xfer->mem_addr); vm_page_t m; int i; vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride -1)) - first_pfn + 1; if (NULL == (vsg->pages = malloc(sizeof(vm_page_t) * vsg->num_pages, DRM_MEM_DRIVER, M_NOWAIT))) return -ENOMEM; vsg->state = dr_via_pages_alloc; if (vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, (vm_offset_t)xfer->mem_addr, vsg->num_pages * PAGE_SIZE, VM_PROT_READ | VM_PROT_WRITE, vsg->pages, vsg->num_pages) < 0) return -EACCES; for (i = 0; i < vsg->num_pages; i++) { m = vsg->pages[i]; vm_page_lock(m); vm_page_wire(m); vm_page_unhold(m); vm_page_unlock(m); } vsg->state = dr_via_pages_locked; DRM_DEBUG("DMA pages locked\n"); return 0; }
/* * Copy a binary buffer from kernel space to user space. * * Returns 0 on success, EFAULT on failure. */ int copyout(const void *kaddr, void *udaddr, size_t len) { struct vmspace *vm = curproc->p_vmspace; struct lwbuf *lwb; struct lwbuf lwb_cache; vm_page_t m; int error; size_t n; error = 0; while (len) { m = vm_fault_page(&vm->vm_map, trunc_page((vm_offset_t)udaddr), VM_PROT_READ|VM_PROT_WRITE, VM_FAULT_NORMAL, &error); if (error) break; n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK); if (n > len) n = len; lwb = lwbuf_alloc(m, &lwb_cache); bcopy(kaddr, (char *)lwbuf_kva(lwb) + ((vm_offset_t)udaddr & PAGE_MASK), n); len -= n; udaddr = (char *)udaddr + n; kaddr = (const char *)kaddr + n; vm_page_dirty(m); lwbuf_free(lwb); vm_page_unhold(m); } return (error); }
/* * Return the requested word from the user-space address. * Returns 0 if it isn't mapped. (For what we're using it * for, if the actual value is 0, that's equivalent to that.) */ static caddr_t GET_WORD(pmap_t map, caddr_t virtual_addr) { caddr_t retval = 0; vm_page_t page; int err; caddr_t old_fault; page = pmap_extract_and_hold(map, (vm_offset_t)virtual_addr, VM_PROT_READ); if (page == 0) { return 0; } // I do this because copyin/copyout aren't re-entrant. old_fault = curpcb->pcb_onfault; err = copyin(virtual_addr, &retval, sizeof(retval)); curpcb->pcb_onfault = old_fault; if (err != 0) { #if SAMPLE_DEBUG printf("%s(%d): copyin(%p, %p, %zd) failed: %d\n", __FUNCTION__, __LINE__, (void*)virtual_addr, &retval, sizeof(retval), err); #endif retval = 0; } vm_page_lock(page); vm_page_unhold(page); vm_page_unlock(page); return retval; }
void vm_gpa_release(void *cookie) { vm_page_t m = cookie; vm_page_lock(m); vm_page_unhold(m); vm_page_unlock(m); }
void vm_fault_unhold_pages(vm_page_t *mp, int count) { KASSERT(count >= 0, ("negative count %d", count)); vm_page_lock_queues(); while (count--) { vm_page_unhold(*mp); mp++; } vm_page_unlock_queues(); }
void exec_unmap_first_page(struct image_params *imgp) { vm_page_t m; if (imgp->firstpage != NULL) { m = sf_buf_page(imgp->firstpage); sf_buf_free(imgp->firstpage); imgp->firstpage = NULL; vm_page_lock(m); vm_page_unhold(m); vm_page_unlock(m); } }
static void wire_ddp_buffer(struct ddp_buffer *db) { int i; vm_page_t p; for (i = 0; i < db->npages; i++) { p = db->pages[i]; vm_page_lock(p); vm_page_wire(p); vm_page_unhold(p); vm_page_unlock(p); } }
/* * Cleanup an XIO so it can be destroyed. The pages associated with the * XIO are released. */ void xio_release(xio_t xio) { int i; vm_page_t m; for (i = 0; i < xio->xio_npages; ++i) { m = xio->xio_pages[i]; if (xio->xio_flags & XIOF_WRITE) vm_page_dirty(m); vm_page_unhold(m); } xio->xio_offset = 0; xio->xio_npages = 0; xio->xio_bytes = 0; xio->xio_error = ENOBUFS; }
/* * Given a user pointer to a page of user memory, return an sf_buf for the * page. Because we may be requesting quite a few sf_bufs, prefer failure to * deadlock and use SFB_NOWAIT. */ static struct sf_buf * zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr) { struct sf_buf *sf; vm_page_t pp; if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ | VM_PROT_WRITE, &pp, 1) < 0) return (NULL); vm_page_lock(pp); vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock(pp); sf = sf_buf_alloc(pp, SFB_NOWAIT); if (sf == NULL) { zbuf_page_free(pp); return (NULL); } return (sf); }
/* * Cleanup an XIO so it can be destroyed. The pages associated with the * XIO are released. */ void xio_release(xio_t xio) { int i; vm_page_t m; lwkt_gettoken(&vm_token); crit_enter(); for (i = 0; i < xio->xio_npages; ++i) { m = xio->xio_pages[i]; if (xio->xio_flags & XIOF_WRITE) vm_page_dirty(m); vm_page_unhold(m); } crit_exit(); lwkt_reltoken(&vm_token); xio->xio_offset = 0; xio->xio_npages = 0; xio->xio_bytes = 0; xio->xio_error = ENOBUFS; }
/* * Hold each of the physical pages that are mapped by the specified range of * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid * and allow the specified types of access, "prot". If all of the implied * pages are successfully held, then the number of held pages is returned * together with pointers to those pages in the array "ma". However, if any * of the pages cannot be held, -1 is returned. */ int vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, vm_prot_t prot, vm_page_t *ma, int max_count) { vm_offset_t end, va; vm_page_t *mp; int count; boolean_t pmap_failed; if (len == 0) return (0); end = round_page(addr + len); addr = trunc_page(addr); /* * Check for illegal addresses. */ if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map)) return (-1); count = howmany(end - addr, PAGE_SIZE); if (count > max_count) panic("vm_fault_quick_hold_pages: count > max_count"); /* * Most likely, the physical pages are resident in the pmap, so it is * faster to try pmap_extract_and_hold() first. */ pmap_failed = FALSE; for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) { *mp = pmap_extract_and_hold(map->pmap, va, prot); if (*mp == NULL) pmap_failed = TRUE; else if ((prot & VM_PROT_WRITE) != 0 && (*mp)->dirty != VM_PAGE_BITS_ALL) { /* * Explicitly dirty the physical page. Otherwise, the * caller's changes may go unnoticed because they are * performed through an unmanaged mapping or by a DMA * operation. * * The object lock is not held here. * See vm_page_clear_dirty_mask(). */ vm_page_dirty(*mp); } } if (pmap_failed) { /* * One or more pages could not be held by the pmap. Either no * page was mapped at the specified virtual address or that * mapping had insufficient permissions. Attempt to fault in * and hold these pages. */ for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) if (*mp == NULL && vm_fault_hold(map, va, prot, VM_FAULT_NORMAL, mp) != KERN_SUCCESS) goto error; } return (count); error: for (mp = ma; mp < ma + count; mp++) if (*mp != NULL) { vm_page_lock(*mp); vm_page_unhold(*mp); vm_page_unlock(*mp); } return (-1); }
/* * p->p_token is held on entry. */ static int procfs_rwmem(struct proc *curp, struct proc *p, struct uio *uio) { int error; int writing; struct vmspace *vm; vm_map_t map; vm_offset_t pageno = 0; /* page number */ vm_prot_t reqprot; vm_offset_t kva; /* * if the vmspace is in the midst of being allocated or deallocated, * or the process is exiting, don't try to grab anything. The * page table usage in that process may be messed up. */ vm = p->p_vmspace; if (p->p_stat == SIDL || p->p_stat == SZOMB) return EFAULT; if ((p->p_flags & (P_WEXIT | P_INEXEC)) || sysref_isinactive(&vm->vm_sysref)) return EFAULT; /* * The map we want... */ vmspace_hold(vm); map = &vm->vm_map; writing = (uio->uio_rw == UIO_WRITE); reqprot = VM_PROT_READ; if (writing) reqprot |= VM_PROT_WRITE | VM_PROT_OVERRIDE_WRITE; kva = kmem_alloc_pageable(&kernel_map, PAGE_SIZE); /* * Only map in one page at a time. We don't have to, but it * makes things easier. This way is trivial - right? */ do { vm_offset_t uva; vm_offset_t page_offset; /* offset into page */ size_t len; vm_page_t m; uva = (vm_offset_t) uio->uio_offset; /* * Get the page number of this segment. */ pageno = trunc_page(uva); page_offset = uva - pageno; /* * How many bytes to copy */ len = szmin(PAGE_SIZE - page_offset, uio->uio_resid); /* * Fault the page on behalf of the process */ m = vm_fault_page(map, pageno, reqprot, VM_FAULT_NORMAL, &error); if (error) { KKASSERT(m == NULL); error = EFAULT; break; } /* * Cleanup tmap then create a temporary KVA mapping and * do the I/O. We can switch between cpus so don't bother * synchronizing across all cores. */ pmap_kenter_quick(kva, VM_PAGE_TO_PHYS(m)); error = uiomove((caddr_t)(kva + page_offset), len, uio); pmap_kremove_quick(kva); /* * release the page and we are done */ vm_page_unhold(m); } while (error == 0 && uio->uio_resid > 0); vmspace_drop(vm); kmem_free(&kernel_map, kva, PAGE_SIZE); return (error); }
int socow_setup(struct mbuf *m0, struct uio *uio) { struct sf_buf *sf; vm_page_t pp; struct iovec *iov; struct vmspace *vmspace; struct vm_map *map; vm_offset_t offset, uva; socow_stats.attempted++; vmspace = curproc->p_vmspace; map = &vmspace->vm_map; uva = (vm_offset_t) uio->uio_iov->iov_base; offset = uva & PAGE_MASK; /* * Verify that access to the given address is allowed from user-space. */ if (vm_fault_quick((caddr_t)uva, VM_PROT_READ) < 0) return (0); /* * verify page is mapped & not already wired for i/o */ pp = pmap_extract_and_hold(map->pmap, uva, VM_PROT_READ); if (pp == NULL) { socow_stats.fail_not_mapped++; return(0); } /* * set up COW */ vm_page_lock(pp); if (vm_page_cowsetup(pp) != 0) { vm_page_unhold(pp); vm_page_unlock(pp); return (0); } /* * wire the page for I/O */ vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock(pp); /* * Allocate an sf buf */ sf = sf_buf_alloc(pp, SFB_CATCH); if (sf == NULL) { vm_page_lock(pp); vm_page_cowclear(pp); vm_page_unwire(pp, 0); /* * Check for the object going away on us. This can * happen since we don't hold a reference to it. * If so, we're responsible for freeing the page. */ if (pp->wire_count == 0 && pp->object == NULL) vm_page_free(pp); vm_page_unlock(pp); socow_stats.fail_sf_buf++; return(0); } /* * attach to mbuf */ MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone, (void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF); m0->m_len = PAGE_SIZE - offset; m0->m_data = (caddr_t)sf_buf_kva(sf) + offset; socow_stats.success++; iov = uio->uio_iov; iov->iov_base = (char *)iov->iov_base + m0->m_len; iov->iov_len -= m0->m_len; uio->uio_resid -= m0->m_len; uio->uio_offset += m0->m_len; if (iov->iov_len == 0) { uio->uio_iov++; uio->uio_iovcnt--; } return(m0->m_len); }
/* * This routine takes a user's map, array of pages, number of pages, and flags * and then does the following: * - validate that the user has access to those pages (flags indicates read * or write) - if not fail * - validate that count is enough to hold range number of pages - if not fail * - fault in any non-resident pages * - if the user is doing a read force a write fault for any COWed pages * - if the user is doing a read mark all pages as dirty * - hold all pages */ int vm_fault_hold_user_pages(vm_map_t map, vm_offset_t addr, vm_page_t *mp, int count, vm_prot_t prot) { vm_offset_t end, va; int faults, rv; pmap_t pmap; vm_page_t m, *pages; pmap = vm_map_pmap(map); pages = mp; addr &= ~PAGE_MASK; /* * Check that virtual address range is legal * This check is somewhat bogus as on some architectures kernel * and user do not share VA - however, it appears that all FreeBSD * architectures define it */ end = addr + (count * PAGE_SIZE); if (end > VM_MAXUSER_ADDRESS) { log(LOG_WARNING, "bad address passed to vm_fault_hold_user_pages"); return (EFAULT); } /* * First optimistically assume that all pages are resident * (and R/W if for write) if so just mark pages as held (and * dirty if for write) and return */ vm_page_lock_queues(); for (pages = mp, faults = 0, va = addr; va < end; va += PAGE_SIZE, pages++) { /* * page queue mutex is recursable so this is OK * it would be really nice if we had an unlocked * version of this so we were only acquiring the * pmap lock 1 time as opposed to potentially * many dozens of times */ *pages = m = pmap_extract_and_hold(pmap, va, prot); if (m == NULL) { faults++; continue; } /* * Preemptively mark dirty - the pages * will never have the modified bit set if * they are only changed via DMA */ if (prot & VM_PROT_WRITE) vm_page_dirty(m); } vm_page_unlock_queues(); if (faults == 0) return (0); /* * Pages either have insufficient permissions or are not present * trigger a fault where neccessary * */ rv = 0; for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) { /* * Account for a very narrow race where the page may be * taken away from us before it is held */ while (*pages == NULL) { rv = vm_fault(map, va, prot, (prot & VM_PROT_WRITE) ? VM_FAULT_DIRTY : VM_FAULT_NORMAL); if (rv) goto error; *pages = pmap_extract_and_hold(pmap, va, prot); } } return (0); error: log(LOG_WARNING, "vm_fault bad return rv=%d va=0x%zx\n", rv, va); vm_page_lock_queues(); for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) if (*pages) { vm_page_unhold(*pages); *pages = NULL; } vm_page_unlock_queues(); return (EFAULT); }
static int uiomove_object_page(vm_object_t obj, size_t len, struct uio *uio) { vm_page_t m; vm_pindex_t idx; size_t tlen; int error, offset, rv; idx = OFF_TO_IDX(uio->uio_offset); offset = uio->uio_offset & PAGE_MASK; tlen = MIN(PAGE_SIZE - offset, len); VM_OBJECT_WLOCK(obj); /* * Parallel reads of the page content from disk are prevented * by exclusive busy. * * Although the tmpfs vnode lock is held here, it is * nonetheless safe to sleep waiting for a free page. The * pageout daemon does not need to acquire the tmpfs vnode * lock to page out tobj's pages because tobj is a OBJT_SWAP * type object. */ m = vm_page_grab(obj, idx, VM_ALLOC_NORMAL); if (m->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(obj, idx, NULL, NULL)) { rv = vm_pager_get_pages(obj, &m, 1, 0); m = vm_page_lookup(obj, idx); if (m == NULL) { printf( "uiomove_object: vm_obj %p idx %jd null lookup rv %d\n", obj, idx, rv); VM_OBJECT_WUNLOCK(obj); return (EIO); } if (rv != VM_PAGER_OK) { printf( "uiomove_object: vm_obj %p idx %jd valid %x pager error %d\n", obj, idx, m->valid, rv); vm_page_lock(m); vm_page_free(m); vm_page_unlock(m); VM_OBJECT_WUNLOCK(obj); return (EIO); } } else vm_page_zero_invalid(m, TRUE); } vm_page_xunbusy(m); vm_page_lock(m); vm_page_hold(m); vm_page_unlock(m); VM_OBJECT_WUNLOCK(obj); error = uiomove_fromphys(&m, offset, tlen, uio); if (uio->uio_rw == UIO_WRITE && error == 0) { VM_OBJECT_WLOCK(obj); vm_page_dirty(m); VM_OBJECT_WUNLOCK(obj); } vm_page_lock(m); vm_page_unhold(m); if (m->queue == PQ_NONE) { vm_page_deactivate(m); } else { /* Requeue to maintain LRU ordering. */ vm_page_requeue(m); } vm_page_unlock(m); return (error); }