/* * Identify the physical page mapped at the given kernel virtual * address. Insert this physical page into the given address space at * the given virtual address, replacing the physical page, if any, * that already exists there. */ static int vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr) { vm_map_t map = mapa; vm_page_t kern_pg, user_pg; vm_object_t uobject; vm_map_entry_t entry; vm_pindex_t upindex; vm_prot_t prot; boolean_t wired; KASSERT((uaddr & PAGE_MASK) == 0, ("vm_pgmoveco: uaddr is not page aligned")); /* * Herein the physical page is validated and dirtied. It is * unwired in sf_buf_mext(). */ kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr)); kern_pg->valid = VM_PAGE_BITS_ALL; KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1, ("vm_pgmoveco: kern_pg is not correctly wired")); if ((vm_map_lookup(&map, uaddr, VM_PROT_WRITE, &entry, &uobject, &upindex, &prot, &wired)) != KERN_SUCCESS) { return(EFAULT); } VM_OBJECT_LOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) goto retry; vm_page_lock_queues(); pmap_remove_all(user_pg); vm_page_free(user_pg); } else { /* * Even if a physical page does not exist in the * object chain's first object, a physical page from a * backing object may be mapped read only. */ if (uobject->backing_object != NULL) pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE); vm_page_lock_queues(); } vm_page_insert(kern_pg, uobject, upindex); vm_page_dirty(kern_pg); vm_page_unlock_queues(); VM_OBJECT_UNLOCK(uobject); vm_map_lookup_done(map, entry); return(KERN_SUCCESS); }
static int phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) { int i, s; s = splvm(); /* * Fill as many pages as vm_fault has allocated for us. */ for (i = 0; i < count; i++) { if ((m[i]->flags & PG_ZERO) == 0) pmap_zero_page(m[i]); vm_page_flag_set(m[i], PG_ZERO); /* Switch off pv_entries */ vm_page_lock_queues(); vm_page_unmanage(m[i]); vm_page_unlock_queues(); m[i]->valid = VM_PAGE_BITS_ALL; m[i]->dirty = 0; /* The requested page must remain busy, the others not. */ if (reqpage != i) { vm_page_flag_clear(m[i], PG_BUSY); m[i]->busy = 0; } } splx(s); return (VM_PAGER_OK); }
static int rtR0MemObjFreeBSDPhysAllocHelper(vm_object_t pObject, u_long cPages, vm_paddr_t VmPhysAddrHigh, u_long uAlignment, bool fContiguous, bool fWire, int rcNoMem) { if (fContiguous) { if (rtR0MemObjFreeBSDContigPhysAllocHelper(pObject, 0, cPages, VmPhysAddrHigh, uAlignment, fWire) != NULL) return VINF_SUCCESS; return rcNoMem; } for (vm_pindex_t iPage = 0; iPage < cPages; iPage++) { vm_page_t pPage = rtR0MemObjFreeBSDContigPhysAllocHelper(pObject, iPage, 1, VmPhysAddrHigh, uAlignment, fWire); if (!pPage) { /* Free all allocated pages */ VM_OBJECT_LOCK(pObject); while (iPage-- > 0) { pPage = vm_page_lookup(pObject, iPage); vm_page_lock_queues(); if (fWire) vm_page_unwire(pPage, 0); vm_page_free(pPage); vm_page_unlock_queues(); } VM_OBJECT_UNLOCK(pObject); return rcNoMem; } } return VINF_SUCCESS; }
/* * Fill as many pages as vm_fault has allocated for us. */ static int phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) { int i; VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); for (i = 0; i < count; i++) { if (m[i]->valid == 0) { if ((m[i]->flags & PG_ZERO) == 0) pmap_zero_page(m[i]); m[i]->valid = VM_PAGE_BITS_ALL; } KASSERT(m[i]->valid == VM_PAGE_BITS_ALL, ("phys_pager_getpages: partially valid page %p", m[i])); } vm_page_lock_queues(); for (i = 0; i < count; i++) { /* Switch off pv_entries */ vm_page_unmanage(m[i]); m[i]->dirty = 0; /* The requested page must remain busy, the others not. */ if (reqpage != i) { vm_page_flag_clear(m[i], PG_BUSY); m[i]->busy = 0; } } vm_page_unlock_queues(); return (VM_PAGER_OK); }
void hibernate_vm_lock(void) { if (getPerProc()->hibernate) { vm_page_lock_queues(); lck_mtx_lock(&vm_page_queue_free_lock); } }
void vm_fault_unhold_pages(vm_page_t *mp, int count) { KASSERT(count >= 0, ("negative count %d", count)); vm_page_lock_queues(); while (count--) { vm_page_unhold(*mp); mp++; } vm_page_unlock_queues(); }
/* * Allocate new wired pages in an object. * The object is assumed to be mapped into the kernel map or * a submap. */ void kmem_alloc_pages( vm_object_t object, vm_offset_t offset, vm_offset_t start, vm_offset_t end, vm_prot_t protection) { /* * Mark the pmap region as not pageable. */ pmap_pageable(kernel_pmap, start, end, FALSE); while (start < end) { vm_page_t mem; vm_object_lock(object); /* * Allocate a page */ while ((mem = vm_page_alloc(object, offset)) == VM_PAGE_NULL) { vm_object_unlock(object); VM_PAGE_WAIT((void (*)()) 0); vm_object_lock(object); } /* * Wire it down */ vm_page_lock_queues(); vm_page_wire(mem); vm_page_unlock_queues(); vm_object_unlock(object); /* * Enter it in the kernel pmap */ PMAP_ENTER(kernel_pmap, start, mem, protection, TRUE); vm_object_lock(object); PAGE_WAKEUP_DONE(mem); vm_object_unlock(object); start += PAGE_SIZE; offset += PAGE_SIZE; } }
/* * Remap wired pages in an object into a new region. * The object is assumed to be mapped into the kernel map or * a submap. */ void kmem_remap_pages( vm_object_t object, vm_offset_t offset, vm_offset_t start, vm_offset_t end, vm_prot_t protection) { /* * Mark the pmap region as not pageable. */ pmap_pageable(kernel_pmap, start, end, FALSE); while (start < end) { vm_page_t mem; vm_object_lock(object); /* * Find a page */ if ((mem = vm_page_lookup(object, offset)) == VM_PAGE_NULL) panic("kmem_remap_pages"); /* * Wire it down (again) */ vm_page_lock_queues(); vm_page_wire(mem); vm_page_unlock_queues(); vm_object_unlock(object); /* * Enter it in the kernel pmap. The page isn't busy, * but this shouldn't be a problem because it is wired. */ PMAP_ENTER(kernel_pmap, start, mem, protection, TRUE); start += PAGE_SIZE; offset += PAGE_SIZE; } }
static void socow_iodone(void *addr, void *args) { struct sf_buf *sf; vm_page_t pp; sf = args; pp = sf_buf_page(sf); sf_buf_free(sf); /* remove COW mapping */ vm_page_lock_queues(); vm_page_cowclear(pp); vm_page_unwire(pp, 0); /* * Check for the object going away on us. This can * happen since we don't hold a reference to it. * If so, we're responsible for freeing the page. */ if (pp->wire_count == 0 && pp->object == NULL) vm_page_free(pp); vm_page_unlock_queues(); socow_stats.iodone++; }
static void vm_pagezero(void __unused *arg) { struct rtprio rtp; struct thread *td; int pages, pri; td = curthread; rtp.prio = RTP_PRIO_MAX; rtp.type = RTP_PRIO_IDLE; pages = 0; mtx_lock_spin(&sched_lock); rtp_to_pri(&rtp, td->td_ksegrp); pri = td->td_priority; mtx_unlock_spin(&sched_lock); idlezero_enable = idlezero_enable_default; for (;;) { if (vm_page_zero_check()) { pages += vm_page_zero_idle(); #ifndef PREEMPTION if (pages > idlezero_maxrun || sched_runnable()) { mtx_lock_spin(&sched_lock); mi_switch(SW_VOL, NULL); mtx_unlock_spin(&sched_lock); pages = 0; } #endif } else { vm_page_lock_queues(); wakeup_needed = TRUE; msleep(&zero_state, &vm_page_queue_mtx, PDROP | pri, "pgzero", hz * 300); pages = 0; } } }
int acpi_sleep_machdep(struct acpi_softc *sc, int state) { ACPI_STATUS status; vm_offset_t oldphys; struct pmap *pm; vm_page_t page; static vm_page_t opage = NULL; int ret = 0; int pteobj_allocated = 0; u_long ef; struct proc *p; if (sc->acpi_wakeaddr == 0) { return (0); } AcpiSetFirmwareWakingVector(sc->acpi_wakephys); ef = read_eflags(); disable_intr(); /* Create Identity Mapping */ if ((p = curproc) == NULL) p = &proc0; pm = vmspace_pmap(p->p_vmspace); if (pm->pm_pteobj == NULL) { pm->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, PTDPTDI + 1); pteobj_allocated = 1; } oldphys = pmap_extract(pm, sc->acpi_wakephys); if (oldphys) { opage = PHYS_TO_VM_PAGE(oldphys); } page = PHYS_TO_VM_PAGE(sc->acpi_wakephys); pmap_enter(pm, sc->acpi_wakephys, page, VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE, 1); ret_addr = 0; if (acpi_savecpu()) { /* Execute Sleep */ p_gdt = (struct region_descriptor *)(sc->acpi_wakeaddr + physical_gdt); p_gdt->rd_limit = r_gdt.rd_limit; p_gdt->rd_base = vtophys(r_gdt.rd_base); WAKECODE_FIXUP(physical_esp, u_int32_t, vtophys(r_esp)); WAKECODE_FIXUP(previous_cr0, u_int32_t, r_cr0); WAKECODE_FIXUP(previous_cr2, u_int32_t, r_cr2); WAKECODE_FIXUP(previous_cr3, u_int32_t, r_cr3); WAKECODE_FIXUP(previous_cr4, u_int32_t, r_cr4); WAKECODE_FIXUP(previous_tr, u_int16_t, r_tr); WAKECODE_BCOPY(previous_gdt, struct region_descriptor, r_gdt); WAKECODE_FIXUP(previous_ldt, u_int16_t, r_ldt); WAKECODE_BCOPY(previous_idt, struct region_descriptor, r_idt); WAKECODE_FIXUP(where_to_recover, void, acpi_restorecpu); WAKECODE_FIXUP(previous_ds, u_int16_t, r_ds); WAKECODE_FIXUP(previous_es, u_int16_t, r_es); WAKECODE_FIXUP(previous_fs, u_int16_t, r_fs); WAKECODE_FIXUP(previous_gs, u_int16_t, r_gs); WAKECODE_FIXUP(previous_ss, u_int16_t, r_ss); if (acpi_get_verbose(sc)) { acpi_printcpu(); } wbinvd(); if (state == ACPI_STATE_S4 && sc->acpi_s4bios) { status = AcpiEnterSleepStateS4Bios(); } else { status = AcpiEnterSleepState(state); } if (status != AE_OK) { device_printf(sc->acpi_dev, "AcpiEnterSleepState failed - %s\n", AcpiFormatException(status)); ret = -1; goto out; } for (;;) ; } else { /* Execute Wakeup */ #if 0 initializecpu(); #endif icu_reinit(); if (acpi_get_verbose(sc)) { acpi_savecpu(); acpi_printcpu(); } } out: vm_page_lock_queues(); pmap_remove(pm, sc->acpi_wakephys, sc->acpi_wakephys + PAGE_SIZE); vm_page_unlock_queues(); if (opage) { pmap_enter(pm, sc->acpi_wakephys, page, VM_PROT_READ | VM_PROT_WRITE, 0); } if (pteobj_allocated) { vm_object_deallocate(pm->pm_pteobj); pm->pm_pteobj = NULL; } write_eflags(ef); return (ret); }
/* * This routine takes a user's map, array of pages, number of pages, and flags * and then does the following: * - validate that the user has access to those pages (flags indicates read * or write) - if not fail * - validate that count is enough to hold range number of pages - if not fail * - fault in any non-resident pages * - if the user is doing a read force a write fault for any COWed pages * - if the user is doing a read mark all pages as dirty * - hold all pages */ int vm_fault_hold_user_pages(vm_map_t map, vm_offset_t addr, vm_page_t *mp, int count, vm_prot_t prot) { vm_offset_t end, va; int faults, rv; pmap_t pmap; vm_page_t m, *pages; pmap = vm_map_pmap(map); pages = mp; addr &= ~PAGE_MASK; /* * Check that virtual address range is legal * This check is somewhat bogus as on some architectures kernel * and user do not share VA - however, it appears that all FreeBSD * architectures define it */ end = addr + (count * PAGE_SIZE); if (end > VM_MAXUSER_ADDRESS) { log(LOG_WARNING, "bad address passed to vm_fault_hold_user_pages"); return (EFAULT); } /* * First optimistically assume that all pages are resident * (and R/W if for write) if so just mark pages as held (and * dirty if for write) and return */ vm_page_lock_queues(); for (pages = mp, faults = 0, va = addr; va < end; va += PAGE_SIZE, pages++) { /* * page queue mutex is recursable so this is OK * it would be really nice if we had an unlocked * version of this so we were only acquiring the * pmap lock 1 time as opposed to potentially * many dozens of times */ *pages = m = pmap_extract_and_hold(pmap, va, prot); if (m == NULL) { faults++; continue; } /* * Preemptively mark dirty - the pages * will never have the modified bit set if * they are only changed via DMA */ if (prot & VM_PROT_WRITE) vm_page_dirty(m); } vm_page_unlock_queues(); if (faults == 0) return (0); /* * Pages either have insufficient permissions or are not present * trigger a fault where neccessary * */ rv = 0; for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) { /* * Account for a very narrow race where the page may be * taken away from us before it is held */ while (*pages == NULL) { rv = vm_fault(map, va, prot, (prot & VM_PROT_WRITE) ? VM_FAULT_DIRTY : VM_FAULT_NORMAL); if (rv) goto error; *pages = pmap_extract_and_hold(pmap, va, prot); } } return (0); error: log(LOG_WARNING, "vm_fault bad return rv=%d va=0x%zx\n", rv, va); vm_page_lock_queues(); for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) if (*pages) { vm_page_unhold(*pages); *pages = NULL; } vm_page_unlock_queues(); return (EFAULT); }
/* * vm_pageout_scan does the dirty work for the pageout daemon. */ void vm_pageout_scan() { register vm_page_t m, next; register int page_shortage; register int s; register int pages_freed; int free; vm_object_t object; /* * Only continue when we want more pages to be "free" */ cnt.v_rev++; s = splimp(); simple_lock(&vm_page_queue_free_lock); free = cnt.v_free_count; simple_unlock(&vm_page_queue_free_lock); splx(s); if (free < cnt.v_free_target) { swapout_threads(); /* * Be sure the pmap system is updated so * we can scan the inactive queue. */ pmap_update(); } /* * Acquire the resident page system lock, * as we may be changing what's resident quite a bit. */ vm_page_lock_queues(); /* * Start scanning the inactive queue for pages we can free. * We keep scanning until we have enough free pages or * we have scanned through the entire queue. If we * encounter dirty pages, we start cleaning them. */ pages_freed = 0; for (m = vm_page_queue_inactive.tqh_first; m != NULL; m = next) { s = splimp(); simple_lock(&vm_page_queue_free_lock); free = cnt.v_free_count; simple_unlock(&vm_page_queue_free_lock); splx(s); if (free >= cnt.v_free_target) break; cnt.v_scan++; next = m->pageq.tqe_next; /* * If the page has been referenced, move it back to the * active queue. */ if (pmap_is_referenced(VM_PAGE_TO_PHYS(m))) { vm_page_activate(m); cnt.v_reactivated++; continue; } /* * If the page is clean, free it up. */ if (m->flags & PG_CLEAN) { object = m->object; if (vm_object_lock_try(object)) { pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); vm_page_free(m); pages_freed++; cnt.v_dfree++; vm_object_unlock(object); } continue; } /* * If the page is dirty but already being washed, skip it. */ if ((m->flags & PG_LAUNDRY) == 0) continue; /* * Otherwise the page is dirty and still in the laundry, * so we start the cleaning operation and remove it from * the laundry. */ object = m->object; if (!vm_object_lock_try(object)) continue; cnt.v_pageouts++; #ifdef CLUSTERED_PAGEOUT if (object->pager && vm_pager_cancluster(object->pager, PG_CLUSTERPUT)) vm_pageout_cluster(m, object); else #endif vm_pageout_page(m, object); thread_wakeup((int) object); vm_object_unlock(object); /* * Former next page may no longer even be on the inactive * queue (due to potential blocking in the pager with the * queues unlocked). If it isn't, we just start over. */ if (next && (next->flags & PG_INACTIVE) == 0) next = vm_page_queue_inactive.tqh_first; } /* * Compute the page shortage. If we are still very low on memory * be sure that we will move a minimal amount of pages from active * to inactive. */ page_shortage = cnt.v_inactive_target - cnt.v_inactive_count; if (page_shortage <= 0 && pages_freed == 0) page_shortage = 1; while (page_shortage > 0) { /* * Move some more pages from active to inactive. */ if ((m = vm_page_queue_active.tqh_first) == NULL) break; vm_page_deactivate(m); page_shortage--; } vm_page_unlock_queues(); }
static int shm_dotruncate(struct shmfd *shmfd, off_t length) { vm_object_t object; vm_page_t m; vm_pindex_t nobjsize; vm_ooffset_t delta; object = shmfd->shm_object; VM_OBJECT_LOCK(object); if (length == shmfd->shm_size) { VM_OBJECT_UNLOCK(object); return (0); } nobjsize = OFF_TO_IDX(length + PAGE_MASK); /* Are we shrinking? If so, trim the end. */ if (length < shmfd->shm_size) { delta = ptoa(object->size - nobjsize); /* Toss in memory pages. */ if (nobjsize < object->size) vm_object_page_remove(object, nobjsize, object->size, FALSE); /* Toss pages from swap. */ if (object->type == OBJT_SWAP) swap_pager_freespace(object, nobjsize, delta); /* Free the swap accounted for shm */ swap_release_by_uid(delta, object->uip); object->charge -= delta; /* * If the last page is partially mapped, then zero out * the garbage at the end of the page. See comments * in vnode_pager_setsize() for more details. * * XXXJHB: This handles in memory pages, but what about * a page swapped out to disk? */ if ((length & PAGE_MASK) && (m = vm_page_lookup(object, OFF_TO_IDX(length))) != NULL && m->valid != 0) { int base = (int)length & PAGE_MASK; int size = PAGE_SIZE - base; pmap_zero_page_area(m, base, size); /* * Update the valid bits to reflect the blocks that * have been zeroed. Some of these valid bits may * have already been set. */ vm_page_set_valid(m, base, size); /* * Round "base" to the next block boundary so that the * dirty bit for a partially zeroed block is not * cleared. */ base = roundup2(base, DEV_BSIZE); vm_page_lock_queues(); vm_page_clear_dirty(m, base, PAGE_SIZE - base); vm_page_unlock_queues(); } else if ((length & PAGE_MASK) && __predict_false(object->cache != NULL)) { vm_page_cache_free(object, OFF_TO_IDX(length), nobjsize); } } else { /* Attempt to reserve the swap */ delta = ptoa(nobjsize - object->size); if (!swap_reserve_by_uid(delta, object->uip)) { VM_OBJECT_UNLOCK(object); return (ENOMEM); } object->charge += delta; } shmfd->shm_size = length; mtx_lock(&shm_timestamp_lock); vfs_timestamp(&shmfd->shm_ctime); shmfd->shm_mtime = shmfd->shm_ctime; mtx_unlock(&shm_timestamp_lock); object->size = nobjsize; VM_OBJECT_UNLOCK(object); return (0); }
int proc_rwmem(struct proc *p, struct uio *uio) { struct vmspace *vm; vm_map_t map; vm_object_t object = NULL; vm_offset_t pageno = 0; /* page number */ vm_prot_t reqprot; vm_offset_t kva; int error, writing; GIANT_REQUIRED; /* * if the vmspace is in the midst of being deallocated or the * process is exiting, don't try to grab anything. The page table * usage in that process can be messed up. */ vm = p->p_vmspace; if ((p->p_flag & P_WEXIT)) return (EFAULT); if (vm->vm_refcnt < 1) return (EFAULT); ++vm->vm_refcnt; /* * The map we want... */ map = &vm->vm_map; writing = uio->uio_rw == UIO_WRITE; reqprot = writing ? (VM_PROT_WRITE | VM_PROT_OVERRIDE_WRITE) : VM_PROT_READ; kva = kmem_alloc_pageable(kernel_map, PAGE_SIZE); /* * Only map in one page at a time. We don't have to, but it * makes things easier. This way is trivial - right? */ do { vm_map_t tmap; vm_offset_t uva; int page_offset; /* offset into page */ vm_map_entry_t out_entry; vm_prot_t out_prot; boolean_t wired; vm_pindex_t pindex; u_int len; vm_page_t m; object = NULL; uva = (vm_offset_t)uio->uio_offset; /* * Get the page number of this segment. */ pageno = trunc_page(uva); page_offset = uva - pageno; /* * How many bytes to copy */ len = min(PAGE_SIZE - page_offset, uio->uio_resid); /* * Fault the page on behalf of the process */ error = vm_fault(map, pageno, reqprot, VM_FAULT_NORMAL); if (error) { error = EFAULT; break; } /* * Now we need to get the page. out_entry, out_prot, wired, * and single_use aren't used. One would think the vm code * would be a *bit* nicer... We use tmap because * vm_map_lookup() can change the map argument. */ tmap = map; error = vm_map_lookup(&tmap, pageno, reqprot, &out_entry, &object, &pindex, &out_prot, &wired); if (error) { error = EFAULT; /* * Make sure that there is no residue in 'object' from * an error return on vm_map_lookup. */ object = NULL; break; } m = vm_page_lookup(object, pindex); /* Allow fallback to backing objects if we are reading */ while (m == NULL && !writing && object->backing_object) { pindex += OFF_TO_IDX(object->backing_object_offset); object = object->backing_object; m = vm_page_lookup(object, pindex); } if (m == NULL) { error = EFAULT; /* * Make sure that there is no residue in 'object' from * an error return on vm_map_lookup. */ object = NULL; vm_map_lookup_done(tmap, out_entry); break; } /* * Wire the page into memory */ vm_page_lock_queues(); vm_page_wire(m); vm_page_unlock_queues(); /* * We're done with tmap now. * But reference the object first, so that we won't loose * it. */ vm_object_reference(object); vm_map_lookup_done(tmap, out_entry); pmap_qenter(kva, &m, 1); /* * Now do the i/o move. */ error = uiomove((caddr_t)(kva + page_offset), len, uio); pmap_qremove(kva, 1); /* * release the page and the object */ vm_page_lock_queues(); vm_page_unwire(m, 1); vm_page_unlock_queues(); vm_object_deallocate(object); object = NULL; } while (error == 0 && uio->uio_resid > 0); if (object) vm_object_deallocate(object); kmem_free(kernel_map, kva, PAGE_SIZE); vmspace_free(vm); return (error); }
int socow_setup(struct mbuf *m0, struct uio *uio) { struct sf_buf *sf; vm_page_t pp; struct iovec *iov; struct vmspace *vmspace; struct vm_map *map; vm_offset_t offset, uva; socow_stats.attempted++; vmspace = curproc->p_vmspace; map = &vmspace->vm_map; uva = (vm_offset_t) uio->uio_iov->iov_base; offset = uva & PAGE_MASK; /* * Verify that access to the given address is allowed from user-space. */ if (vm_fault_quick((caddr_t)uva, VM_PROT_READ) < 0) return (0); /* * verify page is mapped & not already wired for i/o */ pp = pmap_extract_and_hold(map->pmap, uva, VM_PROT_READ); if (pp == NULL) { socow_stats.fail_not_mapped++; return(0); } /* * set up COW */ vm_page_lock_queues(); if (vm_page_cowsetup(pp) != 0) { vm_page_unhold(pp); vm_page_unlock_queues(); return (0); } /* * wire the page for I/O */ vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock_queues(); /* * Allocate an sf buf */ sf = sf_buf_alloc(pp, SFB_CATCH); if (!sf) { vm_page_lock_queues(); vm_page_cowclear(pp); vm_page_unwire(pp, 0); /* * Check for the object going away on us. This can * happen since we don't hold a reference to it. * If so, we're responsible for freeing the page. */ if (pp->wire_count == 0 && pp->object == NULL) vm_page_free(pp); vm_page_unlock_queues(); socow_stats.fail_sf_buf++; return(0); } /* * attach to mbuf */ MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone, (void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF); m0->m_len = PAGE_SIZE - offset; m0->m_data = (caddr_t)sf_buf_kva(sf) + offset; socow_stats.success++; iov = uio->uio_iov; iov->iov_base = (char *)iov->iov_base + m0->m_len; iov->iov_len -= m0->m_len; uio->uio_resid -= m0->m_len; uio->uio_offset += m0->m_len; if (iov->iov_len == 0) { uio->uio_iov++; uio->uio_iovcnt--; } return(m0->m_len); }
DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem) { PRTR0MEMOBJFREEBSD pMemFreeBSD = (PRTR0MEMOBJFREEBSD)pMem; int rc; switch (pMemFreeBSD->Core.enmType) { case RTR0MEMOBJTYPE_PAGE: case RTR0MEMOBJTYPE_LOW: case RTR0MEMOBJTYPE_CONT: rc = vm_map_remove(kernel_map, (vm_offset_t)pMemFreeBSD->Core.pv, (vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb); AssertMsg(rc == KERN_SUCCESS, ("%#x", rc)); break; case RTR0MEMOBJTYPE_LOCK: { vm_map_t pMap = kernel_map; if (pMemFreeBSD->Core.u.Lock.R0Process != NIL_RTR0PROCESS) pMap = &((struct proc *)pMemFreeBSD->Core.u.Lock.R0Process)->p_vmspace->vm_map; rc = vm_map_unwire(pMap, (vm_offset_t)pMemFreeBSD->Core.pv, (vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb, VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); AssertMsg(rc == KERN_SUCCESS, ("%#x", rc)); break; } case RTR0MEMOBJTYPE_RES_VIRT: { vm_map_t pMap = kernel_map; if (pMemFreeBSD->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS) pMap = &((struct proc *)pMemFreeBSD->Core.u.ResVirt.R0Process)->p_vmspace->vm_map; rc = vm_map_remove(pMap, (vm_offset_t)pMemFreeBSD->Core.pv, (vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb); AssertMsg(rc == KERN_SUCCESS, ("%#x", rc)); break; } case RTR0MEMOBJTYPE_MAPPING: { vm_map_t pMap = kernel_map; if (pMemFreeBSD->Core.u.Mapping.R0Process != NIL_RTR0PROCESS) pMap = &((struct proc *)pMemFreeBSD->Core.u.Mapping.R0Process)->p_vmspace->vm_map; rc = vm_map_remove(pMap, (vm_offset_t)pMemFreeBSD->Core.pv, (vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb); AssertMsg(rc == KERN_SUCCESS, ("%#x", rc)); break; } case RTR0MEMOBJTYPE_PHYS: case RTR0MEMOBJTYPE_PHYS_NC: { #if __FreeBSD_version >= 1000030 VM_OBJECT_WLOCK(pMemFreeBSD->pObject); #else VM_OBJECT_LOCK(pMemFreeBSD->pObject); #endif vm_page_t pPage = vm_page_find_least(pMemFreeBSD->pObject, 0); vm_page_lock_queues(); for (vm_page_t pPage = vm_page_find_least(pMemFreeBSD->pObject, 0); pPage != NULL; pPage = vm_page_next(pPage)) { vm_page_unwire(pPage, 0); } vm_page_unlock_queues(); #if __FreeBSD_version >= 1000030 VM_OBJECT_WUNLOCK(pMemFreeBSD->pObject); #else VM_OBJECT_UNLOCK(pMemFreeBSD->pObject); #endif vm_object_deallocate(pMemFreeBSD->pObject); break; } default: AssertMsgFailed(("enmType=%d\n", pMemFreeBSD->Core.enmType)); return VERR_INTERNAL_ERROR; } return VINF_SUCCESS; }
static __inline void ma_vm_page_lock_queues(void) { vm_page_lock_queues(); };