int
sf_buf_unmap(struct sf_buf *sf)
{
pmap_qremove(sf->kva, 1);
return (1);
}
static void
free_pagelist(PAGELIST_T *pagelist, int actual)
{
vm_page_t *pages;
unsigned int num_pages, i;
vcos_log_trace("free_pagelist - %x, %d", (unsigned int)pagelist, actual);
num_pages =
(pagelist->length + pagelist->offset + PAGE_SIZE - 1) / PAGE_SIZE;
pages = (vm_page_t *)(pagelist->addrs + num_pages);
/* Deal with any partial cache lines (fragments) */
if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
FRAGMENTS_T *fragments =
g_fragments_base + (pagelist->type -
PAGELIST_READ_WITH_FRAGMENTS);
int head_bytes, tail_bytes;
if (actual >= 0)
{
/* XXXBSD: might be inefficient */
void *page_address = pmap_mapdev(VM_PAGE_TO_PHYS(pages[0]), PAGE_SIZE*num_pages);
if ((head_bytes = (CACHE_LINE_SIZE - pagelist->offset) & (CACHE_LINE_SIZE - 1)) != 0) {
if (head_bytes > actual)
head_bytes = actual;
memcpy((char *)page_address +
pagelist->offset, fragments->headbuf,
head_bytes);
}
if ((head_bytes < actual) &&
(tail_bytes =
(pagelist->offset + actual) & (CACHE_LINE_SIZE -
1)) != 0) {
memcpy((char *)page_address + PAGE_SIZE*(num_pages - 1) +
((pagelist->offset + actual) & (PAGE_SIZE -
1) & ~(CACHE_LINE_SIZE - 1)),
fragments->tailbuf, tail_bytes);
}
pmap_qremove((vm_offset_t)page_address, PAGE_SIZE*num_pages);
}
mtx_lock(&g_free_fragments_mutex);
*(FRAGMENTS_T **) fragments = g_free_fragments;
g_free_fragments = fragments;
mtx_unlock(&g_free_fragments_mutex);
sema_post(&g_free_fragments_sema);
}
for (i = 0; i < num_pages; i++) {
if (pagelist->type != PAGELIST_WRITE)
vm_page_dirty(pages[i]);
}
vm_page_unhold_pages(pages, num_pages);
free(pagelist, M_VCPAGELIST);
}
void
OS_UnmapPage(Mapping mapping) // IN
{
pmap_qremove((vm_offset_t)mapping, 1);
#if __FreeBSD_version < 1000000
kmem_free(kernel_map, (vm_offset_t)mapping, PAGE_SIZE);
#else
kva_free((vm_offset_t)mapping, PAGE_SIZE);
#endif
}
/*
* No requirements.
*/
void
contigfree(void *addr, unsigned long size, struct malloc_type *type)
{
vm_paddr_t pa;
vm_page_t m;
if (size == 0)
panic("vm_contig_pg_kmap: size must not be 0");
size = round_page(size);
pa = pmap_extract(&kernel_pmap, (vm_offset_t)addr);
pmap_qremove((vm_offset_t)addr, size / PAGE_SIZE);
kmem_free(&kernel_map, (vm_offset_t)addr, size);
m = PHYS_TO_VM_PAGE(pa);
vm_page_free_contig(m, size);
}
static int
spigen_close(struct cdev *cdev, int fflag, int devtype, struct thread *td)
{
device_t dev = cdev->si_drv1;
struct spigen_softc *sc = device_get_softc(dev);
mtx_lock(&sc->sc_mtx);
if (sc->sc_mmap_buffer != NULL) {
pmap_qremove(sc->sc_mmap_kvaddr,
sc->sc_mmap_buffer_size / PAGE_SIZE);
kva_free(sc->sc_mmap_kvaddr, sc->sc_mmap_buffer_size);
sc->sc_mmap_kvaddr = 0;
vm_object_deallocate(sc->sc_mmap_buffer);
sc->sc_mmap_buffer = NULL;
sc->sc_mmap_buffer_size = 0;
}
mtx_unlock(&sc->sc_mtx);
return (0);
}
/*ARGSUSED*/
static int
mmrw(dev_t dev, struct uio *uio, int flags)
{
int o;
u_long c = 0, v;
struct iovec *iov;
int error = 0;
vm_offset_t addr, eaddr;
GIANT_REQUIRED;
while (uio->uio_resid > 0 && error == 0) {
iov = uio->uio_iov;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
if (uio->uio_iovcnt < 0)
panic("mmrw");
continue;
}
switch (minor(dev)) {
/* minor device 0 is physical memory */
case 0:
v = uio->uio_offset;
v &= ~PAGE_MASK;
pmap_kenter((vm_offset_t)ptvmmap, v);
o = (int)uio->uio_offset & PAGE_MASK;
c = (u_long)(PAGE_SIZE - ((long)iov->iov_base & PAGE_MASK));
c = min(c, (u_int)(PAGE_SIZE - o));
c = min(c, (u_int)iov->iov_len);
error = uiomove((caddr_t)&ptvmmap[o], (int)c, uio);
pmap_qremove((vm_offset_t)ptvmmap, 1);
continue;
/* minor device 1 is kernel memory */
case 1:
c = iov->iov_len;
/*
* Make sure that all of the pages are currently resident so
* that we don't create any zero-fill pages.
*/
addr = trunc_page(uio->uio_offset);
eaddr = round_page(uio->uio_offset + c);
if (addr < (vm_offset_t)KERNBASE)
return (EFAULT);
for (; addr < eaddr; addr += PAGE_SIZE)
if (pmap_extract(kernel_pmap, addr) == 0)
return (EFAULT);
if (!kernacc((caddr_t)(long)uio->uio_offset, c,
uio->uio_rw == UIO_READ ?
VM_PROT_READ : VM_PROT_WRITE))
return (EFAULT);
error = uiomove((caddr_t)(long)uio->uio_offset, (int)c, uio);
continue;
default:
return (ENODEV);
}
if (error)
break;
iov->iov_base = (char *)iov->iov_base + c;
iov->iov_len -= c;
uio->uio_offset += c;
uio->uio_resid -= c;
}
return (error);
}
/* ARGSUSED */
int
memrw(struct cdev *dev, struct uio *uio, int flags)
{
struct iovec *iov;
vm_offset_t eva;
vm_offset_t off;
vm_offset_t ova;
vm_offset_t va;
vm_prot_t prot;
vm_paddr_t pa;
vm_size_t cnt;
vm_page_t m;
int error;
int i;
uint32_t colors;
cnt = 0;
colors = 1;
error = 0;
ova = 0;
GIANT_REQUIRED;
while (uio->uio_resid > 0 && error == 0) {
iov = uio->uio_iov;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
if (uio->uio_iovcnt < 0)
panic("memrw");
continue;
}
if (dev2unit(dev) == CDEV_MINOR_MEM) {
pa = uio->uio_offset & ~PAGE_MASK;
if (!is_physical_memory(pa)) {
error = EFAULT;
break;
}
off = uio->uio_offset & PAGE_MASK;
cnt = PAGE_SIZE - ((vm_offset_t)iov->iov_base &
PAGE_MASK);
cnt = ulmin(cnt, PAGE_SIZE - off);
cnt = ulmin(cnt, iov->iov_len);
m = NULL;
for (i = 0; phys_avail[i] != 0; i += 2) {
if (pa >= phys_avail[i] &&
pa < phys_avail[i + 1]) {
m = PHYS_TO_VM_PAGE(pa);
break;
}
}
if (m != NULL) {
if (ova == 0) {
if (dcache_color_ignore == 0)
colors = DCACHE_COLORS;
ova = kmem_alloc_wait(kernel_map,
PAGE_SIZE * colors);
}
if (colors != 1 && m->md.color != -1)
va = ova + m->md.color * PAGE_SIZE;
else
va = ova;
pmap_qenter(va, &m, 1);
error = uiomove((void *)(va + off), cnt,
uio);
pmap_qremove(va, 1);
} else {
va = TLB_PHYS_TO_DIRECT(pa);
error = uiomove((void *)(va + off), cnt,
uio);
}
break;
} else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
va = trunc_page(uio->uio_offset);
eva = round_page(uio->uio_offset + iov->iov_len);
/*
* Make sure that all of the pages are currently
* resident so we don't create any zero fill pages.
*/
for (; va < eva; va += PAGE_SIZE)
if (pmap_kextract(va) == 0)
return (EFAULT);
prot = (uio->uio_rw == UIO_READ) ? VM_PROT_READ :
VM_PROT_WRITE;
va = uio->uio_offset;
if (va < VM_MIN_DIRECT_ADDRESS &&
kernacc((void *)va, iov->iov_len, prot) == FALSE)
return (EFAULT);
error = uiomove((void *)va, iov->iov_len, uio);
break;
}
/* else panic! */
}
if (ova != 0)
kmem_free_wakeup(kernel_map, ova, PAGE_SIZE * colors);
return (error);
}
int
physio(struct cdev *dev, struct uio *uio, int ioflag)
{
struct cdevsw *csw;
struct buf *pbuf;
struct bio *bp;
struct vm_page **pages;
caddr_t sa;
u_int iolen, poff;
int error, i, npages, maxpages;
vm_prot_t prot;
csw = dev->si_devsw;
npages = 0;
sa = NULL;
/* check if character device is being destroyed */
if (csw == NULL)
return (ENXIO);
/* XXX: sanity check */
if(dev->si_iosize_max < PAGE_SIZE) {
printf("WARNING: %s si_iosize_max=%d, using DFLTPHYS.\n",
devtoname(dev), dev->si_iosize_max);
dev->si_iosize_max = DFLTPHYS;
}
/*
* If the driver does not want I/O to be split, that means that we
* need to reject any requests that will not fit into one buffer.
*/
if (dev->si_flags & SI_NOSPLIT &&
(uio->uio_resid > dev->si_iosize_max || uio->uio_resid > MAXPHYS ||
uio->uio_iovcnt > 1)) {
/*
* Tell the user why his I/O was rejected.
*/
if (uio->uio_resid > dev->si_iosize_max)
uprintf("%s: request size=%zd > si_iosize_max=%d; "
"cannot split request\n", devtoname(dev),
uio->uio_resid, dev->si_iosize_max);
if (uio->uio_resid > MAXPHYS)
uprintf("%s: request size=%zd > MAXPHYS=%d; "
"cannot split request\n", devtoname(dev),
uio->uio_resid, MAXPHYS);
if (uio->uio_iovcnt > 1)
uprintf("%s: request vectors=%d > 1; "
"cannot split request\n", devtoname(dev),
uio->uio_iovcnt);
return (EFBIG);
}
/*
* Keep the process UPAGES from being swapped. Processes swapped
* out while holding pbufs, used by swapper, may lead to deadlock.
*/
PHOLD(curproc);
bp = g_alloc_bio();
if (uio->uio_segflg != UIO_USERSPACE) {
pbuf = NULL;
pages = NULL;
} else if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) {
pbuf = NULL;
maxpages = btoc(MIN(uio->uio_resid, MAXPHYS)) + 1;
pages = malloc(sizeof(*pages) * maxpages, M_DEVBUF, M_WAITOK);
} else {
pbuf = uma_zalloc(pbuf_zone, M_WAITOK);
sa = pbuf->b_data;
maxpages = btoc(MAXPHYS);
pages = pbuf->b_pages;
}
prot = VM_PROT_READ;
if (uio->uio_rw == UIO_READ)
prot |= VM_PROT_WRITE; /* Less backwards than it looks */
error = 0;
for (i = 0; i < uio->uio_iovcnt; i++) {
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(curproc);
if (uio->uio_rw == UIO_READ) {
racct_add_force(curproc, RACCT_READBPS,
uio->uio_iov[i].iov_len);
racct_add_force(curproc, RACCT_READIOPS, 1);
} else {
racct_add_force(curproc, RACCT_WRITEBPS,
uio->uio_iov[i].iov_len);
racct_add_force(curproc, RACCT_WRITEIOPS, 1);
}
PROC_UNLOCK(curproc);
}
#endif /* RACCT */
while (uio->uio_iov[i].iov_len) {
g_reset_bio(bp);
if (uio->uio_rw == UIO_READ) {
bp->bio_cmd = BIO_READ;
curthread->td_ru.ru_inblock++;
} else {
bp->bio_cmd = BIO_WRITE;
curthread->td_ru.ru_oublock++;
}
bp->bio_offset = uio->uio_offset;
bp->bio_data = uio->uio_iov[i].iov_base;
bp->bio_length = uio->uio_iov[i].iov_len;
if (bp->bio_length > dev->si_iosize_max)
bp->bio_length = dev->si_iosize_max;
if (bp->bio_length > MAXPHYS)
bp->bio_length = MAXPHYS;
/*
* Make sure the pbuf can map the request.
* The pbuf has kvasize = MAXPHYS, so a request
* larger than MAXPHYS - PAGE_SIZE must be
* page aligned or it will be fragmented.
*/
poff = (vm_offset_t)bp->bio_data & PAGE_MASK;
if (pbuf && bp->bio_length + poff > pbuf->b_kvasize) {
if (dev->si_flags & SI_NOSPLIT) {
uprintf("%s: request ptr %p is not "
"on a page boundary; cannot split "
"request\n", devtoname(dev),
bp->bio_data);
error = EFBIG;
goto doerror;
}
bp->bio_length = pbuf->b_kvasize;
if (poff != 0)
bp->bio_length -= PAGE_SIZE;
}
bp->bio_bcount = bp->bio_length;
bp->bio_dev = dev;
if (pages) {
if ((npages = vm_fault_quick_hold_pages(
&curproc->p_vmspace->vm_map,
(vm_offset_t)bp->bio_data, bp->bio_length,
prot, pages, maxpages)) < 0) {
error = EFAULT;
goto doerror;
}
if (pbuf && sa) {
pmap_qenter((vm_offset_t)sa,
pages, npages);
bp->bio_data = sa + poff;
} else {
bp->bio_ma = pages;
bp->bio_ma_n = npages;
bp->bio_ma_offset = poff;
bp->bio_data = unmapped_buf;
bp->bio_flags |= BIO_UNMAPPED;
}
}
csw->d_strategy(bp);
if (uio->uio_rw == UIO_READ)
biowait(bp, "physrd");
else
biowait(bp, "physwr");
if (pages) {
if (pbuf)
pmap_qremove((vm_offset_t)sa, npages);
vm_page_unhold_pages(pages, npages);
}
iolen = bp->bio_length - bp->bio_resid;
if (iolen == 0 && !(bp->bio_flags & BIO_ERROR))
goto doerror; /* EOF */
uio->uio_iov[i].iov_len -= iolen;
uio->uio_iov[i].iov_base =
(char *)uio->uio_iov[i].iov_base + iolen;
uio->uio_resid -= iolen;
uio->uio_offset += iolen;
if (bp->bio_flags & BIO_ERROR) {
error = bp->bio_error;
goto doerror;
}
}
}
doerror:
if (pbuf)
uma_zfree(pbuf_zone, pbuf);
else if (pages)
free(pages, M_DEVBUF);
g_destroy_bio(bp);
PRELE(curproc);
return (error);
}
/*
struct vnop_putpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_sync;
int *a_rtvals;
vm_ooffset_t a_offset;
};
*/
static int
fuse_vnop_putpages(struct vop_putpages_args *ap)
{
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, error, npages, count;
off_t offset;
int *rtvals;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
vm_ooffset_t fsize;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to putpages"));
fsize = vp->v_object->un_pager.vnp.vnp_size;
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
offset = IDX_TO_OFF(pages[0]->pindex);
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
}
for (i = 0; i < npages; i++)
rtvals[i] = VM_PAGER_AGAIN;
/*
* When putting pages, do not extend file past EOF.
*/
if (offset + count > fsize) {
count = fsize - offset;
if (count < 0)
count = 0;
}
/*
* We use only the kva address for the buffer, but this is extremely
* convienient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodeout);
PCPU_ADD(cnt.v_vnodepgsout, count);
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = offset;
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
VM_OBJECT_WLOCK(pages[i]->object);
vm_page_undirty(pages[i]);
VM_OBJECT_WUNLOCK(pages[i]->object);
}
}
return rtvals[0];
}
/*
* spec_getpages() - get pages associated with device vnode.
*
* Note that spec_read and spec_write do not use the buffer cache, so we
* must fully implement getpages here.
*/
static int
devfs_spec_getpages(struct vop_getpages_args *ap)
{
vm_offset_t kva;
int error;
int i, pcount, size;
struct buf *bp;
vm_page_t m;
vm_ooffset_t offset;
int toff, nextoff, nread;
struct vnode *vp = ap->a_vp;
int blksiz;
int gotreqpage;
error = 0;
pcount = round_page(ap->a_count) / PAGE_SIZE;
/*
* Calculate the offset of the transfer and do sanity check.
*/
offset = IDX_TO_OFF(ap->a_m[0]->pindex) + ap->a_offset;
/*
* Round up physical size for real devices. We cannot round using
* v_mount's block size data because v_mount has nothing to do with
* the device. i.e. it's usually '/dev'. We need the physical block
* size for the device itself.
*
* We can't use v_rdev->si_mountpoint because it only exists when the
* block device is mounted. However, we can use v_rdev.
*/
if (vn_isdisk(vp, NULL))
blksiz = vp->v_rdev->si_bsize_phys;
else
blksiz = DEV_BSIZE;
size = (ap->a_count + blksiz - 1) & ~(blksiz - 1);
bp = getpbuf_kva(NULL);
kva = (vm_offset_t)bp->b_data;
/*
* Map the pages to be read into the kva.
*/
pmap_qenter(kva, ap->a_m, pcount);
/* Build a minimal buffer header. */
bp->b_cmd = BUF_CMD_READ;
bp->b_bcount = size;
bp->b_resid = 0;
bsetrunningbufspace(bp, size);
bp->b_bio1.bio_offset = offset;
bp->b_bio1.bio_done = devfs_spec_getpages_iodone;
mycpu->gd_cnt.v_vnodein++;
mycpu->gd_cnt.v_vnodepgsin += pcount;
/* Do the input. */
vn_strategy(ap->a_vp, &bp->b_bio1);
crit_enter();
/* We definitely need to be at splbio here. */
while (bp->b_cmd != BUF_CMD_DONE)
tsleep(bp, 0, "spread", 0);
crit_exit();
if (bp->b_flags & B_ERROR) {
if (bp->b_error)
error = bp->b_error;
else
error = EIO;
}
/*
* If EOF is encountered we must zero-extend the result in order
* to ensure that the page does not contain garabge. When no
* error occurs, an early EOF is indicated if b_bcount got truncated.
* b_resid is relative to b_bcount and should be 0, but some devices
* might indicate an EOF with b_resid instead of truncating b_bcount.
*/
nread = bp->b_bcount - bp->b_resid;
if (nread < ap->a_count)
bzero((caddr_t)kva + nread, ap->a_count - nread);
pmap_qremove(kva, pcount);
gotreqpage = 0;
for (i = 0, toff = 0; i < pcount; i++, toff = nextoff) {
nextoff = toff + PAGE_SIZE;
m = ap->a_m[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: vm_page_undirty/clear_dirty etc do not clear the
* pmap modified bit. pmap modified bit should have
* already been cleared.
*/
if (nextoff <= nread) {
m->valid = VM_PAGE_BITS_ALL;
vm_page_undirty(m);
} else if (toff < nread) {
/*
* Since this is a VM request, we have to supply the
* unaligned offset to allow vm_page_set_valid()
* to zero sub-DEV_BSIZE'd portions of the page.
*/
vm_page_set_valid(m, 0, nread - toff);
vm_page_clear_dirty_end_nonincl(m, 0, nread - toff);
} else {
m->valid = 0;
vm_page_undirty(m);
}
if (i != ap->a_reqpage) {
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error || (m->valid == VM_PAGE_BITS_ALL)) {
if (m->valid) {
if (m->flags & PG_REFERENCED) {
vm_page_activate(m);
} else {
vm_page_deactivate(m);
}
vm_page_wakeup(m);
} else {
vm_page_free(m);
}
} else {
vm_page_free(m);
}
} else if (m->valid) {
gotreqpage = 1;
/*
* Since this is a VM request, we need to make the
* entire page presentable by zeroing invalid sections.
*/
if (m->valid != VM_PAGE_BITS_ALL)
vm_page_zero_invalid(m, FALSE);
}
}
if (!gotreqpage) {
m = ap->a_m[ap->a_reqpage];
devfs_debug(DEVFS_DEBUG_WARNING,
"spec_getpages:(%s) I/O read failure: (error=%d) bp %p vp %p\n",
devtoname(vp->v_rdev), error, bp, bp->b_vp);
devfs_debug(DEVFS_DEBUG_WARNING,
" size: %d, resid: %d, a_count: %d, valid: 0x%x\n",
size, bp->b_resid, ap->a_count, m->valid);
devfs_debug(DEVFS_DEBUG_WARNING,
" nread: %d, reqpage: %d, pindex: %lu, pcount: %d\n",
nread, ap->a_reqpage, (u_long)m->pindex, pcount);
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
return VM_PAGER_ERROR;
}
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
if (DEVFS_NODE(ap->a_vp))
nanotime(&DEVFS_NODE(ap->a_vp)->mtime);
return VM_PAGER_OK;
}
static int
vnode_pager_generic_getpages_done(struct buf *bp)
{
vm_object_t object;
off_t tfoff, nextoff;
int i, error;
error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0;
object = bp->b_vp->v_object;
if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) {
if (!buf_mapped(bp)) {
bp->b_data = bp->b_kvabase;
pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages,
bp->b_npages);
}
bzero(bp->b_data + bp->b_bcount,
PAGE_SIZE * bp->b_npages - bp->b_bcount);
}
if (buf_mapped(bp)) {
pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
bp->b_data = unmapped_buf;
}
VM_OBJECT_WLOCK(object);
for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex);
i < bp->b_npages; i++, tfoff = nextoff) {
vm_page_t mt;
nextoff = tfoff + PAGE_SIZE;
mt = bp->b_pages[i];
if (nextoff <= object->un_pager.vnp.vnp_size) {
/*
* Read filled up entire page.
*/
mt->valid = VM_PAGE_BITS_ALL;
KASSERT(mt->dirty == 0,
("%s: page %p is dirty", __func__, mt));
KASSERT(!pmap_page_is_mapped(mt),
("%s: page %p is mapped", __func__, mt));
} else {
/*
* Read did not fill up entire page.
*
* Currently we do not set the entire page valid,
* we just try to clear the piece that we couldn't
* read.
*/
vm_page_set_valid_range(mt, 0,
object->un_pager.vnp.vnp_size - tfoff);
KASSERT((mt->dirty & vm_page_bits(0,
object->un_pager.vnp.vnp_size - tfoff)) == 0,
("%s: page %p is dirty", __func__, mt));
}
if (i < bp->b_pgbefore || i >= bp->b_npages - bp->b_pgafter)
vm_page_readahead_finish(mt);
}
VM_OBJECT_WUNLOCK(object);
if (error != 0)
printf("%s: I/O read error %d\n", __func__, error);
return (error);
}
/*
* Vnode op for VM getpages.
* Wish wish .... get rid from multiple IO routines
*
* nwfs_getpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_reqpage, vm_ooffset_t a_offset)
*/
int
nwfs_getpages(struct vop_getpages_args *ap)
{
#ifndef NWFS_RWCACHE
return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
ap->a_reqpage, ap->a_seqaccess);
#else
int i, error, npages;
size_t nextoff, toff;
size_t count;
size_t size;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
struct nwmount *nmp;
struct nwnode *np;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
vp = ap->a_vp;
np = VTONW(vp);
nmp = VFSTONWFS(vp->v_mount);
pages = ap->a_m;
count = (size_t)ap->a_count;
if (vp->v_object == NULL) {
kprintf("nwfs_getpages: called with non-merged cache vnode??\n");
return VM_PAGER_ERROR;
}
bp = getpbuf_kva(&nwfs_pbuf_freecnt);
npages = btoc(count);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
error = ncp_read(NWFSTOCONN(nmp), &np->n_fh, &uio,cred);
pmap_qremove(kva, npages);
relpbuf(bp, &nwfs_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
kprintf("nwfs_getpages: error %d\n",error);
for (i = 0; i < npages; i++) {
if (ap->a_reqpage != i)
vnode_pager_freepage(pages[i]);
}
return VM_PAGER_ERROR;
}
size = count - uio.uio_resid;
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: pmap dirty bit should have already been cleared.
* We do not clear it here.
*/
if (nextoff <= size) {
m->valid = VM_PAGE_BITS_ALL;
m->dirty = 0;
} else {
int nvalid = ((size + DEV_BSIZE - 1) - toff) &
~(DEV_BSIZE - 1);
vm_page_set_validclean(m, 0, nvalid);
}
if (i != ap->a_reqpage) {
/*
* Whether or not to leave the page activated is up in
* the air, but we should put the page on a page queue
* somewhere (it already is in the object). Result:
* It appears that emperical results show that
* deactivating pages is best.
*/
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error) {
if (m->flags & PG_REFERENCED)
vm_page_activate(m);
else
vm_page_deactivate(m);
vm_page_wakeup(m);
} else {
vnode_pager_freepage(m);
}
}
}
return 0;
#endif /* NWFS_RWCACHE */
}
/*
* Vnode op for VM putpages.
* possible bug: all IO done in sync mode
* Note that vop_close always invalidate pages before close, so it's
* not necessary to open vnode.
*
* smbfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count, int a_sync,
* int *a_rtvals, vm_ooffset_t a_offset)
*/
int
smbfs_putpages(struct vop_putpages_args *ap)
{
int error;
struct vnode *vp = ap->a_vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
#ifdef SMBFS_RWGENERIC
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
VOP_OPEN(vp, FWRITE, cred, NULL);
error = vop_stdputpages(ap);
VOP_CLOSE(vp, FWRITE, cred);
return error;
#else
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, npages, count;
int doclose;
int *rtvals;
struct smbmount *smp;
struct smbnode *np;
struct smb_cred scred;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
/* VOP_OPEN(vp, FWRITE, cred, NULL);*/
np = VTOSMB(vp);
smp = VFSTOSMBFS(vp->v_mount);
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
for (i = 0; i < npages; i++) {
rtvals[i] = VM_PAGER_AGAIN;
}
bp = getpbuf_kva(&smbfs_pbuf_freecnt);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
SMBVDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid);
smb_makescred(&scred, td, cred);
/*
* This is kinda nasty. Since smbfs is physically closing the
* fid on close(), we have to reopen it if necessary. There are
* other races here too, such as if another process opens the same
* file while we are blocked in read, or the file is open read-only
* XXX
*/
error = 0;
doclose = 0;
if (np->n_opencount == 0) {
error = smbfs_smb_open(np, SMB_AM_OPENRW, &scred);
if (error == 0)
doclose = 1;
}
if (error == 0)
error = smb_write(smp->sm_share, np->n_fid, &uio, &scred);
if (doclose)
smbfs_smb_close(smp->sm_share, np->n_fid, NULL, &scred);
/* VOP_CLOSE(vp, FWRITE, cred);*/
SMBVDEBUG("paged write done: %d\n", error);
pmap_qremove(kva, npages);
relpbuf(bp, &smbfs_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(pages[i]);
}
}
return rtvals[0];
#endif /* SMBFS_RWGENERIC */
}
/*
* Vnode op for VM getpages.
* Wish wish .... get rid from multiple IO routines
*
* smbfs_getpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_reqpage, vm_ooffset_t a_offset)
*/
int
smbfs_getpages(struct vop_getpages_args *ap)
{
#ifdef SMBFS_RWGENERIC
return vop_stdgetpages(ap);
#else
int i, error, npages;
int doclose;
size_t size, toff, nextoff, count;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
struct smbmount *smp;
struct smbnode *np;
struct smb_cred scred;
vm_page_t *pages;
KKASSERT(td->td_proc);
vp = ap->a_vp;
cred = td->td_proc->p_ucred;
np = VTOSMB(vp);
smp = VFSTOSMBFS(vp->v_mount);
pages = ap->a_m;
count = (size_t)ap->a_count;
if (vp->v_object == NULL) {
kprintf("smbfs_getpages: called with non-merged cache vnode??\n");
return VM_PAGER_ERROR;
}
smb_makescred(&scred, td, cred);
bp = getpbuf_kva(&smbfs_pbuf_freecnt);
npages = btoc(count);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
/*
* This is kinda nasty. Since smbfs is physically closing the
* fid on close(), we have to reopen it if necessary. There are
* other races here too, such as if another process opens the same
* file while we are blocked in read. XXX
*/
error = 0;
doclose = 0;
if (np->n_opencount == 0) {
error = smbfs_smb_open(np, SMB_AM_OPENREAD, &scred);
if (error == 0)
doclose = 1;
}
if (error == 0)
error = smb_read(smp->sm_share, np->n_fid, &uio, &scred);
if (doclose)
smbfs_smb_close(smp->sm_share, np->n_fid, NULL, &scred);
pmap_qremove(kva, npages);
relpbuf(bp, &smbfs_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
kprintf("smbfs_getpages: error %d\n",error);
for (i = 0; i < npages; i++) {
if (ap->a_reqpage != i)
vnode_pager_freepage(pages[i]);
}
return VM_PAGER_ERROR;
}
size = count - uio.uio_resid;
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: pmap dirty bit should have already been cleared.
* We do not clear it here.
*/
if (nextoff <= size) {
m->valid = VM_PAGE_BITS_ALL;
m->dirty = 0;
} else {
int nvalid = ((size + DEV_BSIZE - 1) - toff) &
~(DEV_BSIZE - 1);
vm_page_set_validclean(m, 0, nvalid);
}
if (i != ap->a_reqpage) {
/*
* Whether or not to leave the page activated is up in
* the air, but we should put the page on a page queue
* somewhere (it already is in the object). Result:
* It appears that emperical results show that
* deactivating pages is best.
*/
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error) {
if (m->flags & PG_REFERENCED)
vm_page_activate(m);
else
vm_page_deactivate(m);
vm_page_wakeup(m);
} else {
vnode_pager_freepage(m);
}
}
}
return 0;
#endif /* SMBFS_RWGENERIC */
}
/* ARGSUSED */
int
memrw(struct cdev *dev, struct uio *uio, int flags)
{
int o;
u_int c = 0, v;
struct iovec *iov;
int error = 0;
vm_offset_t addr, eaddr;
while (uio->uio_resid > 0 && error == 0) {
iov = uio->uio_iov;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
if (uio->uio_iovcnt < 0)
panic("memrw");
continue;
}
if (dev2unit(dev) == CDEV_MINOR_MEM) {
int i;
int address_valid = 0;
v = uio->uio_offset;
v &= ~PAGE_MASK;
for (i = 0; dump_avail[i] || dump_avail[i + 1];
i += 2) {
if (v >= dump_avail[i] &&
v < dump_avail[i + 1]) {
address_valid = 1;
break;
}
}
if (!address_valid)
return (EINVAL);
sx_xlock(&tmppt_lock);
pmap_kenter((vm_offset_t)_tmppt, v);
o = (int)uio->uio_offset & PAGE_MASK;
c = (u_int)(PAGE_SIZE - ((int)iov->iov_base & PAGE_MASK));
c = min(c, (u_int)(PAGE_SIZE - o));
c = min(c, (u_int)iov->iov_len);
error = uiomove((caddr_t)&_tmppt[o], (int)c, uio);
pmap_qremove((vm_offset_t)_tmppt, 1);
sx_xunlock(&tmppt_lock);
continue;
}
else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
c = iov->iov_len;
/*
* Make sure that all of the pages are currently
* resident so that we don't create any zero-fill
* pages.
*/
addr = trunc_page(uio->uio_offset);
eaddr = round_page(uio->uio_offset + c);
for (; addr < eaddr; addr += PAGE_SIZE)
if (pmap_extract(kernel_pmap, addr) == 0)
return (EFAULT);
if (!kernacc((caddr_t)(int)uio->uio_offset, c,
uio->uio_rw == UIO_READ ?
VM_PROT_READ : VM_PROT_WRITE))
return (EFAULT);
error = uiomove((caddr_t)(int)uio->uio_offset, (int)c, uio);
continue;
}
/* else panic! */
}
return (error);
}
int
proc_rwmem(struct proc *p, struct uio *uio)
{
struct vmspace *vm;
vm_map_t map;
vm_object_t object = NULL;
vm_offset_t pageno = 0; /* page number */
vm_prot_t reqprot;
vm_offset_t kva;
int error, writing;
GIANT_REQUIRED;
/*
* if the vmspace is in the midst of being deallocated or the
* process is exiting, don't try to grab anything. The page table
* usage in that process can be messed up.
*/
vm = p->p_vmspace;
if ((p->p_flag & P_WEXIT))
return (EFAULT);
if (vm->vm_refcnt < 1)
return (EFAULT);
++vm->vm_refcnt;
/*
* The map we want...
*/
map = &vm->vm_map;
writing = uio->uio_rw == UIO_WRITE;
reqprot = writing ? (VM_PROT_WRITE | VM_PROT_OVERRIDE_WRITE) :
VM_PROT_READ;
kva = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
/*
* Only map in one page at a time. We don't have to, but it
* makes things easier. This way is trivial - right?
*/
do {
vm_map_t tmap;
vm_offset_t uva;
int page_offset; /* offset into page */
vm_map_entry_t out_entry;
vm_prot_t out_prot;
boolean_t wired;
vm_pindex_t pindex;
u_int len;
vm_page_t m;
object = NULL;
uva = (vm_offset_t)uio->uio_offset;
/*
* Get the page number of this segment.
*/
pageno = trunc_page(uva);
page_offset = uva - pageno;
/*
* How many bytes to copy
*/
len = min(PAGE_SIZE - page_offset, uio->uio_resid);
/*
* Fault the page on behalf of the process
*/
error = vm_fault(map, pageno, reqprot, VM_FAULT_NORMAL);
if (error) {
error = EFAULT;
break;
}
/*
* Now we need to get the page. out_entry, out_prot, wired,
* and single_use aren't used. One would think the vm code
* would be a *bit* nicer... We use tmap because
* vm_map_lookup() can change the map argument.
*/
tmap = map;
error = vm_map_lookup(&tmap, pageno, reqprot, &out_entry,
&object, &pindex, &out_prot, &wired);
if (error) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
break;
}
m = vm_page_lookup(object, pindex);
/* Allow fallback to backing objects if we are reading */
while (m == NULL && !writing && object->backing_object) {
pindex += OFF_TO_IDX(object->backing_object_offset);
object = object->backing_object;
m = vm_page_lookup(object, pindex);
}
if (m == NULL) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
vm_map_lookup_done(tmap, out_entry);
break;
}
/*
* Wire the page into memory
*/
vm_page_lock_queues();
vm_page_wire(m);
vm_page_unlock_queues();
/*
* We're done with tmap now.
* But reference the object first, so that we won't loose
* it.
*/
vm_object_reference(object);
vm_map_lookup_done(tmap, out_entry);
pmap_qenter(kva, &m, 1);
/*
* Now do the i/o move.
*/
error = uiomove((caddr_t)(kva + page_offset), len, uio);
pmap_qremove(kva, 1);
/*
* release the page and the object
*/
vm_page_lock_queues();
vm_page_unwire(m, 1);
vm_page_unlock_queues();
vm_object_deallocate(object);
object = NULL;
} while (error == 0 && uio->uio_resid > 0);
if (object)
vm_object_deallocate(object);
kmem_free(kernel_map, kva, PAGE_SIZE);
vmspace_free(vm);
return (error);
}
/* ARGSUSED */
int
memrw(struct cdev *dev, struct uio *uio, int flags)
{
int o;
u_int c = 0;
vm_paddr_t pa;
struct iovec *iov;
int error = 0;
vm_offset_t addr;
/* XXX UPS Why ? */
GIANT_REQUIRED;
if (dev2unit(dev) != CDEV_MINOR_MEM && dev2unit(dev) != CDEV_MINOR_KMEM)
return EIO;
if (dev2unit(dev) == CDEV_MINOR_KMEM && uio->uio_resid > 0) {
if (uio->uio_offset < (vm_offset_t)VADDR(PTDPTDI, 0))
return (EFAULT);
if (!kernacc((caddr_t)(int)uio->uio_offset, uio->uio_resid,
uio->uio_rw == UIO_READ ? VM_PROT_READ : VM_PROT_WRITE))
return (EFAULT);
}
while (uio->uio_resid > 0 && error == 0) {
iov = uio->uio_iov;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
if (uio->uio_iovcnt < 0)
panic("memrw");
continue;
}
if (dev2unit(dev) == CDEV_MINOR_MEM) {
pa = uio->uio_offset;
pa &= ~PAGE_MASK;
} else {
/*
* Extract the physical page since the mapping may
* change at any time. This avoids panics on page
* fault in this case but will cause reading/writing
* to the wrong page.
* Hopefully an application will notice the wrong
* data on read access and refrain from writing.
* This should be replaced by a special uiomove
* type function that just returns an error if there
* is a page fault on a kernel page.
*/
addr = trunc_page(uio->uio_offset);
pa = pmap_extract(kernel_pmap, addr);
if (pa == 0)
return EFAULT;
}
/*
* XXX UPS This should just use sf_buf_alloc.
* Unfortunately sf_buf_alloc needs a vm_page
* and we may want to look at memory not covered
* by the page array.
*/
sx_xlock(&memsxlock);
pmap_kenter((vm_offset_t)ptvmmap, pa);
pmap_invalidate_page(kernel_pmap,(vm_offset_t)ptvmmap);
o = (int)uio->uio_offset & PAGE_MASK;
c = PAGE_SIZE - o;
c = min(c, (u_int)iov->iov_len);
error = uiomove((caddr_t)&ptvmmap[o], (int)c, uio);
pmap_qremove((vm_offset_t)ptvmmap, 1);
sx_xunlock(&memsxlock);
}
return (error);
}
/*
* Vnode op for VM putpages.
* possible bug: all IO done in sync mode
* Note that vop_close always invalidate pages before close, so it's
* not necessary to open vnode.
*
* nwfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_sync, int *a_rtvals, vm_ooffset_t a_offset)
*/
int
nwfs_putpages(struct vop_putpages_args *ap)
{
int error;
struct thread *td = curthread; /* XXX */
struct vnode *vp = ap->a_vp;
struct ucred *cred;
#ifndef NWFS_RWCACHE
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred; /* XXX */
VOP_OPEN(vp, FWRITE, cred, NULL);
error = vnode_pager_generic_putpages(ap->a_vp, ap->a_m, ap->a_count,
ap->a_sync, ap->a_rtvals);
VOP_CLOSE(vp, FWRITE, cred);
return error;
#else
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, npages, count;
int *rtvals;
struct nwmount *nmp;
struct nwnode *np;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred; /* XXX */
/* VOP_OPEN(vp, FWRITE, cred, NULL);*/
np = VTONW(vp);
nmp = VFSTONWFS(vp->v_mount);
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
for (i = 0; i < npages; i++) {
rtvals[i] = VM_PAGER_AGAIN;
}
bp = getpbuf_kva(&nwfs_pbuf_freecnt);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
NCPVNDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid);
error = ncp_write(NWFSTOCONN(nmp), &np->n_fh, &uio, cred);
/* VOP_CLOSE(vp, FWRITE, cred);*/
NCPVNDEBUG("paged write done: %d\n", error);
pmap_qremove(kva, npages);
relpbuf(bp, &nwfs_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(pages[i]);
}
}
return rtvals[0];
#endif /* NWFS_RWCACHE */
}
static void
cpu_initialize_context(unsigned int cpu)
{
/* vcpu_guest_context_t is too large to allocate on the stack.
* Hence we allocate statically and protect it with a lock */
vm_page_t m[NPGPTD + 2];
static vcpu_guest_context_t ctxt;
vm_offset_t boot_stack;
vm_offset_t newPTD;
vm_paddr_t ma[NPGPTD];
int i;
/*
* Page 0,[0-3] PTD
* Page 1, [4] boot stack
* Page [5] PDPT
*
*/
for (i = 0; i < NPGPTD + 2; i++) {
m[i] = vm_page_alloc(NULL, 0,
VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
VM_ALLOC_ZERO);
pmap_zero_page(m[i]);
}
boot_stack = kmem_alloc_nofault(kernel_map, PAGE_SIZE);
newPTD = kmem_alloc_nofault(kernel_map, NPGPTD * PAGE_SIZE);
ma[0] = VM_PAGE_TO_MACH(m[0])|PG_V;
#ifdef PAE
pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD + 1]));
for (i = 0; i < NPGPTD; i++) {
((vm_paddr_t *)boot_stack)[i] =
ma[i] = VM_PAGE_TO_MACH(m[i])|PG_V;
}
#endif
/*
* Copy cpu0 IdlePTD to new IdlePTD - copying only
* kernel mappings
*/
pmap_qenter(newPTD, m, 4);
memcpy((uint8_t *)newPTD + KPTDI*sizeof(vm_paddr_t),
(uint8_t *)PTOV(IdlePTD) + KPTDI*sizeof(vm_paddr_t),
nkpt*sizeof(vm_paddr_t));
pmap_qremove(newPTD, 4);
kmem_free(kernel_map, newPTD, 4 * PAGE_SIZE);
/*
* map actual idle stack to boot_stack
*/
pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD]));
xen_pgdpt_pin(VM_PAGE_TO_MACH(m[NPGPTD + 1]));
rw_wlock(&pvh_global_lock);
for (i = 0; i < 4; i++) {
int pdir = (PTDPTDI + i) / NPDEPG;
int curoffset = (PTDPTDI + i) % NPDEPG;
xen_queue_pt_update((vm_paddr_t)
((ma[pdir] & ~PG_V) + (curoffset*sizeof(vm_paddr_t))),
ma[i]);
}
PT_UPDATES_FLUSH();
rw_wunlock(&pvh_global_lock);
memset(&ctxt, 0, sizeof(ctxt));
ctxt.flags = VGCF_IN_KERNEL;
ctxt.user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.fs = GSEL(GPRIV_SEL, SEL_KPL);
ctxt.user_regs.gs = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.eip = (unsigned long)init_secondary;
ctxt.user_regs.eflags = PSL_KERNEL | 0x1000; /* IOPL_RING1 */
memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
smp_trap_init(ctxt.trap_ctxt);
ctxt.ldt_ents = 0;
ctxt.gdt_frames[0] = (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
ctxt.gdt_ents = 512;
#ifdef __i386__
ctxt.user_regs.esp = boot_stack + PAGE_SIZE;
ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.kernel_sp = boot_stack + PAGE_SIZE;
ctxt.event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.event_callback_eip = (unsigned long)Xhypervisor_callback;
ctxt.failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
ctxt.ctrlreg[3] = VM_PAGE_TO_MACH(m[NPGPTD + 1]);
#else /* __x86_64__ */
ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.kernel_sp = idle->thread.rsp0;
ctxt.event_callback_eip = (unsigned long)hypervisor_callback;
ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
ctxt.syscall_callback_eip = (unsigned long)system_call;
ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
#endif
printf("gdtpfn=%lx pdptpfn=%lx\n",
ctxt.gdt_frames[0],
ctxt.ctrlreg[3] >> PAGE_SHIFT);
PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, &ctxt));
DELAY(3000);
PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL));
}
/*
struct vnop_getpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_reqpage;
vm_ooffset_t a_offset;
};
*/
static int
fuse_vnop_getpages(struct vop_getpages_args *ap)
{
int i, error, nextoff, size, toff, count, npages;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to getpages"));
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
count = ap->a_count;
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
return (VM_PAGER_ERROR);
}
npages = btoc(count);
/*
* If the requested page is partially valid, just return it and
* allow the pager to zero-out the blanks. Partially valid pages
* can only occur at the file EOF.
*/
VM_OBJECT_WLOCK(vp->v_object);
fuse_vm_page_lock_queues();
if (pages[ap->a_reqpage]->valid != 0) {
for (i = 0; i < npages; ++i) {
if (i != ap->a_reqpage) {
fuse_vm_page_lock(pages[i]);
vm_page_free(pages[i]);
fuse_vm_page_unlock(pages[i]);
}
}
fuse_vm_page_unlock_queues();
VM_OBJECT_WUNLOCK(vp->v_object);
return 0;
}
fuse_vm_page_unlock_queues();
VM_OBJECT_WUNLOCK(vp->v_object);
/*
* We use only the kva address for the buffer, but this is extremely
* convienient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodein);
PCPU_ADD(cnt.v_vnodepgsin, npages);
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
FS_DEBUG("error %d\n", error);
VM_OBJECT_WLOCK(vp->v_object);
fuse_vm_page_lock_queues();
for (i = 0; i < npages; ++i) {
if (i != ap->a_reqpage) {
fuse_vm_page_lock(pages[i]);
vm_page_free(pages[i]);
fuse_vm_page_unlock(pages[i]);
}
}
fuse_vm_page_unlock_queues();
VM_OBJECT_WUNLOCK(vp->v_object);
return VM_PAGER_ERROR;
}
/*
* Calculate the number of bytes read and validate only that number
* of bytes. Note that due to pending writes, size may be 0. This
* does not mean that the remaining data is invalid!
*/
size = count - uio.uio_resid;
VM_OBJECT_WLOCK(vp->v_object);
fuse_vm_page_lock_queues();
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
if (nextoff <= size) {
/*
* Read operation filled an entire page
*/
m->valid = VM_PAGE_BITS_ALL;
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else if (size > toff) {
/*
* Read operation filled a partial page.
*/
m->valid = 0;
vm_page_set_valid_range(m, 0, size - toff);
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else {
/*
* Read operation was short. If no error occured
* we may have hit a zero-fill section. We simply
* leave valid set to 0.
*/
;
}
if (i != ap->a_reqpage)
vm_page_readahead_finish(m);
}
fuse_vm_page_unlock_queues();
VM_OBJECT_WUNLOCK(vp->v_object);
return 0;
}