/*
* copies a uio scatter/gather list to an mbuf chain.
*/
int
mb_put_uio(struct mbchain *mbp, struct uio *uiop, int size)
{
long left;
int mtype, error;
mtype = VMSPACE_IS_KERNEL_P(uiop->uio_vmspace) ? MB_MSYSTEM : MB_MUSER;
while (size > 0 && uiop->uio_resid) {
if (uiop->uio_iovcnt <= 0 || uiop->uio_iov == NULL)
return EFBIG;
left = uiop->uio_iov->iov_len;
if (left == 0) {
uiop->uio_iov++;
uiop->uio_iovcnt--;
continue;
}
if (left > size)
left = size;
error = mb_put_mem(mbp, uiop->uio_iov->iov_base, left, mtype);
if (error)
return error;
uiop->uio_offset += left;
uiop->uio_resid -= left;
uiop->uio_iov->iov_base =
(char *)uiop->uio_iov->iov_base + left;
uiop->uio_iov->iov_len -= left;
size -= left;
}
return 0;
}
/*
* This function does the same as uiomove, but takes an explicit
* direction, and does not update the uio structure.
*/
static int
_bus_dma_uiomove(void *buf, struct uio *uio, size_t n, int direction)
{
struct iovec *iov;
int error;
struct vmspace *vm;
char *cp;
size_t resid, cnt;
int i;
iov = uio->uio_iov;
vm = uio->uio_vmspace;
cp = buf;
resid = n;
for (i = 0; i < uio->uio_iovcnt && resid > 0; i++) {
iov = &uio->uio_iov[i];
if (iov->iov_len == 0)
continue;
cnt = MIN(resid, iov->iov_len);
if (!VMSPACE_IS_KERNEL_P(vm) &&
(curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD)
!= 0) {
preempt();
}
if (direction == UIO_READ) {
error = copyout_vmspace(vm, cp, iov->iov_base, cnt);
} else {
error = copyin_vmspace(vm, iov->iov_base, cp, cnt);
}
if (error)
return (error);
cp += cnt;
resid -= cnt;
}
return (0);
}
/*
* Common function for DMA map synchronization. May be called
* by chipset-specific DMA map synchronization functions.
*
* This version works with the virtually-indexed, write-back cache
* found in the MIPS-3/MIPS-4 CPUs available for the Algorithmics.
*/
void
_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
bus_size_t len, int ops)
{
bus_size_t minlen;
#ifdef DIAGNOSTIC
/*
* Mixing PRE and POST operations is not allowed.
*/
if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
(ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
panic("_bus_dmamap_sync: mix PRE and POST");
if (offset >= map->dm_mapsize)
panic("_bus_dmamap_sync: bad offset %"PRIxPADDR
" (map size is %"PRIxPSIZE")",
offset, map->dm_mapsize);
if (len == 0 || (offset + len) > map->dm_mapsize)
panic("_bus_dmamap_sync: bad length");
#endif
/*
* Since we're dealing with a virtually-indexed, write-back
* cache, we need to do the following things:
*
* PREREAD -- Invalidate D-cache. Note we might have
* to also write-back here if we have to use an Index
* op, or if the buffer start/end is not cache-line aligned.
*
* PREWRITE -- Write-back the D-cache. If we have to use
* an Index op, we also have to invalidate. Note that if
* we are doing PREREAD|PREWRITE, we can collapse everything
* into a single op.
*
* POSTREAD -- Nothing.
*
* POSTWRITE -- Nothing.
*/
#ifdef _MIPS_NEED_BUS_DMA_BOUNCE
struct mips_bus_dma_cookie * const cookie = map->_dm_cookie;
if (cookie != NULL && (cookie->id_flags & _BUS_DMA_IS_BOUNCING)
&& (ops & BUS_DMASYNC_PREWRITE)) {
STAT_INCR(write_bounces);
/*
* Copy the caller's buffer to the bounce buffer.
*/
switch (cookie->id_buftype) {
case _BUS_DMA_BUFTYPE_LINEAR:
memcpy((char *)cookie->id_bouncebuf + offset,
cookie->id_origlinearbuf + offset, len);
break;
case _BUS_DMA_BUFTYPE_MBUF:
m_copydata(cookie->id_origmbuf, offset, len,
(char *)cookie->id_bouncebuf + offset);
break;
case _BUS_DMA_BUFTYPE_UIO:
_bus_dma_uiomove((char *)cookie->id_bouncebuf + offset,
cookie->id_origuio, len, UIO_WRITE);
break;
#ifdef DIAGNOSTIC
case _BUS_DMA_BUFTYPE_RAW:
panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_RAW");
break;
case _BUS_DMA_BUFTYPE_INVALID:
panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_INVALID");
break;
default:
panic("_bus_dmamap_sync: unknown buffer type %d\n",
cookie->id_buftype);
break;
#endif /* DIAGNOSTIC */
}
}
#endif /* _MIPS_NEED_BUS_DMA_BOUNCE */
/*
* Flush the write buffer.
* XXX Is this always necessary?
*/
wbflush();
/*
* If the mapping is of COHERENT DMA-safe memory or this isn't a
* PREREAD or PREWRITE, no cache flush is necessary. Check to see
* if we need to bounce it.
*/
if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)
|| (ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) == 0)
goto bounce_it;
/*
* If the mapping belongs to the kernel, or it belongs
* to the currently-running process (XXX actually, vmspace),
* then we can use Hit ops. Otherwise, Index ops.
*
* This should be true the vast majority of the time.
*/
const bool useindex = (!VMSPACE_IS_KERNEL_P(map->_dm_vmspace)
&& map->_dm_vmspace != curproc->p_vmspace);
bus_dma_segment_t *seg = map->dm_segs;
bus_dma_segment_t * const lastseg = seg + map->dm_nsegs;
/*
* Skip segments until offset are withing a segment.
*/
for (; offset >= seg->ds_len; seg++) {
offset -= seg->ds_len;
}
for (; seg < lastseg && len != 0; seg++, offset = 0, len -= minlen) {
/*
* Now at the first segment to sync; nail each segment until we
* have exhausted the length.
*/
vaddr_t vaddr = seg->_ds_vaddr + offset;
minlen = ulmin(len, seg->ds_len - offset);
#ifdef BUS_DMA_DEBUG
printf("bus_dmamap_sync: flushing segment %p "
"(0x%"PRIxBUSADDR"+%"PRIxBUSADDR
", 0x%"PRIxBUSADDR"+0x%"PRIxBUSADDR
") (olen = %"PRIxBUSADDR")...", seg,
vaddr - offset, offset,
vaddr - offset, offset + minlen - 1, len);
#endif
/*
* If we are forced to use Index ops, it's always a
* Write-back,Invalidate, so just do one test.
*/
if (__predict_false(useindex)) {
mips_dcache_wbinv_range_index(vaddr, minlen);
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
continue;
}
switch (ops) {
case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
mips_dcache_wbinv_range(vaddr, minlen);
break;
case BUS_DMASYNC_PREREAD:
#if 1
mips_dcache_wbinv_range(vaddr, minlen);
#else
mips_dcache_inv_range(vaddr, minlen);
#endif
break;
case BUS_DMASYNC_PREWRITE:
mips_dcache_wb_range(vaddr, minlen);
break;
}
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
}
bounce_it:
#ifdef _MIPS_NEED_BUS_DMA_BOUNCE
if ((ops & BUS_DMASYNC_POSTREAD) == 0
|| cookie == NULL
|| (cookie->id_flags & _BUS_DMA_IS_BOUNCING) == 0)
return;
STAT_INCR(read_bounces);
/*
* Copy the bounce buffer to the caller's buffer.
*/
switch (cookie->id_buftype) {
case _BUS_DMA_BUFTYPE_LINEAR:
memcpy(cookie->id_origlinearbuf + offset,
(char *)cookie->id_bouncebuf + offset, len);
break;
case _BUS_DMA_BUFTYPE_MBUF:
m_copyback(cookie->id_origmbuf, offset, len,
(char *)cookie->id_bouncebuf + offset);
break;
case _BUS_DMA_BUFTYPE_UIO:
_bus_dma_uiomove((char *)cookie->id_bouncebuf + offset,
cookie->id_origuio, len, UIO_READ);
break;
#ifdef DIAGNOSTIC
case _BUS_DMA_BUFTYPE_RAW:
panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_RAW");
break;
case _BUS_DMA_BUFTYPE_INVALID:
panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_INVALID");
break;
default:
panic("_bus_dmamap_sync: unknown buffer type %d\n",
cookie->id_buftype);
break;
#endif
}
#endif /* _MIPS_NEED_BUS_DMA_BOUNCE */
;
}
/*
* Utility function to load a linear buffer. segp contains the starting
* segment on entrance, and the ending segment on exit. first indicates
* if this is the first invocation of this function.
*/
static int
_bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map,
void *buf, bus_size_t buflen, struct vmspace *vm, int flags,
int *segp, bool first)
{
bus_size_t sgsize;
paddr_t baddr, curaddr, lastaddr;
vaddr_t vaddr = (vaddr_t)buf, lastvaddr;
int seg = *segp;
bus_dma_segment_t *ds = &map->dm_segs[seg];
bus_dma_segment_t * const eds = &map->dm_segs[map->_dm_segcnt];
const bool d_cache_coherent =
(mips_options.mips_cpu_flags & CPU_MIPS_D_CACHE_COHERENT) != 0;
lastaddr = ds->ds_addr + ds->ds_len;
lastvaddr = ds->_ds_vaddr + ds->ds_len;
const bus_size_t bmask = ~(map->_dm_boundary - 1);
while (buflen > 0) {
/*
* Get the physical address for this segment.
*/
if (!VMSPACE_IS_KERNEL_P(vm))
(void) pmap_extract(vm_map_pmap(&vm->vm_map), vaddr,
&curaddr);
else
curaddr = kvtophys(vaddr);
/*
* If we're beyond the current DMA window, indicate
* that and try to fall back onto something else.
*/
if (curaddr < t->_bounce_alloc_lo
|| (t->_bounce_alloc_hi != 0
&& curaddr >= t->_bounce_alloc_hi))
return (EINVAL);
#if BUS_DMA_DEBUG
printf("dma: addr %#"PRIxPADDR" -> %#"PRIxPADDR"\n", curaddr,
(curaddr - t->_bounce_alloc_lo) + t->_wbase);
#endif
curaddr = (curaddr - t->_bounce_alloc_lo) + t->_wbase;
/*
* Compute the segment size, and adjust counts.
*/
sgsize = PAGE_SIZE - ((uintptr_t)vaddr & PGOFSET);
if (sgsize > buflen) {
sgsize = buflen;
}
if (sgsize > map->dm_maxsegsz) {
sgsize = map->dm_maxsegsz;
}
/*
* Make sure we don't cross any boundaries.
*/
if (map->_dm_boundary > 0) {
baddr = (curaddr + map->_dm_boundary) & bmask;
if (sgsize > baddr - curaddr) {
sgsize = baddr - curaddr;
}
}
/*
* Insert chunk into a segment, coalescing with
* the previous segment if possible.
*/
if (first) {
ds->ds_addr = curaddr;
ds->ds_len = sgsize;
ds->_ds_vaddr = vaddr;
first = false;
} else if (curaddr == lastaddr
&& (d_cache_coherent || lastvaddr == vaddr)
&& ds->ds_len + sgsize <= map->dm_maxsegsz
&& (map->_dm_boundary == 0
|| ((ds->ds_addr ^ curaddr) & bmask) == 0)) {
ds->ds_len += sgsize;
} else {
if (++ds >= eds)
break;
ds->ds_addr = curaddr;
ds->ds_len = sgsize;
ds->_ds_vaddr = vaddr;
}
lastaddr = curaddr + sgsize;
vaddr += sgsize;
buflen -= sgsize;
lastvaddr = vaddr;
}
*segp = ds - map->dm_segs;
/*
* Did we fit?
*/
if (buflen != 0) {
/*
* If there is a chained window, we will automatically
* fall back to it.
*/
return (EFBIG); /* XXX better return value here? */
}
return (0);
}
/*
* Common function for DMA map synchronization. May be called
* by chipset-specific DMA map synchronization functions.
*
* This is the R4000 version.
*/
void
_bus_dmamap_sync_r4k(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
bus_size_t len, int ops)
{
bus_size_t minlen;
bus_addr_t addr;
int i, useindex;
/*
* Mixing PRE and POST operations is not allowed.
*/
if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
(ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
panic("_bus_dmamap_sync_r4k: mix PRE and POST");
#ifdef DIAGNOSTIC
if (offset >= map->dm_mapsize)
panic("_bus_dmamap_sync_r4k: bad offset %lu (map size is %lu)",
offset, map->dm_mapsize);
if (len == 0 || (offset + len) > map->dm_mapsize)
panic("_bus_dmamap_sync_r4k: bad length");
#endif
/*
* The R4000 cache is virtually-indexed, write-back. This means
* we need to do the following things:
*
* PREREAD -- Invalidate D-cache. Note we might have
* to also write-back here if we have to use an Index
* op, or if the buffer start/end is not cache-line aligned.
*
* PREWRITE -- Write-back the D-cache. If we have to use
* an Index op, we also have to invalidate. Note that if
* we are doing PREREAD|PREWRITE, we can collapse everything
* into a single op.
*
* POSTREAD -- Nothing.
*
* POSTWRITE -- Nothing.
*/
/*
* Flush the write buffer.
* XXX Is this always necessary?
*/
wbflush();
ops &= (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
if (ops == 0)
return;
/*
* If the mapping is of COHERENT DMA-safe memory, no cache
* flush is necessary.
*/
if (map->_dm_flags & NEWSMIPS_DMAMAP_COHERENT)
return;
/*
* If the mapping belongs to the kernel, or if it belongs
* to the currently-running process (XXX actually, vmspace),
* then we can use Hit ops. Otherwise, Index ops.
*
* This should be true the vast majority of the time.
*/
if (__predict_true(VMSPACE_IS_KERNEL_P(map->_dm_vmspace) ||
map->_dm_vmspace == curproc->p_vmspace))
useindex = 0;
else
useindex = 1;
for (i = 0; i < map->dm_nsegs && len != 0; i++) {
/* Find the beginning segment. */
if (offset >= map->dm_segs[i].ds_len) {
offset -= map->dm_segs[i].ds_len;
continue;
}
/*
* Now at the first segment to sync; nail
* each segment until we have exhausted the
* length.
*/
minlen = len < map->dm_segs[i].ds_len - offset ?
len : map->dm_segs[i].ds_len - offset;
addr = map->dm_segs[i]._ds_vaddr;
#ifdef BUS_DMA_DEBUG
printf("bus_dmamap_sync: flushing segment %d "
"(0x%lx..0x%lx) ...", i, addr + offset,
addr + offset + minlen - 1);
#endif
/*
* If we are forced to use Index ops, it's always a
* Write-back,Invalidate, so just do one test.
*/
if (__predict_false(useindex)) {
mips_dcache_wbinv_range_index(addr + offset, minlen);
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
offset = 0;
len -= minlen;
continue;
}
switch (ops) {
case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
mips_dcache_wbinv_range(addr + offset, minlen);
break;
case BUS_DMASYNC_PREREAD:
#if 1
mips_dcache_wbinv_range(addr + offset, minlen);
#else
mips_dcache_inv_range(addr + offset, minlen);
#endif
break;
case BUS_DMASYNC_PREWRITE:
mips_dcache_wb_range(addr + offset, minlen);
break;
}
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
offset = 0;
len -= minlen;
}
}
/*
* Utility function to load a linear buffer. lastaddrp holds state
* between invocations (for multiple-buffer loads). segp contains
* the starting segment on entrance, and the ending segment on exit.
* first indicates if this is the first invocation of this function.
*/
int
_bus_dmamap_load_buffer(bus_dmamap_t map, void *buf, bus_size_t buflen,
struct vmspace *vm, int flags, vaddr_t *lastaddrp, int *segp, int first)
{
bus_size_t sgsize;
bus_addr_t curaddr, lastaddr, baddr, bmask;
vaddr_t vaddr = (vaddr_t)buf;
paddr_t pa;
size_t seg;
lastaddr = *lastaddrp;
bmask = ~(map->_dm_boundary - 1);
for (seg = *segp; buflen > 0 ; ) {
/*
* Get the physical address for this segment.
*/
if (!VMSPACE_IS_KERNEL_P(vm))
(void) pmap_extract(vm_map_pmap(&vm->vm_map),
vaddr, &pa);
else
pa = kvtophys(vaddr);
curaddr = pa;
/*
* Compute the segment size, and adjust counts.
*/
sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
if (buflen < sgsize)
sgsize = buflen;
/*
* Make sure we don't cross any boundaries.
*/
if (map->_dm_boundary > 0) {
baddr = (curaddr + map->_dm_boundary) & bmask;
if (sgsize > (baddr - curaddr))
sgsize = (baddr - curaddr);
}
/*
* Insert chunk into a segment, coalescing with
* the previous segment if possible.
*/
if (first) {
map->dm_segs[seg].ds_addr = curaddr;
map->dm_segs[seg].ds_len = sgsize;
map->dm_segs[seg]._ds_vaddr = vaddr;
first = 0;
} else {
if (curaddr == lastaddr &&
(map->dm_segs[seg].ds_len + sgsize) <=
map->dm_maxsegsz &&
(map->_dm_boundary == 0 ||
(map->dm_segs[seg].ds_addr & bmask) ==
(curaddr & bmask)))
map->dm_segs[seg].ds_len += sgsize;
else {
if (++seg >= map->_dm_segcnt)
break;
map->dm_segs[seg].ds_addr = curaddr;
map->dm_segs[seg].ds_len = sgsize;
map->dm_segs[seg]._ds_vaddr = vaddr;
}
}
lastaddr = curaddr + sgsize;
vaddr += sgsize;
buflen -= sgsize;
}
*segp = seg;
*lastaddrp = lastaddr;
/*
* Did we fit?
*/
if (buflen != 0)
return EFBIG; /* XXX Better return value here? */
return 0;
}
int
cd9660_readlink(void *v)
{
struct vop_readlink_args /* {
struct vnode *a_vp;
struct uio *a_uio;
kauth_cred_t a_cred;
} */ *ap = v;
ISONODE *ip;
ISODIR *dirp;
ISOMNT *imp;
struct buf *bp;
struct uio *uio;
u_short symlen;
int error;
char *symname;
bool use_pnbuf;
ip = VTOI(ap->a_vp);
imp = ip->i_mnt;
uio = ap->a_uio;
if (imp->iso_ftype != ISO_FTYPE_RRIP)
return (EINVAL);
/*
* Get parents directory record block that this inode included.
*/
error = bread(imp->im_devvp,
(ip->i_number >> imp->im_bshift) <<
(imp->im_bshift - DEV_BSHIFT),
imp->logical_block_size, NOCRED, 0, &bp);
if (error) {
return (EINVAL);
}
/*
* Setup the directory pointer for this inode
*/
dirp = (ISODIR *)((char *)bp->b_data + (ip->i_number & imp->im_bmask));
/*
* Just make sure, we have a right one....
* 1: Check not cross boundary on block
*/
if ((ip->i_number & imp->im_bmask) + isonum_711(dirp->length)
> imp->logical_block_size) {
brelse(bp, 0);
return (EINVAL);
}
/*
* Now get a buffer
* Abuse a namei buffer for now.
*/
use_pnbuf = !VMSPACE_IS_KERNEL_P(uio->uio_vmspace) ||
uio->uio_iov->iov_len < MAXPATHLEN;
if (use_pnbuf) {
symname = PNBUF_GET();
} else {
symname = uio->uio_iov->iov_base;
}
/*
* Ok, we just gathering a symbolic name in SL record.
*/
if (cd9660_rrip_getsymname(dirp, symname, &symlen, imp) == 0) {
if (use_pnbuf) {
PNBUF_PUT(symname);
}
brelse(bp, 0);
return (EINVAL);
}
/*
* Don't forget before you leave from home ;-)
*/
brelse(bp, 0);
/*
* return with the symbolic name to caller's.
*/
if (use_pnbuf) {
error = uiomove(symname, symlen, uio);
PNBUF_PUT(symname);
return (error);
}
uio->uio_resid -= symlen;
uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + symlen;
uio->uio_iov->iov_len -= symlen;
return (0);
}
/*
* Common function for DMA map synchronization. May be called
* by chipset-specific DMA map synchronization functions.
*/
void
_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
bus_size_t len, int ops)
{
bus_size_t minlen;
vaddr_t vaddr, start, end, preboundary, firstboundary, lastboundary;
int i, useindex;
/*
* Mixing PRE and POST operations is not allowed.
*/
if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
(ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
panic("_bus_dmamap_sync: mix PRE and POST");
#ifdef DIAGNOSTIC
if (offset >= map->dm_mapsize)
panic("_bus_dmamap_sync: bad offset %lu (map size is %lu)",
offset, map->dm_mapsize);
if (len == 0 || (offset + len) > map->dm_mapsize)
panic("_bus_dmamap_sync: bad length");
#endif
/*
* Since we're dealing with a virtually-indexed, write-back
* cache, we need to do the following things:
*
* PREREAD -- Invalidate D-cache. Note we might have
* to also write-back here if we have to use an Index
* op, or if the buffer start/end is not cache-line aligned.
*
* PREWRITE -- Write-back the D-cache. If we have to use
* an Index op, we also have to invalidate. Note that if
* we are doing PREREAD|PREWRITE, we can collapse everything
* into a single op.
*
* POSTREAD -- Nothing.
*
* POSTWRITE -- Nothing.
*/
/*
* Flush the write buffer.
*/
wbflush();
/*
* If the mapping is of COHERENT DMA-safe memory, no cache
* flush is necessary.
*/
if (map->_dm_flags & EWS4800MIPS_DMAMAP_COHERENT)
return;
/*
* No cache flushes are necessary if we're only doing
* POSTREAD or POSTWRITE (i.e. not doing PREREAD or PREWRITE).
*/
if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) == 0)
return;
/*
* If the mapping belongs to the kernel, or it belongs
* to the currently-running process (XXX actually, vmspace),
* then we can use Hit ops. Otherwise, Index ops.
*
* This should be true the vast majority of the time.
*/
if (__predict_true(VMSPACE_IS_KERNEL_P(map->_dm_vmspace) ||
map->_dm_vmspace == curproc->p_vmspace))
useindex = 0;
else
useindex = 1;
for (i = 0; i < map->dm_nsegs && len != 0; i++) {
/* Find the beginning segment. */
if (offset >= map->dm_segs[i].ds_len) {
offset -= map->dm_segs[i].ds_len;
continue;
}
/*
* Now at the first segment to sync; nail
* each segment until we have exhausted the
* length.
*/
minlen = len < map->dm_segs[i].ds_len - offset ?
len : map->dm_segs[i].ds_len - offset;
vaddr = map->dm_segs[i]._ds_vaddr;
#ifdef BUS_DMA_DEBUG
printf("bus_dmamap_sync: flushing segment %d "
"(0x%lx+%lx, 0x%lx+0x%lx) (olen = %ld)...", i,
vaddr, offset, vaddr, offset + minlen - 1, len);
#endif
/*
* If we are forced to use Index ops, it's always a
* Write-back,Invalidate, so just do one test.
*/
if (__predict_false(useindex)) {
mips_dcache_wbinv_range_index(vaddr + offset, minlen);
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
offset = 0;
len -= minlen;
continue;
}
start = vaddr + offset;
switch (ops) {
case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
mips_dcache_wbinv_range(start, minlen);
break;
case BUS_DMASYNC_PREREAD: {
const struct mips_cache_info * const mci = &mips_cache_info;
end = start + minlen;
preboundary = start & ~mci->mci_dcache_align_mask;
firstboundary = (start + mci->mci_dcache_align_mask)
& ~mci->mci_dcache_align_mask;
lastboundary = end & ~mci->mci_dcache_align_mask;
if (preboundary < start && preboundary < lastboundary)
mips_dcache_wbinv_range(preboundary,
mci->mci_dcache_align);
if (firstboundary < lastboundary)
mips_dcache_inv_range(firstboundary,
lastboundary - firstboundary);
if (lastboundary < end)
mips_dcache_wbinv_range(lastboundary,
mci->mci_dcache_align);
break;
}
case BUS_DMASYNC_PREWRITE:
mips_dcache_wb_range(start, minlen);
break;
}
#ifdef BUS_DMA_DEBUG
printf("\n");
#endif
offset = 0;
len -= minlen;
}
}
/*
* Utility function to load a linear buffer. lastaddrp holds state
* between invocations (for multiple-buffer loads). segp contains
* the starting segment on entrance, and the ending segment on exit.
* first indicates if this is the first invocation of this function.
*/
int
_bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map,
void *buf, bus_size_t buflen, struct vmspace *vm, int flags,
paddr_t *lastaddrp, int *segp, int first)
{
bus_size_t sgsize;
bus_addr_t curaddr, lastaddr, baddr, bmask;
vaddr_t vaddr = (vaddr_t)buf;
int seg;
// printf("%s(%p,%p,%p,%u,%p,%#x,%p,%p,%u)\n", __func__,
// t, map, buf, buflen, vm, flags, lastaddrp, segp, first);
lastaddr = *lastaddrp;
bmask = ~(map->_dm_boundary - 1);
for (seg = *segp; buflen > 0 ; ) {
/*
* Get the physical address for this segment.
*/
if (!VMSPACE_IS_KERNEL_P(vm))
(void) pmap_extract(vm_map_pmap(&vm->vm_map),
vaddr, (void *)&curaddr);
else
curaddr = vtophys(vaddr);
/*
* If we're beyond the bounce threshold, notify
* the caller.
*/
if (map->_dm_bounce_thresh != 0 &&
curaddr >= map->_dm_bounce_thresh)
return (EINVAL);
/*
* Compute the segment size, and adjust counts.
*/
sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
if (buflen < sgsize)
sgsize = buflen;
sgsize = min(sgsize, map->dm_maxsegsz);
/*
* Make sure we don't cross any boundaries.
*/
if (map->_dm_boundary > 0) {
baddr = (curaddr + map->_dm_boundary) & bmask;
if (sgsize > (baddr - curaddr))
sgsize = (baddr - curaddr);
}
/*
* Insert chunk into a segment, coalescing with
* the previous segment if possible.
*/
if (first) {
map->dm_segs[seg].ds_addr
= rumpcomp_pci_virt_to_mach((void *)curaddr);
map->dm_segs[seg].ds_len = sgsize;
first = 0;
} else {
if (curaddr == lastaddr &&
(map->dm_segs[seg].ds_len + sgsize) <=
map->dm_maxsegsz &&
(map->_dm_boundary == 0 ||
(map->dm_segs[seg].ds_addr & bmask) ==
(rumpcomp_pci_virt_to_mach((void*)curaddr)&bmask)))
map->dm_segs[seg].ds_len += sgsize;
else {
if (++seg >= map->_dm_segcnt)
break;
map->dm_segs[seg].ds_addr =
rumpcomp_pci_virt_to_mach((void *)curaddr);
map->dm_segs[seg].ds_len = sgsize;
}
}
lastaddr = curaddr + sgsize;
vaddr += sgsize;
buflen -= sgsize;
}
*segp = seg;
*lastaddrp = lastaddr;
/*
* Did we fit?
*/
if (buflen != 0)
return (EFBIG); /* XXX better return value here? */
return (0);
}
/*
* Utility function to load a linear buffer. lastaddrp holds state
* between invocations (for multiple-buffer loads). segp contains
* the starting segment on entrance, and the ending segment on exit.
* first indicates if this is the first invocation of this function.
*/
int
_bus_dmamap_load_buffer_direct_common(bus_dma_tag_t t, bus_dmamap_t map,
void *buf, bus_size_t buflen, struct vmspace *vm, int flags,
paddr_t *lastaddrp, int *segp, int first)
{
bus_size_t sgsize;
bus_addr_t curaddr, lastaddr, baddr, bmask;
vaddr_t vaddr = (vaddr_t)buf;
int seg, cacheable, coherent;
pmap_t pmap;
bool rv;
coherent = BUS_DMA_COHERENT;
lastaddr = *lastaddrp;
bmask = ~(map->_dm_boundary - 1);
if (!VMSPACE_IS_KERNEL_P(vm))
pmap = vm_map_pmap(&vm->vm_map);
else
pmap = pmap_kernel();
for (seg = *segp; buflen > 0 ; ) {
/*
* Get the physical address for this segment.
*/
rv = pmap_extract(pmap, vaddr, &curaddr);
KASSERT(rv);
cacheable = _pmap_page_is_cacheable(pmap, vaddr);
if (cacheable)
coherent = 0;
/*
* Compute the segment size, and adjust counts.
*/
sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
if (buflen < sgsize)
sgsize = buflen;
/*
* Make sure we don't cross any boundaries.
*/
if (map->_dm_boundary > 0) {
baddr = (curaddr + map->_dm_boundary) & bmask;
if (sgsize > (baddr - curaddr))
sgsize = (baddr - curaddr);
}
/*
* Insert chunk into a segment, coalescing with
* the previous segment if possible.
*/
if (first) {
map->dm_segs[seg].ds_addr = curaddr;
map->dm_segs[seg].ds_len = sgsize;
map->dm_segs[seg]._ds_flags =
cacheable ? 0 : BUS_DMA_COHERENT;
first = 0;
} else {
if (curaddr == lastaddr &&
(map->dm_segs[seg].ds_len + sgsize) <=
map->dm_maxsegsz &&
(map->_dm_boundary == 0 ||
(map->dm_segs[seg].ds_addr & bmask) ==
(curaddr & bmask)))
map->dm_segs[seg].ds_len += sgsize;
else {
if (++seg >= map->_dm_segcnt)
break;
map->dm_segs[seg].ds_addr = curaddr;
map->dm_segs[seg].ds_len = sgsize;
map->dm_segs[seg]._ds_flags =
cacheable ? 0 : BUS_DMA_COHERENT;
}
}
lastaddr = curaddr + sgsize;
vaddr += sgsize;
buflen -= sgsize;
}
*segp = seg;
*lastaddrp = lastaddr;
map->_dm_flags &= ~BUS_DMA_COHERENT;
/* BUS_DMA_COHERENT is set only if all segments are uncached */
map->_dm_flags |= coherent;
/*
* Did we fit?
*/
if (buflen != 0) {
/*
* If there is a chained window, we will automatically
* fall back to it.
*/
return EFBIG; /* XXX better return value here? */
}
return 0;
}
/*
* Move data described by a struct uio.
* The uiomove function copies up to n bytes between the kernel-space
* address pointed to by buf and the addresses described by uio, which may
* be in user-space or kernel-space.
*/
int
uiomove(void *buf, size_t n, struct uio *uio)
{
#ifndef T2EX
struct vmspace *vm = uio->uio_vmspace;
#endif
struct iovec *iov;
size_t cnt;
int error = 0;
char *cp = buf;
ASSERT_SLEEPABLE();
#ifdef DIAGNOSTIC
if (uio->uio_rw != UIO_READ && uio->uio_rw != UIO_WRITE)
panic("uiomove: mode");
#endif
while (n > 0 && uio->uio_resid) {
iov = uio->uio_iov;
cnt = iov->iov_len;
if (cnt == 0) {
KASSERT(uio->uio_iovcnt > 0);
uio->uio_iov++;
uio->uio_iovcnt--;
continue;
}
if (cnt > n)
cnt = n;
#ifndef T2EX
if (!VMSPACE_IS_KERNEL_P(vm)) {
if (curcpu()->ci_schedstate.spc_flags &
SPCF_SHOULDYIELD)
preempt();
}
if (uio->uio_rw == UIO_READ) {
error = copyout_vmspace(vm, cp, iov->iov_base,
cnt);
} else {
error = copyin_vmspace(vm, iov->iov_base, cp,
cnt);
}
#else
if (uio->uio_rw == UIO_READ) {
error = copyout(cp, iov->iov_base, cnt);
} else {
error = copyin(iov->iov_base, cp, cnt);
}
#endif
if (error) {
break;
}
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
uio->uio_offset += cnt;
cp += cnt;
KDASSERT(cnt <= n);
n -= cnt;
}
return (error);
}