/*
* Common function for unmapping DMA-safe memory. May be called by
* bus-specific DMA memory unmapping functions.
*/
void
_bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
{
#ifdef DIAGNOSTIC
if ((uintptr_t)kva & PGOFSET)
panic("_bus_dmamem_unmap: bad alignment on %p", kva);
#endif
/*
* Nothing to do if we mapped it with KSEG0 or KSEG1 (i.e.
* not in KSEG2 or XKSEG).
*/
if (MIPS_KSEG0_P(kva) || MIPS_KSEG1_P(kva))
return;
#ifdef _LP64
if (MIPS_XKPHYS_P((vaddr_t)kva))
return;
#endif
size = round_page(size);
pmap_remove(pmap_kernel(), (vaddr_t)kva, (vaddr_t)kva + size);
pmap_update(pmap_kernel());
uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
}
static void *
__BS(vaddr)(void *v, bus_space_handle_t bsh)
{
#if (CHIP_ALIGN_STRIDE != 0)
/* Linear mappings not possible. */
return (NULL);
#elif defined(__mips_n32)
if (MIPS_KSEG0_P(bsh) || MIPS_KSEG1_P(bsh) || MIPS_KSEG2_P(bsh))
return ((void *)(intptr_t)bsh);
return NULL;
#else
return ((void *)bsh);
#endif
}
/*
* Map a (kernel) virtual address to a physical address.
*
* MIPS processor has 3 distinct kernel address ranges:
*
* - kseg0 kernel "virtual address" for the cached physical address space.
* - kseg1 kernel "virtual address" for the uncached physical address space.
* - kseg2 normal kernel "virtual address" mapped via the TLB.
*/
paddr_t
kvtophys(vaddr_t kva)
{
pt_entry_t *pte;
paddr_t phys;
if (kva >= VM_MIN_KERNEL_ADDRESS) {
if (kva >= VM_MAX_KERNEL_ADDRESS)
goto overrun;
pte = kvtopte(kva);
if ((size_t) (pte - Sysmap) >= Sysmapsize) {
printf("oops: Sysmap overrun, max %d index %zd\n",
Sysmapsize, pte - Sysmap);
}
if (!mips_pg_v(pte->pt_entry)) {
printf("kvtophys: pte not valid for %#"PRIxVADDR"\n",
kva);
}
phys = mips_tlbpfn_to_paddr(pte->pt_entry) | (kva & PGOFSET);
return phys;
}
if (MIPS_KSEG1_P(kva))
return MIPS_KSEG1_TO_PHYS(kva);
if (MIPS_KSEG0_P(kva))
return MIPS_KSEG0_TO_PHYS(kva);
#ifdef _LP64
if (MIPS_XKPHYS_P(kva))
return MIPS_XKPHYS_TO_PHYS(kva);
#endif
overrun:
printf("Virtual address %#"PRIxVADDR": cannot map to physical\n", kva);
#ifdef DDB
Debugger();
return 0; /* XXX */
#endif
panic("kvtophys");
}
/*
* Map a (kernel) virtual address to a physical address.
*
* MIPS processor has 3 distinct kernel address ranges:
*
* - kseg0 kernel "virtual address" for the cached physical address space.
* - kseg1 kernel "virtual address" for the uncached physical address space.
* - kseg2 normal kernel "virtual address" mapped via the TLB.
*/
paddr_t
kvtophys(vaddr_t kva)
{
paddr_t phys;
if (MIPS_KSEG1_P(kva))
return MIPS_KSEG1_TO_PHYS(kva);
if (MIPS_KSEG0_P(kva))
return MIPS_KSEG0_TO_PHYS(kva);
if (kva >= VM_MIN_KERNEL_ADDRESS) {
if (kva >= VM_MAX_KERNEL_ADDRESS)
goto overrun;
pt_entry_t * const ptep = pmap_pte_lookup(pmap_kernel(), kva);
if (ptep == NULL)
goto overrun;
if (!pte_valid_p(*ptep)) {
printf("kvtophys: pte not valid for %#"PRIxVADDR"\n",
kva);
}
phys = pte_to_paddr(*ptep) | (kva & PGOFSET);
return phys;
}
#ifdef _LP64
if (MIPS_XKPHYS_P(kva))
return MIPS_XKPHYS_TO_PHYS(kva);
#endif
overrun:
printf("Virtual address %#"PRIxVADDR": cannot map to physical\n", kva);
#ifdef DDB
Debugger();
return 0; /* XXX */
#endif
panic("kvtophys");
}
/*
* Common function for loading a direct-mapped DMA map with a linear
* buffer. Called by bus-specific DMA map load functions with the
* OR value appropriate for indicating "direct-mapped" for that
* chipset.
*/
int
_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
bus_size_t buflen, struct proc *p, int flags)
{
int seg, error;
struct vmspace *vm;
if (map->dm_nsegs > 0) {
#ifdef _MIPS_NEED_BUS_DMA_BOUNCE
struct mips_bus_dma_cookie *cookie = map->_dm_cookie;
if (cookie != NULL) {
if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) {
STAT_INCR(bounced_unloads);
cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING;
}
cookie->id_buftype = _BUS_DMA_BUFTYPE_INVALID;
} else
#endif
STAT_INCR(unloads);
}
/*
* Make sure that on error condition we return "no valid mappings".
*/
map->dm_mapsize = 0;
map->dm_nsegs = 0;
KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
if (buflen > map->_dm_size)
return (EINVAL);
if (p != NULL) {
vm = p->p_vmspace;
} else {
vm = vmspace_kernel();
}
seg = 0;
error = _bus_dmamap_load_buffer(t, map, buf, buflen,
vm, flags, &seg, true);
if (error == 0) {
map->dm_mapsize = buflen;
map->dm_nsegs = seg + 1;
map->_dm_vmspace = vm;
STAT_INCR(loads);
/*
* For linear buffers, we support marking the mapping
* as COHERENT.
*
* XXX Check TLB entries for cache-inhibit bits?
*/
if (mips_options.mips_cpu_flags & CPU_MIPS_D_CACHE_COHERENT)
map->_dm_flags |= _BUS_DMAMAP_COHERENT;
else if (MIPS_KSEG1_P(buf))
map->_dm_flags |= _BUS_DMAMAP_COHERENT;
#ifdef _LP64
else if (MIPS_XKPHYS_P((vaddr_t)buf)
&& MIPS_XKPHYS_TO_CCA((vaddr_t)buf) == MIPS3_PG_TO_CCA(MIPS3_PG_UNCACHED))
map->_dm_flags |= _BUS_DMAMAP_COHERENT;
#endif
return 0;
}
#ifdef _MIPS_NEED_BUS_DMA_BOUNCE
struct mips_bus_dma_cookie *cookie = map->_dm_cookie;
if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) {
error = _bus_dma_load_bouncebuf(t, map, buf, buflen,
_BUS_DMA_BUFTYPE_LINEAR, flags);
}
#endif
return (error);
}
static void
__BS(unmap)(void *v, bus_space_handle_t h, bus_size_t size, int acct)
{
#if !defined(_LP64) || defined(CHIP_EXTENT)
bus_addr_t addr = 0; /* initialize to appease gcc */
#endif
#ifndef _LP64
bool handle_is_km;
/* determine if h is addr obtained from uvm_km_alloc */
handle_is_km = !(MIPS_KSEG0_P(h) || MIPS_KSEG1_P(h));
#ifdef __mips_n32
if (handle_is_km == true)
handle_is_km = !MIPS_XKPHYS_P(h);
#endif
if (handle_is_km == true) {
paddr_t pa;
vaddr_t va = (vaddr_t)trunc_page(h);
vsize_t sz = (vsize_t)round_page((h % PAGE_SIZE) + size);
int s;
s = splhigh();
if (pmap_extract(pmap_kernel(), (vaddr_t)h, &pa) == false)
panic("%s: pmap_extract failed", __func__);
addr = (bus_addr_t)pa;
#if 0
printf("%s:%d: addr %#"PRIxBUSADDR", sz %#"PRIxVSIZE"\n",
__func__, __LINE__, addr, sz);
#endif
/* sanity check: this is why we couldn't map w/ kseg[0,1] */
KASSERT (((addr + sz) & ~MIPS_PHYS_MASK) != 0);
pmap_kremove(va, sz);
pmap_update(pmap_kernel());
uvm_km_free(kernel_map, va, sz, UVM_KMF_VAONLY);
splx(s);
}
#endif /* _LP64 */
#ifdef CHIP_EXTENT
if (acct == 0)
return;
#ifdef EXTENT_DEBUG
printf("%s: freeing handle %#"PRIxBSH" for %#"PRIxBUSSIZE"\n",
__S(__BS(unmap)), h, size);
#endif
#ifdef _LP64
KASSERT(MIPS_XKPHYS_P(h));
addr = MIPS_XKPHYS_TO_PHYS(h);
#else
if (handle_is_km == false) {
if (MIPS_KSEG0_P(h))
addr = MIPS_KSEG0_TO_PHYS(h);
#ifdef __mips_n32
else if (MIPS_XKPHYS_P(h))
addr = MIPS_XKPHYS_TO_PHYS(h);
#endif
else
addr = MIPS_KSEG1_TO_PHYS(h);
}
#endif
#ifdef CHIP_W1_BUS_START
if (addr >= CHIP_W1_SYS_START(v) && addr <= CHIP_W1_SYS_END(v)) {
addr = CHIP_W1_BUS_START(v) + (addr - CHIP_W1_SYS_START(v));
} else
#endif
#ifdef CHIP_W2_BUS_START
if (addr >= CHIP_W2_SYS_START(v) && addr <= CHIP_W2_SYS_END(v)) {
addr = CHIP_W2_BUS_START(v) + (addr - CHIP_W2_SYS_START(v));
} else
#endif
#ifdef CHIP_W3_BUS_START
if (addr >= CHIP_W3_SYS_START(v) && addr <= CHIP_W3_SYS_END(v)) {
addr = CHIP_W3_BUS_START(v) + (addr - CHIP_W3_SYS_START(v));
} else
#endif
{
printf("\n");
#ifdef CHIP_W1_BUS_START
printf("%s: sys window[1]=0x%lx-0x%lx\n",
__S(__BS(unmap)), (u_long)CHIP_W1_SYS_START(v),
(u_long)CHIP_W1_SYS_END(v));
#endif
#ifdef CHIP_W2_BUS_START
printf("%s: sys window[2]=0x%lx-0x%lx\n",
__S(__BS(unmap)), (u_long)CHIP_W2_SYS_START(v),
(u_long)CHIP_W2_SYS_END(v));
#endif
#ifdef CHIP_W3_BUS_START
printf("%s: sys window[3]=0x%lx-0x%lx\n",
__S(__BS(unmap)), (u_long)CHIP_W3_SYS_START(v),
(u_long)CHIP_W3_SYS_END(v));
#endif
panic("%s: don't know how to unmap %#"PRIxBSH, __S(__BS(unmap)), h);
}
#ifdef EXTENT_DEBUG
printf("%s: freeing %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
__S(__BS(unmap)), addr, addr + size - 1);
#endif
int error = extent_free(CHIP_EXTENT(v), addr, size,
EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0));
if (error) {
printf("%s: WARNING: could not unmap"
" %#"PRIxBUSADDR"-%#"PRIxBUSADDR" (error %d)\n",
__S(__BS(unmap)), addr, addr + size - 1, error);
#ifdef EXTENT_DEBUG
extent_print(CHIP_EXTENT(v));
#endif
}
#endif /* CHIP_EXTENT */
#if !defined(_LP64) || defined(CHIP_EXTENT)
__USE(addr);
#endif
}
static int
__BS(map)(void *v, bus_addr_t addr, bus_size_t size, int flags,
bus_space_handle_t *hp, int acct)
{
struct mips_bus_space_translation mbst;
int error;
/*
* Get the translation for this address.
*/
error = __BS(translate)(v, addr, size, flags, &mbst);
if (error)
return (error);
#ifdef CHIP_EXTENT
if (acct == 0)
goto mapit;
#ifdef EXTENT_DEBUG
printf("%s: allocating %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
__S(__BS(map)), addr, addr + size - 1);
#endif
error = extent_alloc_region(CHIP_EXTENT(v), addr, size,
EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0));
if (error) {
#ifdef EXTENT_DEBUG
printf("%s: allocation failed (%d)\n", __S(__BS(map)), error);
extent_print(CHIP_EXTENT(v));
#endif
return (error);
}
mapit:
#endif /* CHIP_EXTENT */
addr = mbst.mbst_sys_start + (addr - mbst.mbst_bus_start);
#if defined(__mips_n32) || defined(_LP64)
if (flags & BUS_SPACE_MAP_CACHEABLE) {
#ifdef __mips_n32
if (((addr + size) & ~MIPS_PHYS_MASK) == 0)
*hp = (intptr_t)MIPS_PHYS_TO_KSEG0(addr);
else
#endif
*hp = MIPS_PHYS_TO_XKPHYS_CACHED(addr);
} else if (flags & BUS_SPACE_MAP_PREFETCHABLE) {
*hp = MIPS_PHYS_TO_XKPHYS_ACC(addr);
} else {
#ifdef __mips_n32
if (((addr + size) & ~MIPS_PHYS_MASK) == 0)
*hp = (intptr_t)MIPS_PHYS_TO_KSEG1(addr);
else
#endif
*hp = MIPS_PHYS_TO_XKPHYS_UNCACHED(addr);
}
#else
if (((addr + size) & ~MIPS_PHYS_MASK) != 0) {
vaddr_t va;
paddr_t pa;
int s;
size = round_page((addr % PAGE_SIZE) + size);
va = uvm_km_alloc(kernel_map, size, PAGE_SIZE,
UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
if (va == 0)
return ENOMEM;
/* check use of handle_is_km in BS(unmap) */
KASSERT(!(MIPS_KSEG0_P(va) || MIPS_KSEG1_P(va)));
*hp = va + (addr & PAGE_MASK);
pa = trunc_page(addr);
s = splhigh();
while (size != 0) {
pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 0);
pa += PAGE_SIZE;
va += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(pmap_kernel());
splx(s);
} else {
if (flags & BUS_SPACE_MAP_CACHEABLE)
*hp = (intptr_t)MIPS_PHYS_TO_KSEG0(addr);
else
*hp = (intptr_t)MIPS_PHYS_TO_KSEG1(addr);
}
#endif
return (0);
}