void *
uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
vm_page_t m;
vm_paddr_t pa;
void *va;
int pflags;
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
VM_WAIT;
} else
break;
}
pa = m->phys_addr;
if ((wait & M_NODUMP) == 0)
dump_add_page(pa);
va = (void *)PHYS_TO_DMAP(pa);
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
pagezero(va);
return (va);
}
void *
uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
{
vm_paddr_t pa;
vm_page_t m;
int pflags;
void *va;
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
pmap_grow_direct_page_cache();
} else
break;
}
pa = VM_PAGE_TO_PHYS(m);
va = (void *)MIPS_PHYS_TO_DIRECT(pa);
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero(va, PAGE_SIZE);
return (va);
}
void *
uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
void *va;
vm_page_t m;
int pflags;
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
VM_WAIT;
} else
break;
}
va = (void *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero(va, PAGE_SIZE);
return (va);
}
void *
uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
void *va;
vm_page_t m;
int pflags;
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
VM_WAIT;
} else
break;
}
va = (void *) VM_PAGE_TO_PHYS(m);
if (!hw_direct_map)
pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m));
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero(va, PAGE_SIZE);
atomic_add_int(&hw_uma_mdpages, 1);
return (va);
}
/*
* Allocates a region from the kernel address map and physically
* contiguous pages within the specified address range to the kernel
* object. Creates a wired mapping from this region to these pages, and
* returns the region's starting virtual address. If M_ZERO is specified
* through the given flags, then the pages are zeroed before they are
* mapped.
*/
vm_offset_t
kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
vm_memattr_t memattr)
{
vmem_t *vmem;
vm_object_t object = kernel_object;
vm_offset_t addr, offset, tmp;
vm_page_t end_m, m;
u_long npages;
int pflags, tries;
size = round_page(size);
vmem = vm_dom[domain].vmd_kernel_arena;
if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr))
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
pflags |= VM_ALLOC_NOWAIT;
npages = atop(size);
VM_OBJECT_WLOCK(object);
tries = 0;
retry:
m = vm_page_alloc_contig_domain(object, atop(offset), domain, pflags,
npages, low, high, alignment, boundary, memattr);
if (m == NULL) {
VM_OBJECT_WUNLOCK(object);
if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
if (!vm_page_reclaim_contig_domain(domain, pflags,
npages, low, high, alignment, boundary) &&
(flags & M_WAITOK) != 0)
vm_wait_domain(domain);
VM_OBJECT_WLOCK(object);
tries++;
goto retry;
}
vmem_free(vmem, addr, size);
return (0);
}
KASSERT(vm_phys_domain(m) == domain,
("kmem_alloc_contig_domain: Domain mismatch %d != %d",
vm_phys_domain(m), domain));
end_m = m + npages;
tmp = addr;
for (; m < end_m; m++) {
if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
m->valid = VM_PAGE_BITS_ALL;
pmap_enter(kernel_pmap, tmp, m, VM_PROT_ALL,
VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
tmp += PAGE_SIZE;
}
VM_OBJECT_WUNLOCK(object);
return (addr);
}
void *
uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
void *ret;
struct arm_small_page *sp;
TAILQ_HEAD(,arm_small_page) *head;
vm_page_t m;
*flags = UMA_SLAB_PRIV;
/*
* For CPUs where we setup page tables as write back, there's no
* need to maintain two separate pools.
*/
if (zone == l2zone && pte_l1_s_cache_mode != pte_l1_s_cache_mode_pt)
head = (void *)&pages_wt;
else
head = (void *)&pages_normal;
mtx_lock(&smallalloc_mtx);
sp = TAILQ_FIRST(head);
if (!sp) {
int pflags;
mtx_unlock(&smallalloc_mtx);
if (zone == l2zone &&
pte_l1_s_cache_mode != pte_l1_s_cache_mode_pt) {
*flags = UMA_SLAB_KMEM;
ret = ((void *)kmem_malloc(kmem_arena, bytes,
M_NOWAIT));
return (ret);
}
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
VM_WAIT;
} else
break;
}
ret = (void *)arm_ptovirt(VM_PAGE_TO_PHYS(m));
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero(ret, PAGE_SIZE);
return (ret);
}
TAILQ_REMOVE(head, sp, pg_list);
TAILQ_INSERT_HEAD(&free_pgdesc, sp, pg_list);
ret = sp->addr;
mtx_unlock(&smallalloc_mtx);
if ((wait & M_ZERO))
bzero(ret, bytes);
return (ret);
}
/*
* kmem_back:
*
* Allocate physical pages for the specified virtual address range.
*/
int
kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr,
vm_size_t size, int flags)
{
vm_offset_t offset, i;
vm_page_t m, mpred;
int pflags;
KASSERT(object == kernel_object,
("kmem_back_domain: only supports kernel object."));
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
if (flags & M_WAITOK)
pflags |= VM_ALLOC_WAITFAIL;
i = 0;
VM_OBJECT_WLOCK(object);
retry:
mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i));
for (; i < size; i += PAGE_SIZE, mpred = m) {
m = vm_page_alloc_domain_after(object, atop(offset + i),
domain, pflags, mpred);
/*
* Ran out of space, free everything up and return. Don't need
* to lock page queues here as we know that the pages we got
* aren't on any queues.
*/
if (m == NULL) {
if ((flags & M_NOWAIT) == 0)
goto retry;
VM_OBJECT_WUNLOCK(object);
kmem_unback(object, addr, i);
return (KERN_NO_SPACE);
}
KASSERT(vm_phys_domain(m) == domain,
("kmem_back_domain: Domain mismatch %d != %d",
vm_phys_domain(m), domain));
if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
KASSERT((m->oflags & VPO_UNMANAGED) != 0,
("kmem_malloc: page %p is managed", m));
m->valid = VM_PAGE_BITS_ALL;
pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
}
VM_OBJECT_WUNLOCK(object);
return (KERN_SUCCESS);
}
/*
* Allocates a region from the kernel address map and physical pages
* within the specified address range to the kernel object. Creates a
* wired mapping from this region to these pages, and returns the
* region's starting virtual address. The allocated pages are not
* necessarily physically contiguous. If M_ZERO is specified through the
* given flags, then the pages are zeroed before they are mapped.
*/
vm_offset_t
kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
vm_paddr_t high, vm_memattr_t memattr)
{
vmem_t *vmem;
vm_object_t object = kernel_object;
vm_offset_t addr, i, offset;
vm_page_t m;
int pflags, tries;
size = round_page(size);
vmem = vm_dom[domain].vmd_kernel_arena;
if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr))
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
pflags |= VM_ALLOC_NOWAIT;
VM_OBJECT_WLOCK(object);
for (i = 0; i < size; i += PAGE_SIZE) {
tries = 0;
retry:
m = vm_page_alloc_contig_domain(object, atop(offset + i),
domain, pflags, 1, low, high, PAGE_SIZE, 0, memattr);
if (m == NULL) {
VM_OBJECT_WUNLOCK(object);
if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
if (!vm_page_reclaim_contig_domain(domain,
pflags, 1, low, high, PAGE_SIZE, 0) &&
(flags & M_WAITOK) != 0)
vm_wait_domain(domain);
VM_OBJECT_WLOCK(object);
tries++;
goto retry;
}
kmem_unback(object, addr, i);
vmem_free(vmem, addr, size);
return (0);
}
KASSERT(vm_phys_domain(m) == domain,
("kmem_alloc_attr_domain: Domain mismatch %d != %d",
vm_phys_domain(m), domain));
if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
m->valid = VM_PAGE_BITS_ALL;
pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
}
VM_OBJECT_WUNLOCK(object);
return (addr);
}
void *
uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
{
vm_paddr_t pa;
vm_page_t m;
int pflags;
void *va;
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
#ifdef MIPS64_NEW_PMAP
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
#else /* ! MIPS64_NEW_PMAP */
m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags);
#endif /* ! MIPS64_NEW_PMAP */
#ifndef __mips_n64
if (m == NULL && vm_page_reclaim_contig(pflags, 1,
0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0))
continue;
#endif
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
VM_WAIT;
} else
break;
}
pa = VM_PAGE_TO_PHYS(m);
if ((wait & M_NODUMP) == 0)
dump_add_page(pa);
va = (void *)MIPS_PHYS_TO_DIRECT(pa);
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero(va, PAGE_SIZE);
return (va);
}
void *
uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
{
vm_paddr_t pa;
vm_page_t m;
int pflags;
void *va;
PMAP_STATS_INC(uma_nsmall_alloc);
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
VM_WAIT;
} else
break;
}
pa = VM_PAGE_TO_PHYS(m);
if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) {
KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0,
("uma_small_alloc: free page %p still has mappings!", m));
PMAP_STATS_INC(uma_nsmall_alloc_oc);
m->md.color = DCACHE_COLOR(pa);
dcache_page_inval(pa);
}
va = (void *)TLB_PHYS_TO_DIRECT(pa);
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
cpu_block_zero(va, PAGE_SIZE);
return (va);
}