/*
* kmem_back:
*
* Allocate physical pages for the specified virtual address range.
*/
int
kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr,
vm_size_t size, int flags)
{
vm_offset_t offset, i;
vm_page_t m, mpred;
int pflags;
KASSERT(object == kernel_object,
("kmem_back_domain: only supports kernel object."));
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
if (flags & M_WAITOK)
pflags |= VM_ALLOC_WAITFAIL;
i = 0;
VM_OBJECT_WLOCK(object);
retry:
mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i));
for (; i < size; i += PAGE_SIZE, mpred = m) {
m = vm_page_alloc_domain_after(object, atop(offset + i),
domain, pflags, mpred);
/*
* Ran out of space, free everything up and return. Don't need
* to lock page queues here as we know that the pages we got
* aren't on any queues.
*/
if (m == NULL) {
if ((flags & M_NOWAIT) == 0)
goto retry;
VM_OBJECT_WUNLOCK(object);
kmem_unback(object, addr, i);
return (KERN_NO_SPACE);
}
KASSERT(vm_phys_domain(m) == domain,
("kmem_back_domain: Domain mismatch %d != %d",
vm_phys_domain(m), domain));
if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
KASSERT((m->oflags & VPO_UNMANAGED) != 0,
("kmem_malloc: page %p is managed", m));
m->valid = VM_PAGE_BITS_ALL;
pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
}
VM_OBJECT_WUNLOCK(object);
return (KERN_SUCCESS);
}
/*
* Allocates a region from the kernel address map and physical pages
* within the specified address range to the kernel object. Creates a
* wired mapping from this region to these pages, and returns the
* region's starting virtual address. The allocated pages are not
* necessarily physically contiguous. If M_ZERO is specified through the
* given flags, then the pages are zeroed before they are mapped.
*/
vm_offset_t
kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
vm_paddr_t high, vm_memattr_t memattr)
{
vmem_t *vmem;
vm_object_t object = kernel_object;
vm_offset_t addr, i, offset;
vm_page_t m;
int pflags, tries;
size = round_page(size);
vmem = vm_dom[domain].vmd_kernel_arena;
if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr))
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
pflags |= VM_ALLOC_NOWAIT;
VM_OBJECT_WLOCK(object);
for (i = 0; i < size; i += PAGE_SIZE) {
tries = 0;
retry:
m = vm_page_alloc_contig_domain(object, atop(offset + i),
domain, pflags, 1, low, high, PAGE_SIZE, 0, memattr);
if (m == NULL) {
VM_OBJECT_WUNLOCK(object);
if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
if (!vm_page_reclaim_contig_domain(domain,
pflags, 1, low, high, PAGE_SIZE, 0) &&
(flags & M_WAITOK) != 0)
vm_wait_domain(domain);
VM_OBJECT_WLOCK(object);
tries++;
goto retry;
}
kmem_unback(object, addr, i);
vmem_free(vmem, addr, size);
return (0);
}
KASSERT(vm_phys_domain(m) == domain,
("kmem_alloc_attr_domain: Domain mismatch %d != %d",
vm_phys_domain(m), domain));
if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
m->valid = VM_PAGE_BITS_ALL;
pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
}
VM_OBJECT_WUNLOCK(object);
return (addr);
}
/*
* Free specified single object.
*/
void
memguard_free(void *ptr)
{
vm_offset_t addr;
u_long req_size, size, sizev;
char *temp;
int i;
addr = trunc_page((uintptr_t)ptr);
req_size = *v2sizep(addr);
sizev = *v2sizev(addr);
size = round_page(req_size);
/*
* Page should not be guarded right now, so force a write.
* The purpose of this is to increase the likelihood of
* catching a double-free, but not necessarily a
* tamper-after-free (the second thread freeing might not
* write before freeing, so this forces it to and,
* subsequently, trigger a fault).
*/
temp = ptr;
for (i = 0; i < size; i += PAGE_SIZE)
temp[i] = 'M';
/*
* This requires carnal knowledge of the implementation of
* kmem_free(), but since we've already replaced kmem_malloc()
* above, it's not really any worse. We want to use the
* vm_map lock to serialize updates to memguard_wasted, since
* we had the lock at increment.
*/
kmem_unback(kmem_object, addr, size);
if (sizev > size)
addr -= PAGE_SIZE;
vmem_xfree(memguard_arena, addr, sizev);
if (req_size < PAGE_SIZE)
memguard_wasted -= (PAGE_SIZE - req_size);
}
