void kdp_machine_init(void) {
pt_entry_t *debugger_ptep;
vm_map_offset_t debugger_window_kva;
if (debug_boot_arg == 0)
return;
vm_map_entry_t e;
kern_return_t kr = vm_map_find_space(kernel_map,
&debugger_window_kva,
PAGE_SIZE, 0,
VM_MAKE_TAG(VM_MEMORY_IOKIT), &e);
if (kr != KERN_SUCCESS) {
panic("%s: vm_map_find_space failed with %d\n", __FUNCTION__, kr);
}
vm_map_unlock(kernel_map);
debugger_ptep = pmap_pte(kernel_pmap, debugger_window_kva);
if (debugger_ptep == NULL) {
pmap_expand(kernel_pmap, debugger_window_kva);
debugger_ptep = pmap_pte(kernel_pmap, debugger_window_kva);
}
}
void
lapic_init(void)
{
int result;
vm_map_entry_t entry;
uint32_t lo;
uint32_t hi;
boolean_t is_boot_processor;
boolean_t is_lapic_enabled;
vm_offset_t lapic_base;
/* Examine the local APIC state */
rdmsr(MSR_IA32_APIC_BASE, lo, hi);
is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0;
is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0;
lapic_base = (lo & MSR_IA32_APIC_BASE_BASE);
kprintf("MSR_IA32_APIC_BASE %p %s %s\n", (void *) lapic_base,
is_lapic_enabled ? "enabled" : "disabled",
is_boot_processor ? "BSP" : "AP");
if (!is_boot_processor || !is_lapic_enabled)
panic("Unexpected local APIC state\n");
/* Establish a map to the local apic */
lapic_start = (vm_offset_t)vm_map_min(kernel_map);
result = vm_map_find_space(kernel_map,
(vm_map_address_t *) &lapic_start,
round_page(LAPIC_SIZE), 0,
VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry);
if (result != KERN_SUCCESS) {
panic("smp_init: vm_map_find_entry FAILED (err=%d)", result);
}
vm_map_unlock(kernel_map);
/* Map in the local APIC non-cacheable, as recommended by Intel
* in section 8.4.1 of the "System Programming Guide".
*/
pmap_enter(pmap_kernel(),
lapic_start,
(ppnum_t) i386_btop(lapic_base),
VM_PROT_READ|VM_PROT_WRITE,
VM_WIMG_IO,
TRUE);
lapic_id = (unsigned long)(lapic_start + LAPIC_ID);
if ((LAPIC_READ(VERSION)&LAPIC_VERSION_MASK) < 0x14) {
panic("Local APIC version 0x%x, 0x14 or more expected\n",
(LAPIC_READ(VERSION)&LAPIC_VERSION_MASK));
}
/* Set up the lapic_id <-> cpu_number map and add this boot processor */
lapic_cpu_map_init();
lapic_cpu_map((LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0);
kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]);
}
static void
legacy_init(void)
{
int result;
vm_map_entry_t entry;
vm_map_offset_t lapic_vbase64;
/* Establish a map to the local apic */
lapic_vbase64 = (vm_offset_t)vm_map_min(kernel_map);
result = vm_map_find_space(kernel_map,
&lapic_vbase64,
round_page(LAPIC_SIZE), 0,
VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry);
/* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t
*/
lapic_vbase = (vm_offset_t) lapic_vbase64;
if (result != KERN_SUCCESS) {
panic("legacy_init: vm_map_find_entry FAILED (err=%d)", result);
}
vm_map_unlock(kernel_map);
/*
* Map in the local APIC non-cacheable, as recommended by Intel
* in section 8.4.1 of the "System Programming Guide".
* In fact, this is redundant because EFI will have assigned an
* MTRR physical range containing the local APIC's MMIO space as
* UC and this will override the default PAT setting.
*/
pmap_enter(pmap_kernel(),
lapic_vbase,
(ppnum_t) i386_btop(lapic_pbase),
VM_PROT_READ|VM_PROT_WRITE,
VM_PROT_NONE,
VM_WIMG_IO,
TRUE);
}
/*
* Initializes the range of reserved virtual addresses used by DIPC
* to map the vm_page's it allocates for messages.
*/
void kkt_map_init(void)
{
vm_offset_t vaddr;
vm_map_entry_t entry;
kern_return_t kr;
unsigned int ind;
/*
* Find space to map vm_page's allocated by DIPC.
*/
kr = vm_map_find_space(kernel_map, &vaddr, (KKT_VIRT_SIZE * PAGE_SIZE),
(vm_offset_t) 0, &entry);
if (kr != KERN_SUCCESS) {
panic("kkt_map_init failed");
};
vm_map_unlock(kernel_map);
kkt_virt_start_vaddr = vaddr;
kkt_virt_status = (char *) kalloc(KKT_VIRT_SIZE * sizeof(char));
if (kkt_virt_status == NULL)
panic("Unable to allocate kkt_virt_status[]");
kkt_virt_vmp = (vm_page_t *) kalloc(KKT_VIRT_SIZE * sizeof(vm_page_t));
if (kkt_virt_vmp == NULL)
panic("Unable to allocate kkt_virt_vmp[]");
for (ind = 0; ind < KKT_VIRT_SIZE; ind ++) {
kkt_virt_status[ind] = FREE_VADDR;
kkt_virt_vmp[ind] = (vm_page_t) NULL;
}
simple_lock_init(&kkt_virt_lock, ETAP_DIPC_PREP_FILL);
mutex_init(&kkt_mapping_lock, ETAP_DIPC_PREP_FILL);
return;
}
kern_return_t
kernel_memory_allocate(
register vm_map_t map,
register vm_offset_t *addrp,
register vm_size_t size,
register vm_offset_t mask,
int flags)
{
vm_object_t object;
vm_object_offset_t offset;
vm_object_offset_t pg_offset;
vm_map_entry_t entry;
vm_map_offset_t map_addr, fill_start;
vm_map_offset_t map_mask;
vm_map_size_t map_size, fill_size;
kern_return_t kr;
vm_page_t mem;
vm_page_t guard_page_list = NULL;
vm_page_t wired_page_list = NULL;
int guard_page_count = 0;
int wired_page_count = 0;
int i;
int vm_alloc_flags;
if (! vm_kernel_ready) {
panic("kernel_memory_allocate: VM is not ready");
}
if (size == 0) {
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
map_size = vm_map_round_page(size);
map_mask = (vm_map_offset_t) mask;
vm_alloc_flags = 0;
/*
* limit the size of a single extent of wired memory
* to try and limit the damage to the system if
* too many pages get wired down
*/
if (map_size > (1 << 30)) {
return KERN_RESOURCE_SHORTAGE;
}
/*
* Guard pages:
*
* Guard pages are implemented as ficticious pages. By placing guard pages
* on either end of a stack, they can help detect cases where a thread walks
* off either end of its stack. They are allocated and set up here and attempts
* to access those pages are trapped in vm_fault_page().
*
* The map_size we were passed may include extra space for
* guard pages. If those were requested, then back it out of fill_size
* since vm_map_find_space() takes just the actual size not including
* guard pages. Similarly, fill_start indicates where the actual pages
* will begin in the range.
*/
fill_start = 0;
fill_size = map_size;
if (flags & KMA_GUARD_FIRST) {
vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE;
fill_start += PAGE_SIZE_64;
fill_size -= PAGE_SIZE_64;
if (map_size < fill_start + fill_size) {
/* no space for a guard page */
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
guard_page_count++;
}
if (flags & KMA_GUARD_LAST) {
vm_alloc_flags |= VM_FLAGS_GUARD_AFTER;
fill_size -= PAGE_SIZE_64;
if (map_size <= fill_start + fill_size) {
/* no space for a guard page */
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
guard_page_count++;
}
wired_page_count = (int) (fill_size / PAGE_SIZE_64);
assert(wired_page_count * PAGE_SIZE_64 == fill_size);
for (i = 0; i < guard_page_count; i++) {
for (;;) {
mem = vm_page_grab_guard();
if (mem != VM_PAGE_NULL)
break;
if (flags & KMA_NOPAGEWAIT) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
vm_page_more_fictitious();
}
mem->pageq.next = (queue_entry_t)guard_page_list;
guard_page_list = mem;
}
for (i = 0; i < wired_page_count; i++) {
uint64_t unavailable;
for (;;) {
if (flags & KMA_LOMEM)
mem = vm_page_grablo();
else
mem = vm_page_grab();
if (mem != VM_PAGE_NULL)
break;
if (flags & KMA_NOPAGEWAIT) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE;
if (unavailable > max_mem || map_size > (max_mem - unavailable)) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
VM_PAGE_WAIT();
}
mem->pageq.next = (queue_entry_t)wired_page_list;
wired_page_list = mem;
}
/*
* Allocate a new object (if necessary). We must do this before
* locking the map, or risk deadlock with the default pager.
*/
if ((flags & KMA_KOBJECT) != 0) {
object = kernel_object;
vm_object_reference(object);
} else {
object = vm_object_allocate(map_size);
}
kr = vm_map_find_space(map, &map_addr,
fill_size, map_mask,
vm_alloc_flags, &entry);
if (KERN_SUCCESS != kr) {
vm_object_deallocate(object);
goto out;
}
entry->object.vm_object = object;
entry->offset = offset = (object == kernel_object) ?
map_addr : 0;
entry->wired_count++;
if (flags & KMA_PERMANENT)
entry->permanent = TRUE;
if (object != kernel_object)
vm_object_reference(object);
vm_object_lock(object);
vm_map_unlock(map);
pg_offset = 0;
if (fill_start) {
if (guard_page_list == NULL)
panic("kernel_memory_allocate: guard_page_list == NULL");
mem = guard_page_list;
guard_page_list = (vm_page_t)mem->pageq.next;
mem->pageq.next = NULL;
vm_page_insert(mem, object, offset + pg_offset);
mem->busy = FALSE;
pg_offset += PAGE_SIZE_64;
}
for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) {
if (wired_page_list == NULL)
panic("kernel_memory_allocate: wired_page_list == NULL");
mem = wired_page_list;
wired_page_list = (vm_page_t)mem->pageq.next;
mem->pageq.next = NULL;
mem->wire_count++;
vm_page_insert(mem, object, offset + pg_offset);
mem->busy = FALSE;
mem->pmapped = TRUE;
mem->wpmapped = TRUE;
PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem,
VM_PROT_READ | VM_PROT_WRITE, object->wimg_bits & VM_WIMG_MASK, TRUE);
if (flags & KMA_NOENCRYPT) {
bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE);
pmap_set_noencrypt(mem->phys_page);
}
}
kern_return_t
kmem_alloc_contig(
vm_map_t map,
vm_offset_t *addrp,
vm_size_t size,
vm_offset_t mask,
ppnum_t max_pnum,
ppnum_t pnum_mask,
int flags)
{
vm_object_t object;
vm_object_offset_t offset;
vm_map_offset_t map_addr;
vm_map_offset_t map_mask;
vm_map_size_t map_size, i;
vm_map_entry_t entry;
vm_page_t m, pages;
kern_return_t kr;
if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT)))
return KERN_INVALID_ARGUMENT;
if (size == 0) {
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
map_size = vm_map_round_page(size);
map_mask = (vm_map_offset_t)mask;
/*
* Allocate a new object (if necessary) and the reference we
* will be donating to the map entry. We must do this before
* locking the map, or risk deadlock with the default pager.
*/
if ((flags & KMA_KOBJECT) != 0) {
object = kernel_object;
vm_object_reference(object);
} else {
object = vm_object_allocate(map_size);
}
kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
if (KERN_SUCCESS != kr) {
vm_object_deallocate(object);
return kr;
}
entry->object.vm_object = object;
entry->offset = offset = (object == kernel_object) ?
map_addr : 0;
/* Take an extra object ref in case the map entry gets deleted */
vm_object_reference(object);
vm_map_unlock(map);
kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags);
if (kr != KERN_SUCCESS) {
vm_map_remove(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), 0);
vm_object_deallocate(object);
*addrp = 0;
return kr;
}
vm_object_lock(object);
for (i = 0; i < map_size; i += PAGE_SIZE) {
m = pages;
pages = NEXT_PAGE(m);
*(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
m->busy = FALSE;
vm_page_insert(m, object, offset + i);
}
vm_object_unlock(object);
if ((kr = vm_map_wire(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), VM_PROT_DEFAULT, FALSE))
!= KERN_SUCCESS) {
if (object == kernel_object) {
vm_object_lock(object);
vm_object_page_remove(object, offset, offset + map_size);
vm_object_unlock(object);
}
vm_map_remove(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), 0);
vm_object_deallocate(object);
return kr;
}
vm_object_deallocate(object);
if (object == kernel_object)
vm_map_simplify(map, map_addr);
*addrp = (vm_offset_t) map_addr;
assert((vm_map_offset_t) *addrp == map_addr);
return KERN_SUCCESS;
}
