void kdp_machine_init(void) { pt_entry_t *debugger_ptep; vm_map_offset_t debugger_window_kva; if (debug_boot_arg == 0) return; vm_map_entry_t e; kern_return_t kr = vm_map_find_space(kernel_map, &debugger_window_kva, PAGE_SIZE, 0, VM_MAKE_TAG(VM_MEMORY_IOKIT), &e); if (kr != KERN_SUCCESS) { panic("%s: vm_map_find_space failed with %d\n", __FUNCTION__, kr); } vm_map_unlock(kernel_map); debugger_ptep = pmap_pte(kernel_pmap, debugger_window_kva); if (debugger_ptep == NULL) { pmap_expand(kernel_pmap, debugger_window_kva); debugger_ptep = pmap_pte(kernel_pmap, debugger_window_kva); } }
void lapic_init(void) { int result; vm_map_entry_t entry; uint32_t lo; uint32_t hi; boolean_t is_boot_processor; boolean_t is_lapic_enabled; vm_offset_t lapic_base; /* Examine the local APIC state */ rdmsr(MSR_IA32_APIC_BASE, lo, hi); is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0; is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; lapic_base = (lo & MSR_IA32_APIC_BASE_BASE); kprintf("MSR_IA32_APIC_BASE %p %s %s\n", (void *) lapic_base, is_lapic_enabled ? "enabled" : "disabled", is_boot_processor ? "BSP" : "AP"); if (!is_boot_processor || !is_lapic_enabled) panic("Unexpected local APIC state\n"); /* Establish a map to the local apic */ lapic_start = (vm_offset_t)vm_map_min(kernel_map); result = vm_map_find_space(kernel_map, (vm_map_address_t *) &lapic_start, round_page(LAPIC_SIZE), 0, VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry); if (result != KERN_SUCCESS) { panic("smp_init: vm_map_find_entry FAILED (err=%d)", result); } vm_map_unlock(kernel_map); /* Map in the local APIC non-cacheable, as recommended by Intel * in section 8.4.1 of the "System Programming Guide". */ pmap_enter(pmap_kernel(), lapic_start, (ppnum_t) i386_btop(lapic_base), VM_PROT_READ|VM_PROT_WRITE, VM_WIMG_IO, TRUE); lapic_id = (unsigned long)(lapic_start + LAPIC_ID); if ((LAPIC_READ(VERSION)&LAPIC_VERSION_MASK) < 0x14) { panic("Local APIC version 0x%x, 0x14 or more expected\n", (LAPIC_READ(VERSION)&LAPIC_VERSION_MASK)); } /* Set up the lapic_id <-> cpu_number map and add this boot processor */ lapic_cpu_map_init(); lapic_cpu_map((LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0); kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]); }
static void legacy_init(void) { int result; vm_map_entry_t entry; vm_map_offset_t lapic_vbase64; /* Establish a map to the local apic */ lapic_vbase64 = (vm_offset_t)vm_map_min(kernel_map); result = vm_map_find_space(kernel_map, &lapic_vbase64, round_page(LAPIC_SIZE), 0, VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry); /* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t */ lapic_vbase = (vm_offset_t) lapic_vbase64; if (result != KERN_SUCCESS) { panic("legacy_init: vm_map_find_entry FAILED (err=%d)", result); } vm_map_unlock(kernel_map); /* * Map in the local APIC non-cacheable, as recommended by Intel * in section 8.4.1 of the "System Programming Guide". * In fact, this is redundant because EFI will have assigned an * MTRR physical range containing the local APIC's MMIO space as * UC and this will override the default PAT setting. */ pmap_enter(pmap_kernel(), lapic_vbase, (ppnum_t) i386_btop(lapic_pbase), VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_IO, TRUE); }
/* * Initializes the range of reserved virtual addresses used by DIPC * to map the vm_page's it allocates for messages. */ void kkt_map_init(void) { vm_offset_t vaddr; vm_map_entry_t entry; kern_return_t kr; unsigned int ind; /* * Find space to map vm_page's allocated by DIPC. */ kr = vm_map_find_space(kernel_map, &vaddr, (KKT_VIRT_SIZE * PAGE_SIZE), (vm_offset_t) 0, &entry); if (kr != KERN_SUCCESS) { panic("kkt_map_init failed"); }; vm_map_unlock(kernel_map); kkt_virt_start_vaddr = vaddr; kkt_virt_status = (char *) kalloc(KKT_VIRT_SIZE * sizeof(char)); if (kkt_virt_status == NULL) panic("Unable to allocate kkt_virt_status[]"); kkt_virt_vmp = (vm_page_t *) kalloc(KKT_VIRT_SIZE * sizeof(vm_page_t)); if (kkt_virt_vmp == NULL) panic("Unable to allocate kkt_virt_vmp[]"); for (ind = 0; ind < KKT_VIRT_SIZE; ind ++) { kkt_virt_status[ind] = FREE_VADDR; kkt_virt_vmp[ind] = (vm_page_t) NULL; } simple_lock_init(&kkt_virt_lock, ETAP_DIPC_PREP_FILL); mutex_init(&kkt_mapping_lock, ETAP_DIPC_PREP_FILL); return; }
kern_return_t kernel_memory_allocate( register vm_map_t map, register vm_offset_t *addrp, register vm_size_t size, register vm_offset_t mask, int flags) { vm_object_t object; vm_object_offset_t offset; vm_object_offset_t pg_offset; vm_map_entry_t entry; vm_map_offset_t map_addr, fill_start; vm_map_offset_t map_mask; vm_map_size_t map_size, fill_size; kern_return_t kr; vm_page_t mem; vm_page_t guard_page_list = NULL; vm_page_t wired_page_list = NULL; int guard_page_count = 0; int wired_page_count = 0; int i; int vm_alloc_flags; if (! vm_kernel_ready) { panic("kernel_memory_allocate: VM is not ready"); } if (size == 0) { *addrp = 0; return KERN_INVALID_ARGUMENT; } map_size = vm_map_round_page(size); map_mask = (vm_map_offset_t) mask; vm_alloc_flags = 0; /* * limit the size of a single extent of wired memory * to try and limit the damage to the system if * too many pages get wired down */ if (map_size > (1 << 30)) { return KERN_RESOURCE_SHORTAGE; } /* * Guard pages: * * Guard pages are implemented as ficticious pages. By placing guard pages * on either end of a stack, they can help detect cases where a thread walks * off either end of its stack. They are allocated and set up here and attempts * to access those pages are trapped in vm_fault_page(). * * The map_size we were passed may include extra space for * guard pages. If those were requested, then back it out of fill_size * since vm_map_find_space() takes just the actual size not including * guard pages. Similarly, fill_start indicates where the actual pages * will begin in the range. */ fill_start = 0; fill_size = map_size; if (flags & KMA_GUARD_FIRST) { vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE; fill_start += PAGE_SIZE_64; fill_size -= PAGE_SIZE_64; if (map_size < fill_start + fill_size) { /* no space for a guard page */ *addrp = 0; return KERN_INVALID_ARGUMENT; } guard_page_count++; } if (flags & KMA_GUARD_LAST) { vm_alloc_flags |= VM_FLAGS_GUARD_AFTER; fill_size -= PAGE_SIZE_64; if (map_size <= fill_start + fill_size) { /* no space for a guard page */ *addrp = 0; return KERN_INVALID_ARGUMENT; } guard_page_count++; } wired_page_count = (int) (fill_size / PAGE_SIZE_64); assert(wired_page_count * PAGE_SIZE_64 == fill_size); for (i = 0; i < guard_page_count; i++) { for (;;) { mem = vm_page_grab_guard(); if (mem != VM_PAGE_NULL) break; if (flags & KMA_NOPAGEWAIT) { kr = KERN_RESOURCE_SHORTAGE; goto out; } vm_page_more_fictitious(); } mem->pageq.next = (queue_entry_t)guard_page_list; guard_page_list = mem; } for (i = 0; i < wired_page_count; i++) { uint64_t unavailable; for (;;) { if (flags & KMA_LOMEM) mem = vm_page_grablo(); else mem = vm_page_grab(); if (mem != VM_PAGE_NULL) break; if (flags & KMA_NOPAGEWAIT) { kr = KERN_RESOURCE_SHORTAGE; goto out; } if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) { kr = KERN_RESOURCE_SHORTAGE; goto out; } unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE; if (unavailable > max_mem || map_size > (max_mem - unavailable)) { kr = KERN_RESOURCE_SHORTAGE; goto out; } VM_PAGE_WAIT(); } mem->pageq.next = (queue_entry_t)wired_page_list; wired_page_list = mem; } /* * Allocate a new object (if necessary). We must do this before * locking the map, or risk deadlock with the default pager. */ if ((flags & KMA_KOBJECT) != 0) { object = kernel_object; vm_object_reference(object); } else { object = vm_object_allocate(map_size); } kr = vm_map_find_space(map, &map_addr, fill_size, map_mask, vm_alloc_flags, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); goto out; } entry->object.vm_object = object; entry->offset = offset = (object == kernel_object) ? map_addr : 0; entry->wired_count++; if (flags & KMA_PERMANENT) entry->permanent = TRUE; if (object != kernel_object) vm_object_reference(object); vm_object_lock(object); vm_map_unlock(map); pg_offset = 0; if (fill_start) { if (guard_page_list == NULL) panic("kernel_memory_allocate: guard_page_list == NULL"); mem = guard_page_list; guard_page_list = (vm_page_t)mem->pageq.next; mem->pageq.next = NULL; vm_page_insert(mem, object, offset + pg_offset); mem->busy = FALSE; pg_offset += PAGE_SIZE_64; } for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) { if (wired_page_list == NULL) panic("kernel_memory_allocate: wired_page_list == NULL"); mem = wired_page_list; wired_page_list = (vm_page_t)mem->pageq.next; mem->pageq.next = NULL; mem->wire_count++; vm_page_insert(mem, object, offset + pg_offset); mem->busy = FALSE; mem->pmapped = TRUE; mem->wpmapped = TRUE; PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem, VM_PROT_READ | VM_PROT_WRITE, object->wimg_bits & VM_WIMG_MASK, TRUE); if (flags & KMA_NOENCRYPT) { bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE); pmap_set_noencrypt(mem->phys_page); } }
kern_return_t kmem_alloc_contig( vm_map_t map, vm_offset_t *addrp, vm_size_t size, vm_offset_t mask, ppnum_t max_pnum, ppnum_t pnum_mask, int flags) { vm_object_t object; vm_object_offset_t offset; vm_map_offset_t map_addr; vm_map_offset_t map_mask; vm_map_size_t map_size, i; vm_map_entry_t entry; vm_page_t m, pages; kern_return_t kr; if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT))) return KERN_INVALID_ARGUMENT; if (size == 0) { *addrp = 0; return KERN_INVALID_ARGUMENT; } map_size = vm_map_round_page(size); map_mask = (vm_map_offset_t)mask; /* * Allocate a new object (if necessary) and the reference we * will be donating to the map entry. We must do this before * locking the map, or risk deadlock with the default pager. */ if ((flags & KMA_KOBJECT) != 0) { object = kernel_object; vm_object_reference(object); } else { object = vm_object_allocate(map_size); } kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); return kr; } entry->object.vm_object = object; entry->offset = offset = (object == kernel_object) ? map_addr : 0; /* Take an extra object ref in case the map entry gets deleted */ vm_object_reference(object); vm_map_unlock(map); kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags); if (kr != KERN_SUCCESS) { vm_map_remove(map, vm_map_trunc_page(map_addr), vm_map_round_page(map_addr + map_size), 0); vm_object_deallocate(object); *addrp = 0; return kr; } vm_object_lock(object); for (i = 0; i < map_size; i += PAGE_SIZE) { m = pages; pages = NEXT_PAGE(m); *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL; m->busy = FALSE; vm_page_insert(m, object, offset + i); } vm_object_unlock(object); if ((kr = vm_map_wire(map, vm_map_trunc_page(map_addr), vm_map_round_page(map_addr + map_size), VM_PROT_DEFAULT, FALSE)) != KERN_SUCCESS) { if (object == kernel_object) { vm_object_lock(object); vm_object_page_remove(object, offset, offset + map_size); vm_object_unlock(object); } vm_map_remove(map, vm_map_trunc_page(map_addr), vm_map_round_page(map_addr + map_size), 0); vm_object_deallocate(object); return kr; } vm_object_deallocate(object); if (object == kernel_object) vm_map_simplify(map, map_addr); *addrp = (vm_offset_t) map_addr; assert((vm_map_offset_t) *addrp == map_addr); return KERN_SUCCESS; }