/* * kmap_alloc_wait: * * Allocates pageable memory from a sub-map of the kernel. If the submap * has no room, the caller sleeps waiting for more memory in the submap. * * This routine may block. */ vm_offset_t kmap_alloc_wait(vm_map_t map, vm_size_t size) { vm_offset_t addr; size = round_page(size); if (!swap_reserve(size)) return (0); for (;;) { /* * To make this work for more than one map, use the map's lock * to lock out sleepers/wakers. */ vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0) break; /* no space now; see if we can ever get space */ if (vm_map_max(map) - vm_map_min(map) < size) { vm_map_unlock(map); swap_release(size); return (0); } map->needs_wakeup = TRUE; vm_map_unlock_and_wait(map, 0); } vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, MAP_ACC_CHARGED); vm_map_unlock(map); return (addr); }
vaddr_t uvm_km_valloc_align(struct vm_map *map, vsize_t size, vsize_t align) { vaddr_t kva; UVMHIST_FUNC("uvm_km_valloc"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0); KASSERT(vm_map_pmap(map) == pmap_kernel()); size = round_page(size); kva = vm_map_min(map); /* hint */ /* * allocate some virtual space. will be demand filled by kernel_object. */ if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object, UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) { UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0); return(0); } UVMHIST_LOG(maphist, "<- done (kva=0x%lx)", kva,0,0,0); return(kva); }
void sh_startup() { vaddr_t minaddr, maxaddr; printf("%s", version); if (*cpu_model != '\0') printf("%s\n", cpu_model); #ifdef DEBUG printf("general exception handler:\t%d byte\n", sh_vector_generic_end - sh_vector_generic); printf("TLB miss exception handler:\t%d byte\n", #if defined(SH3) && defined(SH4) CPU_IS_SH3 ? sh3_vector_tlbmiss_end - sh3_vector_tlbmiss : sh4_vector_tlbmiss_end - sh4_vector_tlbmiss #elif defined(SH3) sh3_vector_tlbmiss_end - sh3_vector_tlbmiss #elif defined(SH4) sh4_vector_tlbmiss_end - sh4_vector_tlbmiss #endif ); printf("interrupt exception handler:\t%d byte\n", sh_vector_interrupt_end - sh_vector_interrupt); #endif /* DEBUG */ printf("real mem = %u (%uMB)\n", ptoa(physmem), ptoa(physmem) / 1024 / 1024); /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* * Allocate a submap for physio */ phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, VM_PHYS_SIZE, 0, FALSE, NULL); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free) / 1024 / 1024); if (boothowto & RB_CONFIG) { #ifdef BOOT_CONFIG user_config(); #else printf("kernel does not support -c; continuing..\n"); #endif } }
/* * cpu_startup: allocate memory for variable-sized tables, initialize CPU, and * do auto-configuration. */ void cpu_startup() { vaddr_t minaddr, maxaddr; #ifdef PMAPDEBUG extern int pmapdebug; int opmapdebug = pmapdebug; pmapdebug = 0; /* Shut up pmap debug during bootstrap. */ #endif /* * Good {morning,afternoon,evening,night}. */ printf(version); printf("real mem = %lu (%luMB)\n", ptoa((psize_t)physmem), ptoa((psize_t)physmem)/1024/1024); printf("rsvd mem = %lu (%luMB)\n", ptoa((psize_t)rsvdmem), ptoa((psize_t)rsvdmem)/1024/1024); /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* Allocate a submap for physio. */ phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, VM_PHYS_SIZE, 0, FALSE, NULL); #ifdef PMAPDEBUG pmapdebug = opmapdebug; #endif printf("avail mem = %lu (%luMB)\n", ptoa((psize_t)uvmexp.free), ptoa((psize_t)uvmexp.free)/1024/1024); /* * Set up CPU-specific registers, cache, etc. */ initcpu(); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); /* * Configure the system. */ if (boothowto & RB_CONFIG) { #ifdef BOOT_CONFIG user_config(); #else printf("kernel does not support -c; continuing..\n"); #endif } }
/* * Helper routines to allow the backing object of a shared memory file * descriptor to be mapped in the kernel. */ int shm_map(struct file *fp, size_t size, off_t offset, void **memp) { struct shmfd *shmfd; vm_offset_t kva, ofs; vm_object_t obj; int rv; if (fp->f_type != DTYPE_SHM) return (EINVAL); shmfd = fp->f_data; obj = shmfd->shm_object; VM_OBJECT_LOCK(obj); /* * XXXRW: This validation is probably insufficient, and subject to * sign errors. It should be fixed. */ if (offset >= shmfd->shm_size || offset + size > round_page(shmfd->shm_size)) { VM_OBJECT_UNLOCK(obj); return (EINVAL); } shmfd->shm_kmappings++; vm_object_reference_locked(obj); VM_OBJECT_UNLOCK(obj); /* Map the object into the kernel_map and wire it. */ kva = vm_map_min(kernel_map); ofs = offset & PAGE_MASK; offset = trunc_page(offset); size = round_page(size + ofs); rv = vm_map_find(kernel_map, obj, offset, &kva, size, VMFS_ALIGNED_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0); if (rv == KERN_SUCCESS) { rv = vm_map_wire(kernel_map, kva, kva + size, VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); if (rv == KERN_SUCCESS) { *memp = (void *)(kva + ofs); return (0); } vm_map_remove(kernel_map, kva, kva + size); } else vm_object_deallocate(obj); /* On failure, drop our mapping reference. */ VM_OBJECT_LOCK(obj); shmfd->shm_kmappings--; VM_OBJECT_UNLOCK(obj); return (vm_mmap_to_errno(rv)); }
void lapic_init(void) { int result; vm_map_entry_t entry; uint32_t lo; uint32_t hi; boolean_t is_boot_processor; boolean_t is_lapic_enabled; vm_offset_t lapic_base; /* Examine the local APIC state */ rdmsr(MSR_IA32_APIC_BASE, lo, hi); is_boot_processor = (lo & MSR_IA32_APIC_BASE_BSP) != 0; is_lapic_enabled = (lo & MSR_IA32_APIC_BASE_ENABLE) != 0; lapic_base = (lo & MSR_IA32_APIC_BASE_BASE); kprintf("MSR_IA32_APIC_BASE %p %s %s\n", (void *) lapic_base, is_lapic_enabled ? "enabled" : "disabled", is_boot_processor ? "BSP" : "AP"); if (!is_boot_processor || !is_lapic_enabled) panic("Unexpected local APIC state\n"); /* Establish a map to the local apic */ lapic_start = (vm_offset_t)vm_map_min(kernel_map); result = vm_map_find_space(kernel_map, (vm_map_address_t *) &lapic_start, round_page(LAPIC_SIZE), 0, VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry); if (result != KERN_SUCCESS) { panic("smp_init: vm_map_find_entry FAILED (err=%d)", result); } vm_map_unlock(kernel_map); /* Map in the local APIC non-cacheable, as recommended by Intel * in section 8.4.1 of the "System Programming Guide". */ pmap_enter(pmap_kernel(), lapic_start, (ppnum_t) i386_btop(lapic_base), VM_PROT_READ|VM_PROT_WRITE, VM_WIMG_IO, TRUE); lapic_id = (unsigned long)(lapic_start + LAPIC_ID); if ((LAPIC_READ(VERSION)&LAPIC_VERSION_MASK) < 0x14) { panic("Local APIC version 0x%x, 0x14 or more expected\n", (LAPIC_READ(VERSION)&LAPIC_VERSION_MASK)); } /* Set up the lapic_id <-> cpu_number map and add this boot processor */ lapic_cpu_map_init(); lapic_cpu_map((LAPIC_READ(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0); kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]); }
void cpu_startup() { vaddr_t minaddr, maxaddr; /* * Good {morning,afternoon,evening,night}. */ printf(version); printf("real mem = %lu (%luMB)\n", ptoa(physmem), ptoa(physmem)/1024/1024); /* * Grab machine dependent memory spaces */ platform->startup(); /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* * Allocate map for physio. */ phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, VM_PHYS_SIZE, 0, FALSE, NULL); printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free)/1024/1024); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); /* * Configure the system. */ if (boothowto & RB_CONFIG) { #ifdef BOOT_CONFIG user_config(); #else printf("kernel does not support -c; continuing..\n"); #endif } }
vaddr_t uvm_km_valloc_prefer_wait(struct vm_map *map, vsize_t size, voff_t prefer) { vaddr_t kva; UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0); KASSERT(vm_map_pmap(map) == pmap_kernel()); size = round_page(size); if (size > vm_map_max(map) - vm_map_min(map)) return(0); while (1) { kva = vm_map_min(map); /* hint */ /* * allocate some virtual space. will be demand filled * by kernel_object. */ if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object, prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) == 0)) { UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0); return(kva); } /* * failed. sleep for a while (on map) */ UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0); tsleep((caddr_t)map, PVM, "vallocwait", 0); } /*NOTREACHED*/ }
int exec_sigcode_map(struct proc *p, struct emul *e) { vsize_t sz; sz = (vaddr_t)e->e_esigcode - (vaddr_t)e->e_sigcode; /* * If we don't have a sigobject for this emulation, create one. * * sigobject is an anonymous memory object (just like SYSV shared * memory) that we keep a permanent reference to and that we map * in all processes that need this sigcode. The creation is simple, * we create an object, add a permanent reference to it, map it in * kernel space, copy out the sigcode to it and unmap it. * Then we map it with PROT_READ|PROT_EXEC into the process just * the way sys_mmap would map it. */ if (e->e_sigobject == NULL) { vaddr_t va; int r; e->e_sigobject = uao_create(sz, 0); uao_reference(e->e_sigobject); /* permanent reference */ va = vm_map_min(kernel_map); /* hint */ if ((r = uvm_map(kernel_map, &va, round_page(sz), e->e_sigobject, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE, UVM_ADV_RANDOM, 0)))) { uao_detach(e->e_sigobject); return (ENOMEM); } memcpy((void *)va, e->e_sigcode, sz); uvm_unmap(kernel_map, va, va + round_page(sz)); } /* Just a hint to uvm_mmap where to put it. */ p->p_sigcode = uvm_map_hint(p, VM_PROT_READ|VM_PROT_EXECUTE); uao_reference(e->e_sigobject); if (uvm_map(&p->p_vmspace->vm_map, &p->p_sigcode, round_page(sz), e->e_sigobject, 0, 0, UVM_MAPFLAG(UVM_PROT_RX, UVM_PROT_RX, UVM_INH_SHARE, UVM_ADV_RANDOM, 0))) { uao_detach(e->e_sigobject); return (ENOMEM); } return (0); }
/* * Import kva into the kernel arena. */ static int kva_import(void *unused, vmem_size_t size, int flags, vmem_addr_t *addrp) { vm_offset_t addr; int result; addr = vm_map_min(kernel_map); result = vm_map_find(kernel_map, NULL, 0, &addr, size, 0, VMFS_SUPER_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); if (result != KERN_SUCCESS) return (ENOMEM); *addrp = addr; return (0); }
void buf_mem_init(vsize_t size) { TAILQ_INIT(&buf_valist); buf_kva_start = vm_map_min(kernel_map); if (uvm_map(kernel_map, &buf_kva_start, size, NULL, UVM_UNKNOWN_OFFSET, PAGE_SIZE, UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_NORMAL, 0))) panic("bufinit: can't reserve VM for buffers"); buf_kva_end = buf_kva_start + size; buf_object = &buf_object_store; uvm_objinit(buf_object, NULL, 1); }
/* * tunable_mbinit() has to be run before any mbuf allocations are done. */ static void tunable_mbinit(void *dummy) { #ifndef __rtems__ quad_t realmem; /* * The default limit for all mbuf related memory is 1/2 of all * available kernel memory (physical or kmem). * At most it can be 3/4 of available kernel memory. */ realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_map_max(kmem_map) - vm_map_min(kmem_map)); maxmbufmem = realmem / 2; TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem); if (maxmbufmem > realmem / 4 * 3) maxmbufmem = realmem / 4 * 3; #else /* __rtems__ */ maxmbufmem = rtems_bsd_get_allocator_domain_size( RTEMS_BSD_ALLOCATOR_DOMAIN_MBUF); #endif /* __rtems__ */ TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); if (nmbclusters == 0) nmbclusters = maxmbufmem / MCLBYTES / 4; TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop); if (nmbjumbop == 0) nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4; TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9); if (nmbjumbo9 == 0) nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6; TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16); if (nmbjumbo16 == 0) nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6; /* * We need at least as many mbufs as we have clusters of * the various types added together. */ TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs); if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) nmbufs = lmax(maxmbufmem / MSIZE / 5, nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16); }
static int rtR0MemObjFreeBSDAllocHelper(PRTR0MEMOBJFREEBSD pMemFreeBSD, bool fExecutable, vm_paddr_t VmPhysAddrHigh, bool fContiguous, int rcNoMem) { vm_offset_t MapAddress = vm_map_min(kernel_map); size_t cPages = atop(pMemFreeBSD->Core.cb); int rc; pMemFreeBSD->pObject = vm_object_allocate(OBJT_PHYS, cPages); /* No additional object reference for auto-deallocation upon unmapping. */ #if __FreeBSD_version >= 1000055 rc = vm_map_find(kernel_map, pMemFreeBSD->pObject, 0, &MapAddress, pMemFreeBSD->Core.cb, 0, VMFS_ANY_SPACE, fExecutable ? VM_PROT_ALL : VM_PROT_RW, VM_PROT_ALL, 0); #else rc = vm_map_find(kernel_map, pMemFreeBSD->pObject, 0, &MapAddress, pMemFreeBSD->Core.cb, VMFS_ANY_SPACE, fExecutable ? VM_PROT_ALL : VM_PROT_RW, VM_PROT_ALL, 0); #endif if (rc == KERN_SUCCESS) { rc = rtR0MemObjFreeBSDPhysAllocHelper(pMemFreeBSD->pObject, cPages, VmPhysAddrHigh, PAGE_SIZE, fContiguous, false, rcNoMem); if (RT_SUCCESS(rc)) { vm_map_wire(kernel_map, MapAddress, MapAddress + pMemFreeBSD->Core.cb, VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); /* Store start address */ pMemFreeBSD->Core.pv = (void *)MapAddress; return VINF_SUCCESS; } vm_map_remove(kernel_map, MapAddress, MapAddress + pMemFreeBSD->Core.cb); } else { rc = rcNoMem; /** @todo fix translation (borrow from darwin) */ vm_object_deallocate(pMemFreeBSD->pObject); } rtR0MemObjDelete(&pMemFreeBSD->Core); return rc; }
/* * tunable_mbinit() has to be run before any mbuf allocations are done. */ static void tunable_mbinit(void *dummy) { quad_t realmem, maxmbufmem; /* * The default limit for all mbuf related memory is 1/2 of all * available kernel memory (physical or kmem). * At most it can be 3/4 of available kernel memory. */ realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_map_max(kernel_map) - vm_map_min(kernel_map)); maxmbufmem = realmem / 2; TUNABLE_QUAD_FETCH("kern.maxmbufmem", &maxmbufmem); if (maxmbufmem > realmem / 4 * 3) maxmbufmem = realmem / 4 * 3; TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); if (nmbclusters == 0) nmbclusters = maxmbufmem / MCLBYTES / 4; TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop); if (nmbjumbop == 0) nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4; TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9); if (nmbjumbo9 == 0) nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6; TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16); if (nmbjumbo16 == 0) nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6; /* * We need at least as many mbufs as we have clusters of * the various types added together. */ TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs); if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) nmbufs = lmax(maxmbufmem / MSIZE / 5, nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16); }
static int range_test(struct vm_map *map, vaddr_t addr, vsize_t size, bool ismmap) { vaddr_t vm_min_address = vm_map_min(map); vaddr_t vm_max_address = vm_map_max(map); vaddr_t eaddr = addr + size; int res = 0; if (addr < vm_min_address) return EINVAL; if (eaddr > vm_max_address) return ismmap ? EFBIG : EINVAL; if (addr > eaddr) /* no wrapping! */ return ismmap ? EOVERFLOW : EINVAL; #ifdef MD_MMAP_RANGE_TEST res = MD_MMAP_RANGE_TEST(addr, eaddr); #endif return res; }
kern_return_t kmem_alloc_pageable( vm_map_t map, vm_offset_t *addrp, vm_size_t size) { vm_offset_t addr; kern_return_t kr; addr = vm_map_min(map); kr = vm_map_enter(map, &addr, round_page(size), (vm_offset_t) 0, TRUE, VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { printf_once("no more room for kmem_alloc_pageable in %p\n", map); return kr; } *addrp = addr; return KERN_SUCCESS; }
void kmem_submap( vm_map_t map, vm_map_t parent, vm_offset_t *min, vm_offset_t *max, vm_size_t size, boolean_t pageable) { vm_offset_t addr; kern_return_t kr; size = round_page(size); /* * Need reference on submap object because it is internal * to the vm_system. vm_object_enter will never be called * on it (usual source of reference for vm_map_enter). */ vm_object_reference(vm_submap_object); addr = vm_map_min(parent); kr = vm_map_enter(parent, &addr, size, (vm_offset_t) 0, TRUE, vm_submap_object, (vm_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) panic("kmem_submap"); pmap_reference(vm_map_pmap(parent)); vm_map_setup(map, vm_map_pmap(parent), addr, addr + size, pageable); kr = vm_map_submap(parent, addr, addr + size, map); if (kr != KERN_SUCCESS) panic("kmem_submap"); *min = addr; *max = addr + size; }
/* * kmem_suballoc: * * Allocates a map to manage a subrange * of the kernel virtual address space. * * Arguments are as follows: * * parent Map to take range from * min, max Returned endpoints of map * size Size of range to find * superpage_align Request that min is superpage aligned */ vm_map_t kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max, vm_size_t size, boolean_t superpage_align) { int ret; vm_map_t result; size = round_page(size); *min = vm_map_min(parent); ret = vm_map_find(parent, NULL, 0, min, size, 0, superpage_align ? VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_ACC_NO_CHARGE); if (ret != KERN_SUCCESS) panic("kmem_suballoc: bad status return of %d", ret); *max = *min + size; result = vm_map_create(vm_map_pmap(parent), *min, *max); if (result == NULL) panic("kmem_suballoc: cannot create submap"); if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS) panic("kmem_suballoc: unable to change range to submap"); return (result); }
/* * Return a fudged value to be used for vm_kmem_size for allocating * the kmem_map. The memguard memory will be a submap. */ unsigned long memguard_fudge(unsigned long km_size, const struct vm_map *parent_map) { u_long mem_pgs, parent_size; vm_memguard_divisor = 10; /* CTFLAG_RDTUN doesn't work during the early boot process. */ TUNABLE_INT_FETCH("vm.memguard.divisor", &vm_memguard_divisor); parent_size = vm_map_max(parent_map) - vm_map_min(parent_map) + PAGE_SIZE; /* Pick a conservative value if provided value sucks. */ if ((vm_memguard_divisor <= 0) || ((parent_size / vm_memguard_divisor) == 0)) vm_memguard_divisor = 10; /* * Limit consumption of physical pages to * 1/vm_memguard_divisor of system memory. If the KVA is * smaller than this then the KVA limit comes into play first. * This prevents memguard's page promotions from completely * using up memory, since most malloc(9) calls are sub-page. */ mem_pgs = vm_cnt.v_page_count; memguard_physlimit = (mem_pgs / vm_memguard_divisor) * PAGE_SIZE; /* * We want as much KVA as we can take safely. Use at most our * allotted fraction of the parent map's size. Limit this to * twice the physical memory to avoid using too much memory as * pagetable pages (size must be multiple of PAGE_SIZE). */ memguard_mapsize = round_page(parent_size / vm_memguard_divisor); if (memguard_mapsize / (2 * PAGE_SIZE) > mem_pgs) memguard_mapsize = mem_pgs * 2 * PAGE_SIZE; if (km_size + memguard_mapsize > parent_size) memguard_mapsize = 0; return (km_size + memguard_mapsize); }
static void legacy_init(void) { int result; vm_map_entry_t entry; vm_map_offset_t lapic_vbase64; /* Establish a map to the local apic */ lapic_vbase64 = (vm_offset_t)vm_map_min(kernel_map); result = vm_map_find_space(kernel_map, &lapic_vbase64, round_page(LAPIC_SIZE), 0, VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry); /* Convert 64-bit vm_map_offset_t to "pointer sized" vm_offset_t */ lapic_vbase = (vm_offset_t) lapic_vbase64; if (result != KERN_SUCCESS) { panic("legacy_init: vm_map_find_entry FAILED (err=%d)", result); } vm_map_unlock(kernel_map); /* * Map in the local APIC non-cacheable, as recommended by Intel * in section 8.4.1 of the "System Programming Guide". * In fact, this is redundant because EFI will have assigned an * MTRR physical range containing the local APIC's MMIO space as * UC and this will override the default PAT setting. */ pmap_enter(pmap_kernel(), lapic_vbase, (ppnum_t) i386_btop(lapic_pbase), VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_IO, TRUE); }
int kern_dump(void) { vm_map_t map; unsigned int thread_count, segment_count; unsigned int command_size = 0, header_size = 0, tstate_size = 0; uint64_t hoffset = 0, foffset = 0, nfoffset = 0; unsigned int max_header_size = 0; vm_offset_t header, txstart; vm_map_offset_t vmoffset; struct mach_header_64 *mh64; struct segment_command_64 *sc64; mach_vm_size_t size = 0; vm_prot_t prot = 0; vm_prot_t maxprot = 0; mythread_state_flavor_t flavors[MAX_TSTATE_FLAVORS]; vm_size_t nflavors; vm_size_t i; uint32_t nesting_depth = 0; kern_return_t kret = 0; struct vm_region_submap_info_64 vbr; mach_msg_type_number_t vbrcount = 0; tir_t tir1; int error = 0; int panic_error = 0; map = kernel_map; thread_count = 1; segment_count = get_vmmap_entries(map); printf("Kernel map has %d entries\n", segment_count); nflavors = kdp_mynum_flavors; bcopy((char *)thread_flavor_array,(char *) flavors,sizeof(thread_flavor_array)); for (i = 0; i < nflavors; i++) tstate_size += (uint32_t)(sizeof(mythread_state_flavor_t) + (flavors[i].count * sizeof(int))); command_size = (uint32_t)((segment_count) * sizeof(struct segment_command_64) + thread_count * sizeof(struct thread_command) + tstate_size * thread_count); header_size = command_size + (uint32_t)sizeof(struct mach_header_64); header = (vm_offset_t) command_buffer; /* * Set up Mach-O header for currently executing 32 bit kernel. */ printf ("Generated Mach-O header size was %d\n", header_size); mh64 = (struct mach_header_64 *) header; mh64->magic = MH_MAGIC_64; mh64->cputype = cpu_type(); mh64->cpusubtype = cpu_subtype(); mh64->filetype = MH_CORE; mh64->ncmds = segment_count + thread_count; mh64->sizeofcmds = command_size; mh64->flags = 0; mh64->reserved = 0; hoffset = sizeof(struct mach_header_64); /* offset into header */ foffset = (uint32_t)round_page(header_size); /* offset into file */ /* Padding */ if ((foffset - header_size) < (4*sizeof(struct segment_command_64))) { foffset += (uint32_t)((4*sizeof(struct segment_command_64)) - (foffset-header_size)); } max_header_size = (unsigned int)foffset; vmoffset = vm_map_min(map); /* Transmit the Mach-O MH_CORE header, and seek forward past the * area reserved for the segment and thread commands * to begin data transmission */ if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct mach_header_64), (caddr_t) mh64) < 0)) { printf ("kdp_send_crashdump_data failed with error %d\n", panic_error); error = panic_error; goto out; } if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } printf ("Transmitting kernel state, please wait: "); while ((segment_count > 0) || (kret == KERN_SUCCESS)){ while (1) { /* * Get region information for next region. */ vbrcount = VM_REGION_SUBMAP_INFO_COUNT_64; if((kret = mach_vm_region_recurse(map, &vmoffset, &size, &nesting_depth, (vm_region_recurse_info_t)&vbr, &vbrcount)) != KERN_SUCCESS) { break; } if(vbr.is_submap) { nesting_depth++; continue; } else { break; } } if(kret != KERN_SUCCESS) break; prot = vbr.protection; maxprot = vbr.max_protection; /* * Fill in segment command structure. */ if (hoffset > max_header_size) break; sc64 = (struct segment_command_64 *) (header); sc64->cmd = LC_SEGMENT_64; sc64->cmdsize = sizeof(struct segment_command_64); sc64->segname[0] = 0; sc64->vmaddr = vmoffset; sc64->vmsize = size; sc64->fileoff = foffset; sc64->filesize = size; sc64->maxprot = maxprot; sc64->initprot = prot; sc64->nsects = 0; if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct segment_command_64) , (caddr_t) sc64)) < 0) { printf ("kdp_send_crashdump_data failed with error %d\n", panic_error); error = panic_error; goto out; } /* Do not transmit memory tagged VM_MEMORY_IOKIT - instead, * seek past that region on the server - this creates a * hole in the file. */ if ((vbr.user_tag != VM_MEMORY_IOKIT)) { if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } txstart = vmoffset; if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, (unsigned int)size, (caddr_t) txstart)) < 0) { printf ("kdp_send_crashdump_data failed with error %d\n", panic_error); error = panic_error; goto out; } } hoffset += (unsigned int)sizeof(struct segment_command_64); foffset += (unsigned int)size; vmoffset += size; segment_count--; } tir1.header = header; tir1.hoffset = 0; tir1.flavors = flavors; tir1.tstate_size = tstate_size; /* Now send out the LC_THREAD load command, with the thread information * for the current activation. * Note that the corefile can contain LC_SEGMENT commands with file * offsets that point past the edge of the corefile, in the event that * the last N VM regions were all I/O mapped or otherwise * non-transferable memory, not followed by a normal VM region; * i.e. there will be no hole that reaches to the end of the core file. */ kern_collectth_state (current_thread(), &tir1); if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, tir1.hoffset , (caddr_t) header)) < 0) { printf ("kdp_send_crashdump_data failed with error %d\n", panic_error); error = panic_error; goto out; } /* last packet */ if ((panic_error = kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0))) < 0) { printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); error = panic_error; goto out; } out: return (error); }
/* * Hold each of the physical pages that are mapped by the specified range of * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid * and allow the specified types of access, "prot". If all of the implied * pages are successfully held, then the number of held pages is returned * together with pointers to those pages in the array "ma". However, if any * of the pages cannot be held, -1 is returned. */ int vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, vm_prot_t prot, vm_page_t *ma, int max_count) { vm_offset_t end, va; vm_page_t *mp; int count; boolean_t pmap_failed; if (len == 0) return (0); end = round_page(addr + len); addr = trunc_page(addr); /* * Check for illegal addresses. */ if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map)) return (-1); count = howmany(end - addr, PAGE_SIZE); if (count > max_count) panic("vm_fault_quick_hold_pages: count > max_count"); /* * Most likely, the physical pages are resident in the pmap, so it is * faster to try pmap_extract_and_hold() first. */ pmap_failed = FALSE; for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) { *mp = pmap_extract_and_hold(map->pmap, va, prot); if (*mp == NULL) pmap_failed = TRUE; else if ((prot & VM_PROT_WRITE) != 0 && (*mp)->dirty != VM_PAGE_BITS_ALL) { /* * Explicitly dirty the physical page. Otherwise, the * caller's changes may go unnoticed because they are * performed through an unmanaged mapping or by a DMA * operation. * * The object lock is not held here. * See vm_page_clear_dirty_mask(). */ vm_page_dirty(*mp); } } if (pmap_failed) { /* * One or more pages could not be held by the pmap. Either no * page was mapped at the specified virtual address or that * mapping had insufficient permissions. Attempt to fault in * and hold these pages. */ for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) if (*mp == NULL && vm_fault_hold(map, va, prot, VM_FAULT_NORMAL, mp) != KERN_SUCCESS) goto error; } return (count); error: for (mp = ma; mp < ma + count; mp++) if (*mp != NULL) { vm_page_lock(*mp); vm_page_unhold(*mp); vm_page_unlock(*mp); } return (-1); }
static int link_elf_load_file(linker_class_t cls, const char *filename, linker_file_t *result) { struct nameidata nd; struct thread *td = curthread; /* XXX */ Elf_Ehdr *hdr; Elf_Shdr *shdr; Elf_Sym *es; int nbytes, i, j; vm_offset_t mapbase; size_t mapsize; int error = 0; ssize_t resid; int flags; elf_file_t ef; linker_file_t lf; int symtabindex; int symstrindex; int shstrindex; int nsym; int pb, rl, ra; int alignmask; shdr = NULL; lf = NULL; mapsize = 0; hdr = NULL; NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td); flags = FREAD; error = vn_open(&nd, &flags, 0, NULL); if (error) return error; NDFREE(&nd, NDF_ONLY_PNBUF); if (nd.ni_vp->v_type != VREG) { error = ENOEXEC; goto out; } #ifdef MAC error = mac_kld_check_load(td->td_ucred, nd.ni_vp); if (error) { goto out; } #endif /* Read the elf header from the file. */ hdr = malloc(sizeof(*hdr), M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)hdr, sizeof(*hdr), 0, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = ENOEXEC; goto out; } if (!IS_ELF(*hdr)) { error = ENOEXEC; goto out; } if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { link_elf_error(filename, "Unsupported file layout"); error = ENOEXEC; goto out; } if (hdr->e_ident[EI_VERSION] != EV_CURRENT || hdr->e_version != EV_CURRENT) { link_elf_error(filename, "Unsupported file version"); error = ENOEXEC; goto out; } if (hdr->e_type != ET_REL) { error = ENOSYS; goto out; } if (hdr->e_machine != ELF_TARG_MACH) { link_elf_error(filename, "Unsupported machine"); error = ENOEXEC; goto out; } lf = linker_make_file(filename, &link_elf_class); if (!lf) { error = ENOMEM; goto out; } ef = (elf_file_t) lf; ef->nprogtab = 0; ef->e_shdr = 0; ef->nreltab = 0; ef->nrelatab = 0; /* Allocate and read in the section header */ nbytes = hdr->e_shnum * hdr->e_shentsize; if (nbytes == 0 || hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { error = ENOEXEC; goto out; } shdr = malloc(nbytes, M_LINKER, M_WAITOK); ef->e_shdr = shdr; error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shdr, nbytes, hdr->e_shoff, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid) { error = ENOEXEC; goto out; } /* Scan the section header for information and table sizing. */ nsym = 0; symtabindex = -1; symstrindex = -1; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: ef->nprogtab++; break; case SHT_SYMTAB: nsym++; symtabindex = i; symstrindex = shdr[i].sh_link; break; case SHT_REL: ef->nreltab++; break; case SHT_RELA: ef->nrelatab++; break; case SHT_STRTAB: break; } } if (ef->nprogtab == 0) { link_elf_error(filename, "file has no contents"); error = ENOEXEC; goto out; } if (nsym != 1) { /* Only allow one symbol table for now */ link_elf_error(filename, "file has no valid symbol table"); error = ENOEXEC; goto out; } if (symstrindex < 0 || symstrindex > hdr->e_shnum || shdr[symstrindex].sh_type != SHT_STRTAB) { link_elf_error(filename, "file has invalid symbol strings"); error = ENOEXEC; goto out; } /* Allocate space for tracking the load chunks */ if (ef->nprogtab != 0) ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab), M_LINKER, M_WAITOK | M_ZERO); if (ef->nreltab != 0) ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab), M_LINKER, M_WAITOK | M_ZERO); if (ef->nrelatab != 0) ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab), M_LINKER, M_WAITOK | M_ZERO); if (symtabindex == -1) panic("lost symbol table index"); /* Allocate space for and load the symbol table */ ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); ef->ddbsymtab = malloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->ddbsymtab, shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } if (symstrindex == -1) panic("lost symbol string index"); /* Allocate space for and load the symbol strings */ ef->ddbstrcnt = shdr[symstrindex].sh_size; ef->ddbstrtab = malloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, nd.ni_vp, ef->ddbstrtab, shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } /* Do we have a string table for the section names? */ shstrindex = -1; if (hdr->e_shstrndx != 0 && shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { shstrindex = hdr->e_shstrndx; ef->shstrcnt = shdr[shstrindex].sh_size; ef->shstrtab = malloc(shdr[shstrindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, nd.ni_vp, ef->shstrtab, shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } } /* Size up code/data(progbits) and bss(nobits). */ alignmask = 0; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: alignmask = shdr[i].sh_addralign - 1; mapsize += alignmask; mapsize &= ~alignmask; mapsize += shdr[i].sh_size; break; } } /* * We know how much space we need for the text/data/bss/etc. * This stuff needs to be in a single chunk so that profiling etc * can get the bounds and gdb can associate offsets with modules */ ef->object = vm_object_allocate(OBJT_DEFAULT, round_page(mapsize) >> PAGE_SHIFT); if (ef->object == NULL) { error = ENOMEM; goto out; } ef->address = (caddr_t) vm_map_min(kernel_map); /* * In order to satisfy amd64's architectural requirements on the * location of code and data in the kernel's address space, request a * mapping that is above the kernel. */ #ifdef __amd64__ mapbase = KERNBASE; #else mapbase = VM_MIN_KERNEL_ADDRESS; #endif error = vm_map_find(kernel_map, ef->object, 0, &mapbase, round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL, VM_PROT_ALL, 0); if (error) { vm_object_deallocate(ef->object); ef->object = 0; goto out; } /* Wire the pages */ error = vm_map_wire(kernel_map, mapbase, mapbase + round_page(mapsize), VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES); if (error != KERN_SUCCESS) { error = ENOMEM; goto out; } /* Inform the kld system about the situation */ lf->address = ef->address = (caddr_t)mapbase; lf->size = mapsize; /* * Now load code/data(progbits), zero bss(nobits), allocate space for * and load relocs */ pb = 0; rl = 0; ra = 0; alignmask = 0; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: alignmask = shdr[i].sh_addralign - 1; mapbase += alignmask; mapbase &= ~alignmask; if (ef->shstrtab && shdr[i].sh_name != 0) ef->progtab[pb].name = ef->shstrtab + shdr[i].sh_name; else if (shdr[i].sh_type == SHT_PROGBITS) ef->progtab[pb].name = "<<PROGBITS>>"; else ef->progtab[pb].name = "<<NOBITS>>"; if (ef->progtab[pb].name != NULL && !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) ef->progtab[pb].addr = dpcpu_alloc(shdr[i].sh_size); #ifdef VIMAGE else if (ef->progtab[pb].name != NULL && !strcmp(ef->progtab[pb].name, VNET_SETNAME)) ef->progtab[pb].addr = vnet_data_alloc(shdr[i].sh_size); #endif else ef->progtab[pb].addr = (void *)(uintptr_t)mapbase; if (ef->progtab[pb].addr == NULL) { error = ENOSPC; goto out; } ef->progtab[pb].size = shdr[i].sh_size; ef->progtab[pb].sec = i; if (shdr[i].sh_type == SHT_PROGBITS) { error = vn_rdwr(UIO_READ, nd.ni_vp, ef->progtab[pb].addr, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } /* Initialize the per-cpu or vnet area. */ if (ef->progtab[pb].addr != (void *)mapbase && !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) dpcpu_copy(ef->progtab[pb].addr, shdr[i].sh_size); #ifdef VIMAGE else if (ef->progtab[pb].addr != (void *)mapbase && !strcmp(ef->progtab[pb].name, VNET_SETNAME)) vnet_data_copy(ef->progtab[pb].addr, shdr[i].sh_size); #endif } else bzero(ef->progtab[pb].addr, shdr[i].sh_size); /* Update all symbol values with the offset. */ for (j = 0; j < ef->ddbsymcnt; j++) { es = &ef->ddbsymtab[j]; if (es->st_shndx != i) continue; es->st_value += (Elf_Addr)ef->progtab[pb].addr; } mapbase += shdr[i].sh_size; pb++; break; case SHT_REL: ef->reltab[rl].rel = malloc(shdr[i].sh_size, M_LINKER, M_WAITOK); ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel); ef->reltab[rl].sec = shdr[i].sh_info; error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->reltab[rl].rel, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } rl++; break; case SHT_RELA: ef->relatab[ra].rela = malloc(shdr[i].sh_size, M_LINKER, M_WAITOK); ef->relatab[ra].nrela = shdr[i].sh_size / sizeof(Elf_Rela); ef->relatab[ra].sec = shdr[i].sh_info; error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->relatab[ra].rela, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td); if (error) goto out; if (resid != 0){ error = EINVAL; goto out; } ra++; break; } } if (pb != ef->nprogtab) panic("lost progbits"); if (rl != ef->nreltab) panic("lost reltab"); if (ra != ef->nrelatab) panic("lost relatab"); if (mapbase != (vm_offset_t)ef->address + mapsize) panic("mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)\n", (u_long)mapbase, ef->address, (u_long)mapsize, (u_long)(vm_offset_t)ef->address + mapsize); /* Local intra-module relocations */ link_elf_reloc_local(lf); /* Pull in dependencies */ VOP_UNLOCK(nd.ni_vp, 0); error = linker_load_dependencies(lf); vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); if (error) goto out; /* External relocations */ error = relocate_file(ef); if (error) goto out; /* Notify MD code that a module is being loaded. */ error = elf_cpu_load_file(lf); if (error) goto out; *result = lf; out: VOP_UNLOCK(nd.ni_vp, 0); vn_close(nd.ni_vp, FREAD, td->td_ucred, td); if (error && lf) linker_file_unload(lf, LINKER_UNLOAD_FORCE); if (hdr) free(hdr, M_LINKER); return error; }
/* * Machine-dependent startup code */ void cpu_startup() { caddr_t v; int sz; #ifdef DEBUG extern int pmapdebug; int opmapdebug = pmapdebug; #endif vaddr_t minaddr, maxaddr; extern struct user *proc0paddr; #ifdef DEBUG pmapdebug = 0; #endif if (CPU_ISSUN4M) { extern int stackgap_random; stackgap_random = STACKGAP_RANDOM_SUN4M; } /* * fix message buffer mapping, note phys addr of msgbuf is 0 */ pmap_map(MSGBUF_VA, 0, MSGBUFSIZE, VM_PROT_READ|VM_PROT_WRITE); initmsgbuf((caddr_t)(MSGBUF_VA + (CPU_ISSUN4 ? 4096 : 0)), MSGBUFSIZE); proc0.p_addr = proc0paddr; /* * Good {morning,afternoon,evening,night}. */ printf(version); /*identifycpu();*/ printf("real mem = %u (%uMB)\n", ptoa(physmem), ptoa(physmem)/1024/1024); /* * Find out how much space we need, allocate it, * and then give everything true virtual addresses. */ sz = (int)allocsys((caddr_t)0); if ((v = (caddr_t)uvm_km_alloc(kernel_map, round_page(sz))) == 0) panic("startup: no room for tables"); if (allocsys(v) - v != sz) panic("startup: table size inconsistency"); /* * Determine how many buffers to allocate. * We allocate bufcachepercent% of memory for buffer space. */ if (bufpages == 0) bufpages = physmem * bufcachepercent / 100; /* Restrict to at most 25% filled kvm */ if (bufpages > (VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE / 4) bufpages = (VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE / 4; /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16*NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* * Allocate a map for physio. Others use a submap of the kernel * map, but we want one completely separate, even though it uses * the same pmap. */ dvma_base = CPU_ISSUN4M ? DVMA4M_BASE : DVMA_BASE; dvma_end = CPU_ISSUN4M ? DVMA4M_END : DVMA_END; #if defined(SUN4M) if (CPU_ISSUN4M) { /* * The DVMA space we want partially overrides kernel_map. * Allocate it in kernel_map as well to prevent it from being * used for other things. */ if (uvm_map(kernel_map, &dvma_base, vm_map_max(kernel_map) - dvma_base, NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_NORMAL, 0))) panic("startup: can not steal dvma map"); } #endif phys_map = uvm_map_create(pmap_kernel(), dvma_base, dvma_end, VM_MAP_INTRSAFE); if (phys_map == NULL) panic("unable to create DVMA map"); /* * Allocate DVMA space and dump into a privately managed * resource map for double mappings which is usable from * interrupt contexts. */ if (uvm_km_valloc_wait(phys_map, (dvma_end-dvma_base)) != dvma_base) panic("unable to allocate from DVMA map"); dvmamap_extent = extent_create("dvmamap", dvma_base, dvma_end, M_DEVBUF, NULL, 0, EX_NOWAIT); if (dvmamap_extent == 0) panic("unable to allocate extent for dvma"); #ifdef DEBUG pmapdebug = opmapdebug; #endif printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free)/1024/1024); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); /* Early interrupt handlers initialization */ intr_init(); }
DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment, unsigned fProt, size_t offSub, size_t cbSub) { // AssertMsgReturn(!offSub && !cbSub, ("%#x %#x\n", offSub, cbSub), VERR_NOT_SUPPORTED); AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED); /* * Check that the specified alignment is supported. */ if (uAlignment > PAGE_SIZE) return VERR_NOT_SUPPORTED; int rc; PRTR0MEMOBJFREEBSD pMemToMapFreeBSD = (PRTR0MEMOBJFREEBSD)pMemToMap; /* calc protection */ vm_prot_t ProtectionFlags = 0; if ((fProt & RTMEM_PROT_NONE) == RTMEM_PROT_NONE) ProtectionFlags = VM_PROT_NONE; if ((fProt & RTMEM_PROT_READ) == RTMEM_PROT_READ) ProtectionFlags |= VM_PROT_READ; if ((fProt & RTMEM_PROT_WRITE) == RTMEM_PROT_WRITE) ProtectionFlags |= VM_PROT_WRITE; if ((fProt & RTMEM_PROT_EXEC) == RTMEM_PROT_EXEC) ProtectionFlags |= VM_PROT_EXECUTE; vm_offset_t Addr = vm_map_min(kernel_map); if (cbSub == 0) cbSub = pMemToMap->cb - offSub; vm_object_reference(pMemToMapFreeBSD->pObject); rc = vm_map_find(kernel_map, /* Map to insert the object in */ pMemToMapFreeBSD->pObject, /* Object to map */ offSub, /* Start offset in the object */ &Addr, /* Start address IN/OUT */ cbSub, /* Size of the mapping */ #if __FreeBSD_version >= 1000055 0, /* Upper bound of mapping */ #endif VMFS_ANY_SPACE, /* Whether a suitable address should be searched for first */ ProtectionFlags, /* protection flags */ VM_PROT_ALL, /* Maximum protection flags */ 0); /* copy-on-write and similar flags */ if (rc == KERN_SUCCESS) { rc = vm_map_wire(kernel_map, Addr, Addr + cbSub, VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES); AssertMsg(rc == KERN_SUCCESS, ("%#x\n", rc)); PRTR0MEMOBJFREEBSD pMemFreeBSD = (PRTR0MEMOBJFREEBSD)rtR0MemObjNew(sizeof(RTR0MEMOBJFREEBSD), RTR0MEMOBJTYPE_MAPPING, (void *)Addr, cbSub); if (pMemFreeBSD) { Assert((vm_offset_t)pMemFreeBSD->Core.pv == Addr); pMemFreeBSD->Core.u.Mapping.R0Process = NIL_RTR0PROCESS; *ppMem = &pMemFreeBSD->Core; return VINF_SUCCESS; } rc = vm_map_remove(kernel_map, Addr, Addr + cbSub); AssertMsg(rc == KERN_SUCCESS, ("Deleting mapping failed\n")); } else vm_object_deallocate(pMemToMapFreeBSD->pObject); return VERR_NO_MEMORY; }
/* * Machine-dependent startup code */ void cpu_startup() { #ifdef DEBUG extern int pmapdebug; int opmapdebug = pmapdebug; #endif vaddr_t minaddr, maxaddr; paddr_t msgbufpa; extern struct user *proc0paddr; #ifdef DEBUG pmapdebug = 0; #endif if (CPU_ISSUN4M) stackgap_random = STACKGAP_RANDOM_SUN4M; /* * Re-map the message buffer from its temporary address * at KERNBASE to MSGBUF_VA. */ /* Get physical address of the message buffer */ pmap_extract(pmap_kernel(), (vaddr_t)KERNBASE, &msgbufpa); /* Invalidate the current mapping at KERNBASE. */ pmap_kremove((vaddr_t)KERNBASE, PAGE_SIZE); pmap_update(pmap_kernel()); /* Enter the new mapping */ pmap_map(MSGBUF_VA, msgbufpa, msgbufpa + PAGE_SIZE, PROT_READ | PROT_WRITE); /* Re-initialize the message buffer. */ initmsgbuf((caddr_t)(MSGBUF_VA + (CPU_ISSUN4 ? 4096 : 0)), MSGBUFSIZE); proc0.p_addr = proc0paddr; /* * Good {morning,afternoon,evening,night}. */ printf(version); /*identifycpu();*/ printf("real mem = %lu (%luMB)\n", ptoa(physmem), ptoa(physmem)/1024/1024); /* * uvm_km_init() has allocated all the virtual memory below the * end of the kernel image. If VM_MIN_KERNEL_ADDRESS is below * KERNBASE, we need to reclaim that range. */ if (vm_min_kernel_address < (vaddr_t)KERNBASE) { uvm_unmap(kernel_map, vm_min_kernel_address, (vaddr_t)KERNBASE); } /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16*NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* * Set up userland PIE limits. PIE is disabled on sun4/4c/4e due * to the limited address space. */ if (CPU_ISSUN4M) { vm_pie_max_addr = VM_MAXUSER_ADDRESS / 4; } dvma_init(); #ifdef DEBUG pmapdebug = opmapdebug; #endif printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free)/1024/1024); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); }
/* * Destroy old address space, and allocate a new stack. * The new stack is only sgrowsiz large because it is grown * automatically on a page fault. */ int exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv) { int error; struct proc *p = imgp->proc; struct vmspace *vmspace = p->p_vmspace; vm_object_t obj; struct rlimit rlim_stack; vm_offset_t sv_minuser, stack_addr; vm_map_t map; u_long ssiz; imgp->vmspace_destroyed = 1; imgp->sysent = sv; /* May be called with Giant held */ EVENTHANDLER_DIRECT_INVOKE(process_exec, p, imgp); /* * Blow away entire process VM, if address space not shared, * otherwise, create a new VM space so that other threads are * not disrupted */ map = &vmspace->vm_map; if (map_at_zero) sv_minuser = sv->sv_minuser; else sv_minuser = MAX(sv->sv_minuser, PAGE_SIZE); if (vmspace->vm_refcnt == 1 && vm_map_min(map) == sv_minuser && vm_map_max(map) == sv->sv_maxuser && cpu_exec_vmspace_reuse(p, map)) { shmexit(vmspace); pmap_remove_pages(vmspace_pmap(vmspace)); vm_map_remove(map, vm_map_min(map), vm_map_max(map)); /* * An exec terminates mlockall(MCL_FUTURE), ASLR state * must be re-evaluated. */ vm_map_lock(map); vm_map_modflags(map, 0, MAP_WIREFUTURE | MAP_ASLR | MAP_ASLR_IGNSTART); vm_map_unlock(map); } else { error = vmspace_exec(p, sv_minuser, sv->sv_maxuser); if (error) return (error); vmspace = p->p_vmspace; map = &vmspace->vm_map; } map->flags |= imgp->map_flags; /* Map a shared page */ obj = sv->sv_shared_page_obj; if (obj != NULL) { vm_object_reference(obj); error = vm_map_fixed(map, obj, 0, sv->sv_shared_page_base, sv->sv_shared_page_len, VM_PROT_READ | VM_PROT_EXECUTE, VM_PROT_READ | VM_PROT_EXECUTE, MAP_INHERIT_SHARE | MAP_ACC_NO_CHARGE); if (error != KERN_SUCCESS) { vm_object_deallocate(obj); return (vm_mmap_to_errno(error)); } } /* Allocate a new stack */ if (imgp->stack_sz != 0) { ssiz = trunc_page(imgp->stack_sz); PROC_LOCK(p); lim_rlimit_proc(p, RLIMIT_STACK, &rlim_stack); PROC_UNLOCK(p); if (ssiz > rlim_stack.rlim_max) ssiz = rlim_stack.rlim_max; if (ssiz > rlim_stack.rlim_cur) { rlim_stack.rlim_cur = ssiz; kern_setrlimit(curthread, RLIMIT_STACK, &rlim_stack); } } else if (sv->sv_maxssiz != NULL) { ssiz = *sv->sv_maxssiz; } else { ssiz = maxssiz; } stack_addr = sv->sv_usrstack - ssiz; error = vm_map_stack(map, stack_addr, (vm_size_t)ssiz, obj != NULL && imgp->stack_prot != 0 ? imgp->stack_prot : sv->sv_stackprot, VM_PROT_ALL, MAP_STACK_GROWS_DOWN); if (error != KERN_SUCCESS) return (vm_mmap_to_errno(error)); /* * vm_ssize and vm_maxsaddr are somewhat antiquated concepts, but they * are still used to enforce the stack rlimit on the process stack. */ vmspace->vm_ssize = sgrowsiz >> PAGE_SHIFT; vmspace->vm_maxsaddr = (char *)stack_addr; return (0); }
/* * cpu_startup: allocate memory for variable-sized tables, initialize CPU, and * do auto-configuration. */ void cpu_startup() { vaddr_t minaddr, maxaddr; #ifdef PMAPDEBUG extern int pmapdebug; int opmapdebug = pmapdebug; pmapdebug = 0; /* Shut up pmap debug during bootstrap. */ #endif /* * Good {morning,afternoon,evening,night}. */ printf(version); printf("real mem = %u (%uMB)\n", ptoa(physmem), ptoa(physmem)/1024/1024); printf("rsvd mem = %u (%uMB)\n", ptoa(rsvdmem), ptoa(rsvdmem)/1024/1024); /* * Determine how many buffers to allocate. * We allocate bufcachepercent% of memory for buffer space. */ if (bufpages == 0) bufpages = physmem * bufcachepercent / 100; /* Restrict to at most 25% filled kvm. */ if (bufpages > (VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE / 4) bufpages = (VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE / 4; /* * Allocate a submap for exec arguments. This map effectively * limits the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); /* Allocate a submap for physio. */ phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, VM_PHYS_SIZE, 0, FALSE, NULL); #ifdef PMAPDEBUG pmapdebug = opmapdebug; #endif printf("avail mem = %u (%uMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free)/1024/1024); extent_malloc_flags = EX_MALLOCOK; /* * Set up CPU-specific registers, cache, etc. */ initcpu(); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); /* * Configure the system. */ if (boothowto & RB_CONFIG) { #ifdef BOOT_CONFIG user_config(); #else printf("kernel does not support -c; continuing..\n"); #endif } }
static int link_elf_obj_load_file(const char *filename, linker_file_t * result) { struct nlookupdata nd; struct thread *td = curthread; /* XXX */ struct proc *p = td->td_proc; char *pathname; struct vnode *vp; Elf_Ehdr *hdr; Elf_Shdr *shdr; Elf_Sym *es; int nbytes, i, j; vm_offset_t mapbase; size_t mapsize; int error = 0; int resid; elf_file_t ef; linker_file_t lf; int symtabindex; int symstrindex; int shstrindex; int nsym; int pb, rl, ra; int alignmask; /* XXX Hack for firmware loading where p == NULL */ if (p == NULL) { p = &proc0; } KKASSERT(p != NULL); if (p->p_ucred == NULL) { kprintf("link_elf_obj_load_file: cannot load '%s' from filesystem" " this early\n", filename); return ENOENT; } shdr = NULL; lf = NULL; mapsize = 0; hdr = NULL; pathname = linker_search_path(filename); if (pathname == NULL) return ENOENT; error = nlookup_init(&nd, pathname, UIO_SYSSPACE, NLC_FOLLOW | NLC_LOCKVP); if (error == 0) error = vn_open(&nd, NULL, FREAD, 0); kfree(pathname, M_LINKER); if (error) { nlookup_done(&nd); return error; } vp = nd.nl_open_vp; nd.nl_open_vp = NULL; nlookup_done(&nd); /* * Read the elf header from the file. */ hdr = kmalloc(sizeof(*hdr), M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, vp, (void *)hdr, sizeof(*hdr), 0, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = ENOEXEC; goto out; } if (!IS_ELF(*hdr)) { error = ENOEXEC; goto out; } if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { link_elf_obj_error(filename, "Unsupported file layout"); error = ENOEXEC; goto out; } if (hdr->e_ident[EI_VERSION] != EV_CURRENT || hdr->e_version != EV_CURRENT) { link_elf_obj_error(filename, "Unsupported file version"); error = ENOEXEC; goto out; } if (hdr->e_type != ET_REL) { error = ENOSYS; goto out; } if (hdr->e_machine != ELF_TARG_MACH) { link_elf_obj_error(filename, "Unsupported machine"); error = ENOEXEC; goto out; } ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO); lf = linker_make_file(filename, ef, &link_elf_obj_file_ops); if (lf == NULL) { kfree(ef, M_LINKER); error = ENOMEM; goto out; } ef->nprogtab = 0; ef->e_shdr = NULL; ef->nreltab = 0; ef->nrelatab = 0; /* Allocate and read in the section header */ nbytes = hdr->e_shnum * hdr->e_shentsize; if (nbytes == 0 || hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { error = ENOEXEC; goto out; } shdr = kmalloc(nbytes, M_LINKER, M_WAITOK); ef->e_shdr = shdr; error = vn_rdwr(UIO_READ, vp, (caddr_t) shdr, nbytes, hdr->e_shoff, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid) { error = ENOEXEC; goto out; } /* Scan the section header for information and table sizing. */ nsym = 0; symtabindex = -1; symstrindex = -1; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: ef->nprogtab++; break; case SHT_SYMTAB: nsym++; symtabindex = i; symstrindex = shdr[i].sh_link; break; case SHT_REL: ef->nreltab++; break; case SHT_RELA: ef->nrelatab++; break; case SHT_STRTAB: break; } } if (ef->nprogtab == 0) { link_elf_obj_error(filename, "file has no contents"); error = ENOEXEC; goto out; } if (nsym != 1) { /* Only allow one symbol table for now */ link_elf_obj_error(filename, "file has no valid symbol table"); error = ENOEXEC; goto out; } if (symstrindex < 0 || symstrindex > hdr->e_shnum || shdr[symstrindex].sh_type != SHT_STRTAB) { link_elf_obj_error(filename, "file has invalid symbol strings"); error = ENOEXEC; goto out; } /* Allocate space for tracking the load chunks */ if (ef->nprogtab != 0) ef->progtab = kmalloc(ef->nprogtab * sizeof(*ef->progtab), M_LINKER, M_WAITOK | M_ZERO); if (ef->nreltab != 0) ef->reltab = kmalloc(ef->nreltab * sizeof(*ef->reltab), M_LINKER, M_WAITOK | M_ZERO); if (ef->nrelatab != 0) ef->relatab = kmalloc(ef->nrelatab * sizeof(*ef->relatab), M_LINKER, M_WAITOK | M_ZERO); if ((ef->nprogtab != 0 && ef->progtab == NULL) || (ef->nreltab != 0 && ef->reltab == NULL) || (ef->nrelatab != 0 && ef->relatab == NULL)) { error = ENOMEM; goto out; } if (symtabindex == -1) panic("lost symbol table index"); /* Allocate space for and load the symbol table */ ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); ef->ddbsymtab = kmalloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, vp, (void *)ef->ddbsymtab, shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } if (symstrindex == -1) panic("lost symbol string index"); /* Allocate space for and load the symbol strings */ ef->ddbstrcnt = shdr[symstrindex].sh_size; ef->ddbstrtab = kmalloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, vp, ef->ddbstrtab, shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } /* Do we have a string table for the section names? */ shstrindex = -1; if (hdr->e_shstrndx != 0 && shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { shstrindex = hdr->e_shstrndx; ef->shstrcnt = shdr[shstrindex].sh_size; ef->shstrtab = kmalloc(shdr[shstrindex].sh_size, M_LINKER, M_WAITOK); error = vn_rdwr(UIO_READ, vp, ef->shstrtab, shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } } /* Size up code/data(progbits) and bss(nobits). */ alignmask = 0; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: alignmask = shdr[i].sh_addralign - 1; mapsize += alignmask; mapsize &= ~alignmask; mapsize += shdr[i].sh_size; break; } } /* * We know how much space we need for the text/data/bss/etc. This * stuff needs to be in a single chunk so that profiling etc can get * the bounds and gdb can associate offsets with modules */ ef->object = vm_object_allocate(OBJT_DEFAULT, round_page(mapsize) >> PAGE_SHIFT); if (ef->object == NULL) { error = ENOMEM; goto out; } vm_object_hold(ef->object); vm_object_reference_locked(ef->object); ef->address = (caddr_t) vm_map_min(&kernel_map); ef->bytes = 0; /* * In order to satisfy x86_64's architectural requirements on the * location of code and data in the kernel's address space, request a * mapping that is above the kernel. * * vkernel64's text+data is outside the managed VM space entirely. */ #if defined(__x86_64__) && defined(_KERNEL_VIRTUAL) error = vkernel_module_memory_alloc(&mapbase, round_page(mapsize)); vm_object_drop(ef->object); #else mapbase = KERNBASE; error = vm_map_find(&kernel_map, ef->object, NULL, 0, &mapbase, round_page(mapsize), PAGE_SIZE, TRUE, VM_MAPTYPE_NORMAL, VM_PROT_ALL, VM_PROT_ALL, FALSE); vm_object_drop(ef->object); if (error) { vm_object_deallocate(ef->object); ef->object = NULL; goto out; } /* Wire the pages */ error = vm_map_wire(&kernel_map, mapbase, mapbase + round_page(mapsize), 0); #endif if (error != KERN_SUCCESS) { error = ENOMEM; goto out; } /* Inform the kld system about the situation */ lf->address = ef->address = (caddr_t) mapbase; lf->size = round_page(mapsize); ef->bytes = mapsize; /* * Now load code/data(progbits), zero bss(nobits), allocate space for * and load relocs */ pb = 0; rl = 0; ra = 0; alignmask = 0; for (i = 0; i < hdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: alignmask = shdr[i].sh_addralign - 1; mapbase += alignmask; mapbase &= ~alignmask; if (ef->shstrtab && shdr[i].sh_name != 0) ef->progtab[pb].name = ef->shstrtab + shdr[i].sh_name; else if (shdr[i].sh_type == SHT_PROGBITS) ef->progtab[pb].name = "<<PROGBITS>>"; else ef->progtab[pb].name = "<<NOBITS>>"; #if 0 if (ef->progtab[pb].name != NULL && !strcmp(ef->progtab[pb].name, "set_pcpu")) ef->progtab[pb].addr = dpcpu_alloc(shdr[i].sh_size); #ifdef VIMAGE else if (ef->progtab[pb].name != NULL && !strcmp(ef->progtab[pb].name, VNET_SETNAME)) ef->progtab[pb].addr = vnet_data_alloc(shdr[i].sh_size); #endif else #endif ef->progtab[pb].addr = (void *)(uintptr_t) mapbase; if (ef->progtab[pb].addr == NULL) { error = ENOSPC; goto out; } ef->progtab[pb].size = shdr[i].sh_size; ef->progtab[pb].sec = i; if (shdr[i].sh_type == SHT_PROGBITS) { error = vn_rdwr(UIO_READ, vp, ef->progtab[pb].addr, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } #if 0 /* Initialize the per-cpu or vnet area. */ if (ef->progtab[pb].addr != (void *)mapbase && !strcmp(ef->progtab[pb].name, "set_pcpu")) dpcpu_copy(ef->progtab[pb].addr, shdr[i].sh_size); #ifdef VIMAGE else if (ef->progtab[pb].addr != (void *)mapbase && !strcmp(ef->progtab[pb].name, VNET_SETNAME)) vnet_data_copy(ef->progtab[pb].addr, shdr[i].sh_size); #endif #endif } else bzero(ef->progtab[pb].addr, shdr[i].sh_size); /* Update all symbol values with the offset. */ for (j = 0; j < ef->ddbsymcnt; j++) { es = &ef->ddbsymtab[j]; if (es->st_shndx != i) continue; es->st_value += (Elf_Addr) ef->progtab[pb].addr; } mapbase += shdr[i].sh_size; pb++; break; case SHT_REL: ef->reltab[rl].rel = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK); ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel); ef->reltab[rl].sec = shdr[i].sh_info; error = vn_rdwr(UIO_READ, vp, (void *)ef->reltab[rl].rel, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } rl++; break; case SHT_RELA: ef->relatab[ra].rela = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK); ef->relatab[ra].nrela = shdr[i].sh_size / sizeof(Elf_Rela); ef->relatab[ra].sec = shdr[i].sh_info; error = vn_rdwr(UIO_READ, vp, (void *)ef->relatab[ra].rela, shdr[i].sh_size, shdr[i].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid); if (error) goto out; if (resid != 0) { error = EINVAL; goto out; } ra++; break; } } if (pb != ef->nprogtab) panic("lost progbits"); if (rl != ef->nreltab) panic("lost reltab"); if (ra != ef->nrelatab) panic("lost relatab"); if (mapbase != (vm_offset_t) ef->address + mapsize) panic("mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)", mapbase, ef->address, mapsize, (vm_offset_t) ef->address + mapsize); /* Local intra-module relocations */ link_elf_obj_reloc_local(lf); /* Pull in dependencies */ error = linker_load_dependencies(lf); if (error) goto out; /* External relocations */ error = relocate_file(lf); if (error) goto out; *result = lf; out: if (error && lf) linker_file_unload(lf /*, LINKER_UNLOAD_FORCE */); if (hdr) kfree(hdr, M_LINKER); vn_unlock(vp); vn_close(vp, FREAD, NULL); return error; }
void cpu_startup() { vaddr_t minaddr, maxaddr; extern char cpu_model[]; /* * Initialize error message buffer. */ initmsgbuf((caddr_t)msgbufp, round_page(MSGBUFSIZE)); /* * Good {morning,afternoon,evening,night}. * Also call CPU init on systems that need that. */ printf("%s%s [%08X %08X]\n", version, cpu_model, vax_cpudata, vax_siedata); if (dep_call->cpu_conf) (*dep_call->cpu_conf)(); printf("real mem = %u (%uMB)\n", ptoa(physmem), ptoa(physmem)/1024/1024); mtpr(AST_NO, PR_ASTLVL); spl0(); /* * Allocate a submap for exec arguments. This map effectively limits * the number of processes exec'ing at any time. */ minaddr = vm_map_min(kernel_map); exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); #if VAX46 || VAX48 || VAX49 || VAX53 || VAX60 /* * Allocate a submap for physio. This map effectively limits the * number of processes doing physio at any one time. * * Note that machines on which all mass storage I/O controllers * can perform address translation, do not need this. */ if (vax_boardtype == VAX_BTYP_46 || vax_boardtype == VAX_BTYP_48 || vax_boardtype == VAX_BTYP_49 || vax_boardtype == VAX_BTYP_1303 || vax_boardtype == VAX_BTYP_60) phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, VM_PHYS_SIZE, 0, FALSE, NULL); #endif printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), ptoa(uvmexp.free)/1024/1024); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); #ifdef DDB if (boothowto & RB_KDB) Debugger(); #endif /* * Configure the system. */ if (boothowto & RB_CONFIG) { #ifdef BOOT_CONFIG user_config(); #else printf("kernel does not support -c; continuing..\n"); #endif } }