struct dcb *spawn_bsp_init(const char *name) { MSG("spawning '%s' on BSP core\n", name); /* Only the first core can run this code */ assert(cpu_is_bsp()); /* Allocate bootinfo */ lpaddr_t bootinfo_phys = bsp_alloc_phys_aligned(BOOTINFO_SIZE, BASE_PAGE_SIZE); memset((void *)local_phys_to_mem(bootinfo_phys), 0, BOOTINFO_SIZE); /* store pointer to bootinfo in kernel virtual memory */ bootinfo = (struct bootinfo *) local_phys_to_mem(bootinfo_phys); /* Construct cmdline args */ char bootinfochar[16]; snprintf(bootinfochar, sizeof(bootinfochar), "%u", INIT_BOOTINFO_VBASE); const char *argv[] = { "init", bootinfochar }; int argc = 2; /* perform common spawning of init domain */ struct dcb *init_dcb = spawn_init_common(name, argc, argv,bootinfo_phys, bsp_alloc_phys, bsp_alloc_phys_aligned); /* map boot info into init's VSPACE */ spawn_init_map(init_l3, ARMV8_INIT_VBASE, INIT_BOOTINFO_VBASE, bootinfo_phys, BOOTINFO_SIZE, INIT_PERM_RW); /* load the image */ genvaddr_t init_ep, got_base; struct startup_l3_info l3_info = { init_l3, ARMV8_INIT_VBASE }; load_init_image(&l3_info, BSP_INIT_MODULE_NAME, &init_ep, &got_base); MSG("init loaded with entry=0x%" PRIxGENVADDR " and GOT=0x%" PRIxGENVADDR "\n", init_ep, got_base); struct dispatcher_shared_aarch64 *disp_aarch64 = get_dispatcher_shared_aarch64(init_dcb->disp); /* setting GOT pointers */ disp_aarch64->got_base = got_base; /* XXX - Why does the kernel do this? -DC */ disp_aarch64->enabled_save_area.named.x10 = got_base; disp_aarch64->disabled_save_area.named.x10 = got_base; /* setting entry points */ disp_aarch64->disabled_save_area.named.pc = init_ep; disp_aarch64->disabled_save_area.named.spsr = AARCH64_MODE_USR | CPSR_F_MASK; /* Create caps for init to use */ create_module_caps(&spawn_state); lpaddr_t init_alloc_end = bsp_alloc_phys(0); create_phys_caps(armv8_glbl_core_data->start_kernel_ram, init_alloc_end); /* Fill bootinfo struct */ bootinfo->mem_spawn_core = KERNEL_IMAGE_SIZE; // Size of kernel return init_dcb; }
/** * \brief Map init user-space memory. * * This function maps pages of the init user-space module. It expects * the virtual base address 'vbase' of a program segment of the init executable, * its size 'size' and its ELF64 access control flags. It maps pages * into physical memory that is allocated on the fly and puts * corresponding frame caps into init's segcn. * * \param vbase Virtual base address of program segment. * \param size Size of program segment in bytes. * \param flags ELF64 access control flags of program segment. * \param ret Used to return base region pointer */ errval_t startup_alloc_init(void *state, genvaddr_t gvbase, size_t size, uint32_t flags, void **ret) { errval_t err; struct spawn_state *spawn_state = state; lvaddr_t vbase = (lvaddr_t)gvbase; /* XXX */ lvaddr_t offset = BASE_PAGE_OFFSET(vbase); /* Page align the parameters */ paging_align(&vbase, NULL, &size, BASE_PAGE_SIZE); lpaddr_t pbase = 0, paddr = 0; for(lvaddr_t i = vbase; i < vbase + size; i += BASE_PAGE_SIZE) { if (apic_is_bsp()) { paddr = bsp_alloc_phys(BASE_PAGE_SIZE); } else { paddr = app_alloc_phys(BASE_PAGE_SIZE); } if(pbase == 0) { pbase = paddr; } err = startup_map_init(i, paddr, BASE_PAGE_SIZE, flags); assert(err_is_ok(err)); } if (apic_is_bsp()) { // Create frame caps for segcn paddr += BASE_PAGE_SIZE; debug(SUBSYS_STARTUP, "Allocated physical memory [0x%"PRIxLPADDR", 0x%"PRIxLPADDR"]\n", pbase, paddr - pbase); err = create_caps_to_cnode(pbase, paddr - pbase, RegionType_RootTask, spawn_state, bootinfo); if (err_is_fail(err)) { return err; } } assert(ret != NULL); *ret = (void *)(vbase + offset); return SYS_ERR_OK; }
/// Setup the module cnode, which contains frame caps to all multiboot modules void create_module_caps(struct spawn_state *st) { errval_t err; /* Create caps for multiboot modules */ struct multiboot_modinfo *module = (struct multiboot_modinfo *)local_phys_to_mem(glbl_core_data->mods_addr); // Allocate strings area lpaddr_t mmstrings_phys = bsp_alloc_phys(BASE_PAGE_SIZE); lvaddr_t mmstrings_base = local_phys_to_mem(mmstrings_phys); lvaddr_t mmstrings = mmstrings_base; // create cap for strings area in first slot of modulecn assert(st->modulecn_slot == 0); err = caps_create_new(ObjType_Frame, mmstrings_phys, BASE_PAGE_BITS, BASE_PAGE_BITS, caps_locate_slot(CNODE(st->modulecn), st->modulecn_slot++)); assert(err_is_ok(err)); /* Walk over multiboot modules, creating frame caps */ for (int i = 0; i < glbl_core_data->mods_count; i++) { struct multiboot_modinfo *m = &module[i]; // Set memory regions within bootinfo struct mem_region *region = &bootinfo->regions[bootinfo->regions_length++]; genpaddr_t remain = MULTIBOOT_MODULE_SIZE(*m); genpaddr_t base_addr = local_phys_to_gen_phys(m->mod_start); region->mr_type = RegionType_Module; region->mr_base = base_addr; region->mrmod_slot = st->modulecn_slot; // first slot containing caps region->mrmod_size = remain; // size of image _in bytes_ region->mrmod_data = mmstrings - mmstrings_base; // offset of string in area // round up to page size for caps remain = ROUND_UP(remain, BASE_PAGE_SIZE); // Create max-sized caps to multiboot module in module cnode while (remain > 0) { assert((base_addr & BASE_PAGE_MASK) == 0); assert((remain & BASE_PAGE_MASK) == 0); // determine size of next chunk uint8_t block_size = bitaddralign(remain, base_addr); assert(st->modulecn_slot < (1UL << st->modulecn->cap.u.cnode.bits)); // create as DevFrame cap to avoid zeroing memory contents err = caps_create_new(ObjType_DevFrame, base_addr, block_size, block_size, caps_locate_slot(CNODE(st->modulecn), st->modulecn_slot++)); assert(err_is_ok(err)); // Advance by that chunk base_addr += ((genpaddr_t)1 << block_size); remain -= ((genpaddr_t)1 << block_size); } // Copy multiboot module string to mmstrings area strcpy((char *)mmstrings, MBADDR_ASSTRING(m->string)); mmstrings += strlen(MBADDR_ASSTRING(m->string)) + 1; assert(mmstrings < mmstrings_base + BASE_PAGE_SIZE); } }
static lpaddr_t bsp_alloc_phys_aligned(size_t size, size_t align) { bsp_init_alloc_addr = round_up(bsp_init_alloc_addr, align); return bsp_alloc_phys(size); }
void arm_kernel_startup(void *pointer) { /* Initialize the core_data */ /* Used when bringing up other cores, must be at consistent global address * seen by all cores */ // Initialize system timers timers_init(config_timeslice); struct dcb *init_dcb; if (cpu_is_bsp()) { MSG("Doing BSP related bootup \n"); /* Initialize the location to allocate phys memory from */ printf("start_free_ram = 0x%lx\n", armv8_glbl_core_data->start_free_ram); bsp_init_alloc_addr = armv8_glbl_core_data->start_free_ram; /* allocate initial KCB */ kcb_current= (struct kcb *)local_phys_to_mem( bsp_alloc_phys(sizeof(*kcb_current))); assert(kcb_current); memset(kcb_current, 0, sizeof(*kcb_current)); init_dcb = spawn_bsp_init(BSP_INIT_MODULE_NAME); platform_gic_init(); // pit_start(0); } else { MSG("Doing non-BSP related bootup \n"); struct armv8_core_data *core_data = (struct armv8_core_data *)pointer; my_core_id = core_data->dst_core_id; /* Initialize the allocator */ app_alloc_phys_start = (core_data->memory.base); app_alloc_phys_end = (core_data->memory.length + app_alloc_phys_start); MSG("Memory: %lx, %lx, size=%zu kB\n", app_alloc_phys_start, app_alloc_phys_end, (app_alloc_phys_end - app_alloc_phys_start + 1) >> 10); kcb_current= (struct kcb *)local_phys_to_mem(core_data->kcb); init_dcb = spawn_app_init(core_data, APP_INIT_MODULE_NAME); // uint32_t irq = gic_get_active_irq(); // gic_ack_irq(irq); } // enable interrupt forwarding to cpu platform_gic_cpu_interface_enable(); MSG("Calling dispatch from arm_kernel_startup, entry point %#"PRIxLVADDR"\n", get_dispatcher_shared_aarch64(init_dcb->disp)->disabled_save_area.named.pc); // Should not return dispatch(init_dcb); panic("Error spawning init!"); }
/// Setup the module cnode, which contains frame caps to all multiboot modules void create_module_caps(struct spawn_state *st) { errval_t err; /* Create caps for multiboot modules */ struct multiboot_header_tag *multiboot = (struct multiboot_header_tag *)local_phys_to_mem(armv8_glbl_core_data->multiboot_image.base); // Allocate strings area lpaddr_t mmstrings_phys = bsp_alloc_phys(BASE_PAGE_SIZE); lvaddr_t mmstrings_base = local_phys_to_mem(mmstrings_phys); lvaddr_t mmstrings = mmstrings_base; // create cap for strings area in first slot of modulecn assert(st->modulecn_slot == 0); err = caps_create_new(ObjType_Frame, mmstrings_phys, BASE_PAGE_SIZE, BASE_PAGE_SIZE, my_core_id, caps_locate_slot(CNODE(st->modulecn), st->modulecn_slot++)); assert(err_is_ok(err)); //Nag bootinfo->regions_length = 0; /* Walk over multiboot modules, creating frame caps */ size_t position = 0; size_t size = armv8_glbl_core_data->multiboot_image.length; struct mem_region *region; lpaddr_t acpi_base = (lpaddr_t)-1; /* add the ACPI regions */ struct multiboot_tag_new_acpi *acpi_new; acpi_new = (struct multiboot_tag_new_acpi *) multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_ACPI_NEW); if (acpi_new) { acpi_base = mem_to_local_phys((lvaddr_t)&acpi_new->rsdp[0]); } else { struct multiboot_tag_old_acpi *acpi_old; acpi_old = (struct multiboot_tag_old_acpi *) multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_ACPI_OLD); if (acpi_old) { acpi_base = mem_to_local_phys((lvaddr_t)&acpi_old->rsdp[0]); } } if (acpi_base != (lpaddr_t)-1) { region = &bootinfo->regions[bootinfo->regions_length++]; region->mr_base = acpi_base; region->mr_type = RegionType_ACPI_TABLE; } /* add the module regions */ position = 0; struct multiboot_tag_module_64 *module = (struct multiboot_tag_module_64 *) multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_MODULE_64); while (module) { // Set memory regions within bootinfo region = &bootinfo->regions[bootinfo->regions_length++]; genpaddr_t remain = module->mod_end - module->mod_start; genpaddr_t base_addr = local_phys_to_gen_phys(module->mod_start); region->mr_type = RegionType_Module; region->mr_base = base_addr; region->mrmod_slot = st->modulecn_slot; // first slot containing caps region->mrmod_size = remain; // size of image _in bytes_ region->mrmod_data = mmstrings - mmstrings_base; // offset of string in area // round up to page size for caps remain = ROUND_UP(remain, BASE_PAGE_SIZE); assert((base_addr & BASE_PAGE_MASK) == 0); assert((remain & BASE_PAGE_MASK) == 0); assert(st->modulecn_slot < cnode_get_slots(&st->modulecn->cap)); // create as DevFrame cap to avoid zeroing memory contents err = caps_create_new(ObjType_DevFrame, base_addr, remain, remain, my_core_id, caps_locate_slot(CNODE(st->modulecn), st->modulecn_slot++)); assert(err_is_ok(err)); // Copy multiboot module string to mmstrings area strcpy((char *)mmstrings, module->cmdline); mmstrings += strlen(module->cmdline) + 1; assert(mmstrings < mmstrings_base + BASE_PAGE_SIZE); module = ((void *) module) + module->size; position += module->size; module = (struct multiboot_tag_module_64 *) multiboot2_find_header( (struct multiboot_header_tag *) module, size - position, MULTIBOOT_TAG_TYPE_MODULE_64); } }
void cleanup_bios_regions(char *mmap_addr, char **new_mmap_addr, uint32_t *new_mmap_length) { assert(new_mmap_addr); assert(new_mmap_length); #define PRINT_REGIONS(map, map_length) do {\ for(char * printcur = map; printcur < map + map_length;) {\ struct multiboot_mmap * printcurmmap = (struct multiboot_mmap * SAFE)TC(printcur);\ printf("\t0x%08"PRIx64" - 0x%08"PRIx64" Type: %"PRIu32" Length: 0x%"PRIx64"\n", printcurmmap->base_addr, printcurmmap->base_addr + printcurmmap->length, printcurmmap->type, printcurmmap->length);\ printcur += printcurmmap->size + 4;\ }\ } while (0) printf("Raw MMAP from BIOS\n"); PRINT_REGIONS(mmap_addr, glbl_core_data->mmap_length); // normalize memory regions lpaddr_t clean_base = bsp_alloc_phys(glbl_core_data->mmap_length); char *clean_mmap_addr = (char *)local_phys_to_mem(clean_base); uint32_t clean_mmap_length = glbl_core_data->mmap_length; memcpy(clean_mmap_addr, mmap_addr, glbl_core_data->mmap_length); // first of all, sort regions by base address // yes, it's a bubble sort, but the dataset is small and usually in the right order bool swapped; do { swapped = false; for(char * cur = clean_mmap_addr; cur < clean_mmap_addr + clean_mmap_length;) { struct multiboot_mmap * curmmap = (struct multiboot_mmap * SAFE)TC(cur); if (cur + curmmap->size + 4 >= clean_mmap_addr + clean_mmap_length) break; // do not try to move this check into the forloop as entries do not have to be the same length struct multiboot_mmap * nextmmap = (struct multiboot_mmap * SAFE)TC(cur + curmmap->size + 4); if (nextmmap->base_addr < curmmap->base_addr || (nextmmap->base_addr == curmmap->base_addr && nextmmap->length > curmmap->length)) { // swap assert(curmmap->size == 20); // FIXME: The multiboot specification does not require this size assert(nextmmap->size == 20); struct multiboot_mmap tmp; tmp = *curmmap; *curmmap = *nextmmap; *nextmmap = tmp; swapped = true; } cur += curmmap->size + 4; } } while(swapped); printf("Sorted MMAP\n"); PRINT_REGIONS(clean_mmap_addr, clean_mmap_length); // now merge consecutive memory regions of the same or lower type for(char * cur = clean_mmap_addr; cur < clean_mmap_addr + clean_mmap_length;) { struct multiboot_mmap * curmmap = (struct multiboot_mmap * SAFE)TC(cur); if (cur + curmmap->size + 4 >= clean_mmap_addr + clean_mmap_length) break; // do not try to move this check into the forloop as entries do not have to be the same length struct multiboot_mmap * nextmmap = (struct multiboot_mmap * SAFE)TC(cur + curmmap->size + 4); /* On some machines (brie1) the IOAPIC region is only 1kB. * Currently we're not able to map regions that are <4kB so we * make sure that every region (if there is no problematic overlap) * is at least BASE_PAGE_SIZEd (==4kB) here. */ if ((curmmap->length < BASE_PAGE_SIZE) && (curmmap->base_addr + BASE_PAGE_SIZE <= nextmmap->base_addr)) { curmmap->length = BASE_PAGE_SIZE; } #define DISCARD_NEXT_MMAP do {\ uint32_t discardsize = nextmmap->size + 4;\ memmove(cur + curmmap->size + 4, cur + curmmap->size + 4 + discardsize, clean_mmap_length - (cur - clean_mmap_addr) - curmmap->size - 4 - discardsize);\ clean_mmap_length -= discardsize;\ } while (0) #define BUBBLE_NEXT_MMAP do {\ for (char * bubblecur = cur + curmmap->size + 4; bubblecur < clean_mmap_addr + clean_mmap_length;){\ struct multiboot_mmap * bubblecur_mmap = (struct multiboot_mmap * SAFE)TC(bubblecur);\ if (bubblecur + bubblecur_mmap->size + 4 >= clean_mmap_addr + clean_mmap_length)\ break;\ struct multiboot_mmap * bubblenext_mmap = (struct multiboot_mmap * SAFE)TC(bubblecur + bubblecur_mmap->size + 4);\ if (bubblenext_mmap->base_addr < bubblecur_mmap->base_addr || (bubblecur_mmap->base_addr == bubblenext_mmap->base_addr && bubblenext_mmap->length > bubblecur_mmap->length)) {\ struct multiboot_mmap bubbletmp; bubbletmp = *bubblecur_mmap; *bubblecur_mmap = *bubblenext_mmap; *bubblenext_mmap = bubbletmp;\ } else break;\ }} while(0) bool reduced = false; do { reduced = false; if (curmmap->base_addr == nextmmap->base_addr) { // regions start at the same location if (curmmap->length == nextmmap->length) { // trivial case. They are the same. Choose higher type and discard next curmmap->type = max(curmmap->type, nextmmap->type); DISCARD_NEXT_MMAP; reduced = true; continue; } else { // next region is smaller (we sorted that way) if (nextmmap->type <= curmmap->type) { // next regions type is the same or smaller. discard DISCARD_NEXT_MMAP; reduced = true; continue; } else { // next regions type is higher, so it gets priority // change type of current region and shrink next uint32_t tmptype = curmmap->type; uint64_t newlength = curmmap->length - nextmmap->length; curmmap->type = nextmmap->type; curmmap->length = nextmmap->length; nextmmap->type = tmptype; nextmmap->base_addr += nextmmap->length; nextmmap->length = newlength; // now we need to bubble next to the right place to restore order BUBBLE_NEXT_MMAP; reduced = true; continue; } } } // regions overlap if (nextmmap->base_addr > curmmap->base_addr && nextmmap->base_addr < curmmap->base_addr + curmmap->length) { // same type if (curmmap->type == nextmmap->type) { // simple. just extend if necessary and discard next if (nextmmap->base_addr + nextmmap->length > curmmap->base_addr + curmmap->length) curmmap->length = (nextmmap->base_addr + nextmmap->length) - curmmap->base_addr; DISCARD_NEXT_MMAP; reduced = true; continue; } else { // type is not the same if (nextmmap->base_addr + nextmmap->length < curmmap->base_addr + curmmap->length) { // there is a chunk at the end. create a new region struct multiboot_mmap tmpmmap; tmpmmap.size = 20; tmpmmap.base_addr = nextmmap->base_addr + nextmmap->length; tmpmmap.length = (curmmap->base_addr + curmmap->length) - (nextmmap->base_addr + nextmmap->length); tmpmmap.type = curmmap->type; // move everything to make room assert(clean_mmap_length + tmpmmap.length + 4 < BOOTINFO_SIZE); memmove(cur + curmmap->size + 4 + tmpmmap.size + 4, cur + curmmap->size + 4, clean_mmap_length - ((cur - clean_mmap_addr) + curmmap->size + 4)); clean_mmap_length += tmpmmap.size + 4; // insert new *nextmmap = tmpmmap; // restore order BUBBLE_NEXT_MMAP; reduced = true; } // after the previous step, the next region either ends // at the same location as the current or is longer uint64_t overlap = (curmmap->base_addr + curmmap->length) - nextmmap->base_addr; if (curmmap-> type > nextmmap->type) { // current has priority, shrink next and extend current nextmmap->length -= overlap; nextmmap->base_addr += overlap; curmmap->length += overlap; if (nextmmap->length == 0) DISCARD_NEXT_MMAP; reduced = true; continue; } else { // next has priority, shrink current and extend next nextmmap->length += overlap; nextmmap->base_addr -= overlap; curmmap->length -= overlap; reduced = true; continue; } } } } while (reduced); cur += curmmap->size + 4; #undef DISCARD_NEXT_MMAP #undef BUBBLE_NEXT_MMAP } printf("Preprocessed MMAP\n"); PRINT_REGIONS(clean_mmap_addr, clean_mmap_length); // we can only map pages. Therefore page align regions for(char * cur = clean_mmap_addr; cur < clean_mmap_addr + clean_mmap_length;) { struct multiboot_mmap * curmmap = (struct multiboot_mmap * SAFE)TC(cur); if (cur + curmmap->size + 4 >= clean_mmap_addr + clean_mmap_length) break; // do not try to move this check into the forloop as entries do not have to be the same length struct multiboot_mmap * nextmmap = (struct multiboot_mmap * SAFE)TC(cur + curmmap->size + 4); if (nextmmap->base_addr & BASE_PAGE_MASK) { uint64_t offset = nextmmap->base_addr - ((nextmmap->base_addr >> BASE_PAGE_BITS) << BASE_PAGE_BITS); // round in favour of higher type if (curmmap->type > nextmmap->type) { curmmap->length += BASE_PAGE_SIZE - offset; nextmmap->base_addr += BASE_PAGE_SIZE - offset; nextmmap->length -= BASE_PAGE_SIZE - offset; } else { curmmap->length -= offset; nextmmap->base_addr -= offset; nextmmap->length += offset; } } cur += curmmap->size + 4; }