/// Setup the module cnode, which contains frame caps to all multiboot modules
void create_module_caps(struct spawn_state *st)
{
errval_t err;
/* Create caps for multiboot modules */
struct multiboot_modinfo *module =
(struct multiboot_modinfo *)local_phys_to_mem(glbl_core_data->mods_addr);
// Allocate strings area
lpaddr_t mmstrings_phys = bsp_alloc_phys(BASE_PAGE_SIZE);
lvaddr_t mmstrings_base = local_phys_to_mem(mmstrings_phys);
lvaddr_t mmstrings = mmstrings_base;
// create cap for strings area in first slot of modulecn
assert(st->modulecn_slot == 0);
err = caps_create_new(ObjType_Frame, mmstrings_phys, BASE_PAGE_BITS,
BASE_PAGE_BITS,
caps_locate_slot(CNODE(st->modulecn),
st->modulecn_slot++));
assert(err_is_ok(err));
/* Walk over multiboot modules, creating frame caps */
for (int i = 0; i < glbl_core_data->mods_count; i++) {
struct multiboot_modinfo *m = &module[i];
// Set memory regions within bootinfo
struct mem_region *region =
&bootinfo->regions[bootinfo->regions_length++];
genpaddr_t remain = MULTIBOOT_MODULE_SIZE(*m);
genpaddr_t base_addr = local_phys_to_gen_phys(m->mod_start);
region->mr_type = RegionType_Module;
region->mr_base = base_addr;
region->mrmod_slot = st->modulecn_slot; // first slot containing caps
region->mrmod_size = remain; // size of image _in bytes_
region->mrmod_data = mmstrings - mmstrings_base; // offset of string in area
// round up to page size for caps
remain = ROUND_UP(remain, BASE_PAGE_SIZE);
// Create max-sized caps to multiboot module in module cnode
while (remain > 0) {
assert((base_addr & BASE_PAGE_MASK) == 0);
assert((remain & BASE_PAGE_MASK) == 0);
// determine size of next chunk
uint8_t block_size = bitaddralign(remain, base_addr);
assert(st->modulecn_slot < (1UL << st->modulecn->cap.u.cnode.bits));
// create as DevFrame cap to avoid zeroing memory contents
err = caps_create_new(ObjType_DevFrame, base_addr, block_size,
block_size,
caps_locate_slot(CNODE(st->modulecn),
st->modulecn_slot++));
assert(err_is_ok(err));
// Advance by that chunk
base_addr += ((genpaddr_t)1 << block_size);
remain -= ((genpaddr_t)1 << block_size);
}
// Copy multiboot module string to mmstrings area
strcpy((char *)mmstrings, MBADDR_ASSTRING(m->string));
mmstrings += strlen(MBADDR_ASSTRING(m->string)) + 1;
assert(mmstrings < mmstrings_base + BASE_PAGE_SIZE);
}
}
/// Create physical address range or RAM caps to unused physical memory
static void create_phys_caps(lpaddr_t init_alloc_addr)
{
errval_t err;
/* Walk multiboot MMAP structure, and create appropriate caps for memory */
char *mmap_addr = MBADDR_ASSTRING(glbl_core_data->mmap_addr);
genpaddr_t last_end_addr = 0;
for(char *m = mmap_addr; m < mmap_addr + glbl_core_data->mmap_length;)
{
struct multiboot_mmap *mmap = (struct multiboot_mmap * SAFE)TC(m);
debug(SUBSYS_STARTUP, "MMAP %llx--%llx Type %"PRIu32"\n",
mmap->base_addr, mmap->base_addr + mmap->length,
mmap->type);
if (last_end_addr >= init_alloc_addr
&& mmap->base_addr > last_end_addr)
{
/* we have a gap between regions. add this as a physaddr range */
debug(SUBSYS_STARTUP, "physical address range %llx--%llx\n",
last_end_addr, mmap->base_addr);
err = create_caps_to_cnode(last_end_addr,
mmap->base_addr - last_end_addr,
RegionType_PhyAddr, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
if (mmap->type == MULTIBOOT_MEM_TYPE_RAM)
{
genpaddr_t base_addr = mmap->base_addr;
genpaddr_t end_addr = base_addr + mmap->length;
// only map RAM which is greater than init_alloc_addr
if (end_addr > local_phys_to_gen_phys(init_alloc_addr))
{
if (base_addr < local_phys_to_gen_phys(init_alloc_addr)) {
base_addr = local_phys_to_gen_phys(init_alloc_addr);
}
debug(SUBSYS_STARTUP, "RAM %llx--%llx\n", base_addr, end_addr);
assert(end_addr >= base_addr);
err = create_caps_to_cnode(base_addr, end_addr - base_addr,
RegionType_Empty, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
}
else if (mmap->base_addr > local_phys_to_gen_phys(init_alloc_addr))
{
/* XXX: The multiboot spec just says that mapping types other than
* RAM are "reserved", but GRUB always maps the ACPI tables as type
* 3, and things like the IOAPIC tend to show up as type 2 or 4,
* so we map all these regions as platform data
*/
debug(SUBSYS_STARTUP, "platform %llx--%llx\n", mmap->base_addr,
mmap->base_addr + mmap->length);
assert(mmap->base_addr > local_phys_to_gen_phys(init_alloc_addr));
err = create_caps_to_cnode(mmap->base_addr, mmap->length,
RegionType_PlatformData, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
last_end_addr = mmap->base_addr + mmap->length;
m += mmap->size + 4;
}
// Assert that we have some physical address space
assert(last_end_addr != 0);
if (last_end_addr < PADDR_SPACE_SIZE)
{
/*
* FIXME: adding the full range results in too many caps to add
* to the cnode (and we can't handle such big caps in user-space
* yet anyway) so instead we limit it to something much smaller
*/
genpaddr_t size = PADDR_SPACE_SIZE - last_end_addr;
const genpaddr_t phys_region_limit = 1ULL << 32; // PCI implementation limit
if (last_end_addr > phys_region_limit) {
size = 0; // end of RAM is already too high!
} else if (last_end_addr + size > phys_region_limit) {
size = phys_region_limit - last_end_addr;
}
debug(SUBSYS_STARTUP, "end physical address range %llx--%llx\n",
last_end_addr, last_end_addr + size);
err = create_caps_to_cnode(last_end_addr, size,
RegionType_PhyAddr, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
}
/**
* \brief Map a region of physical memory into physical memory address space.
*
* Maps the region of physical memory, based at base and sized size bytes
* to the same-sized virtual memory region. All pages are flagged according to
* bitmap. This function automatically fills the needed page directory entries
* in the page hierarchy rooted at pml4. base and size will be made
* page-aligned by this function.
*
* \param base Base address of memory region
* \param size Size in bytes of memory region
* \param bitmap Bitmap of flags for page tables/directories
*
* \return 0 on success, -1 on error (out of range)
*/
static int paging_x86_32_map_mem(lpaddr_t base, size_t size, uint64_t bitmap)
{
lvaddr_t vaddr, vbase = local_phys_to_mem(base);
lpaddr_t addr;
paging_align(&vbase, &base, &size, X86_32_MEM_PAGE_SIZE);
// Is mapped region out of range?
assert(local_phys_to_gen_phys(base + size) <= X86_32_PADDR_SPACE_LIMIT);
if(local_phys_to_gen_phys(base + size) > X86_32_PADDR_SPACE_LIMIT) {
printk(LOG_ERR, "Mapped region [%"PRIxLPADDR",%"PRIxLPADDR"]"
"out of physical address range!",
base, base + size);
return -1;
}
assert(local_phys_to_gen_phys(vbase + size) <= X86_32_VADDR_SPACE_SIZE);
// Map pages, tables and directories
for(vaddr = vbase, addr = base;;
vaddr += X86_32_MEM_PAGE_SIZE, addr += X86_32_MEM_PAGE_SIZE) {
#ifdef CONFIG_PAE
union x86_32_pdpte_entry *pdpte_base = &pdpte[X86_32_PDPTE_BASE(vaddr)];
union x86_32_ptable_entry *pdir_base =
&mem_pdir[X86_32_PDPTE_BASE(addr)][X86_32_PDIR_BASE(vaddr)];
#else
union x86_32_pdir_entry *pdir_base = &pdir[X86_32_PDIR_BASE(vaddr)];
# ifndef CONFIG_PSE
union x86_32_ptable_entry *ptable_base =
&mem_ptable[X86_32_PDIR_BASE(addr)][X86_32_PTABLE_BASE(vaddr)];
# endif
#endif
if(vbase + size != 0) {
if(vaddr >= vbase + size) {
break;
}
}
#ifdef CONFIG_PAE
debug(SUBSYS_PAGING, "Mapping 2M page: vaddr = 0x%x, addr = 0x%x, "
"PDPTE_BASE = %u, PDIR_BASE = %u -- ", vaddr,
addr, X86_32_PDPTE_BASE(vaddr), X86_32_PDIR_BASE(vaddr));
mapit(pdpte_base, pdir_base, addr, bitmap);
#else
# ifdef CONFIG_PSE
debug(SUBSYS_PAGING, "Mapping 4M page: vaddr = 0x%x, addr = 0x%x, "
"PDIR_BASE = %u -- ", vaddr,
addr, X86_32_PDIR_BASE(vaddr));
mapit(pdir_base, addr, bitmap);
# else
debug(SUBSYS_PAGING, "Mapping 4K page: vaddr = 0x%"PRIxLVADDR", "
"addr = 0x%"PRIxLVADDR", "
"PDIR_BASE = %"PRIuLPADDR", PTABLE_BASE = %"PRIuLPADDR" -- ", vaddr,
addr, X86_32_PDIR_BASE(vaddr), X86_32_PTABLE_BASE(vaddr));
mapit(pdir_base, ptable_base, addr, bitmap);
# endif
#endif
if(vbase + size == 0) {
// Bail out if mapped last page of address space to prevent overflow
if(vaddr == 0xffe00000) {
break;
}
}
}
return 0;
}
/// Setup the module cnode, which contains frame caps to all multiboot modules
void create_module_caps(struct spawn_state *st)
{
errval_t err;
/* Create caps for multiboot modules */
struct multiboot_header_tag *multiboot =
(struct multiboot_header_tag *)local_phys_to_mem(armv8_glbl_core_data->multiboot_image.base);
// Allocate strings area
lpaddr_t mmstrings_phys = bsp_alloc_phys(BASE_PAGE_SIZE);
lvaddr_t mmstrings_base = local_phys_to_mem(mmstrings_phys);
lvaddr_t mmstrings = mmstrings_base;
// create cap for strings area in first slot of modulecn
assert(st->modulecn_slot == 0);
err = caps_create_new(ObjType_Frame, mmstrings_phys, BASE_PAGE_SIZE,
BASE_PAGE_SIZE, my_core_id,
caps_locate_slot(CNODE(st->modulecn),
st->modulecn_slot++));
assert(err_is_ok(err));
//Nag
bootinfo->regions_length = 0;
/* Walk over multiboot modules, creating frame caps */
size_t position = 0;
size_t size = armv8_glbl_core_data->multiboot_image.length;
struct mem_region *region;
lpaddr_t acpi_base = (lpaddr_t)-1;
/* add the ACPI regions */
struct multiboot_tag_new_acpi *acpi_new;
acpi_new = (struct multiboot_tag_new_acpi *)
multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_ACPI_NEW);
if (acpi_new) {
acpi_base = mem_to_local_phys((lvaddr_t)&acpi_new->rsdp[0]);
} else {
struct multiboot_tag_old_acpi *acpi_old;
acpi_old = (struct multiboot_tag_old_acpi *)
multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_ACPI_OLD);
if (acpi_old) {
acpi_base = mem_to_local_phys((lvaddr_t)&acpi_old->rsdp[0]);
}
}
if (acpi_base != (lpaddr_t)-1) {
region = &bootinfo->regions[bootinfo->regions_length++];
region->mr_base = acpi_base;
region->mr_type = RegionType_ACPI_TABLE;
}
/* add the module regions */
position = 0;
struct multiboot_tag_module_64 *module = (struct multiboot_tag_module_64 *)
multiboot2_find_header(multiboot, size, MULTIBOOT_TAG_TYPE_MODULE_64);
while (module) {
// Set memory regions within bootinfo
region = &bootinfo->regions[bootinfo->regions_length++];
genpaddr_t remain = module->mod_end - module->mod_start;
genpaddr_t base_addr = local_phys_to_gen_phys(module->mod_start);
region->mr_type = RegionType_Module;
region->mr_base = base_addr;
region->mrmod_slot = st->modulecn_slot; // first slot containing caps
region->mrmod_size = remain; // size of image _in bytes_
region->mrmod_data = mmstrings - mmstrings_base; // offset of string in area
// round up to page size for caps
remain = ROUND_UP(remain, BASE_PAGE_SIZE);
assert((base_addr & BASE_PAGE_MASK) == 0);
assert((remain & BASE_PAGE_MASK) == 0);
assert(st->modulecn_slot < cnode_get_slots(&st->modulecn->cap));
// create as DevFrame cap to avoid zeroing memory contents
err = caps_create_new(ObjType_DevFrame, base_addr, remain,
remain, my_core_id,
caps_locate_slot(CNODE(st->modulecn),
st->modulecn_slot++));
assert(err_is_ok(err));
// Copy multiboot module string to mmstrings area
strcpy((char *)mmstrings, module->cmdline);
mmstrings += strlen(module->cmdline) + 1;
assert(mmstrings < mmstrings_base + BASE_PAGE_SIZE);
module = ((void *) module) + module->size;
position += module->size;
module = (struct multiboot_tag_module_64 *) multiboot2_find_header(
(struct multiboot_header_tag *) module, size - position,
MULTIBOOT_TAG_TYPE_MODULE_64);
}
}
/// Create physical address range or RAM caps to unused physical memory
static void create_phys_caps(lpaddr_t init_alloc_addr)
{
errval_t err;
#ifndef __scc__
// map first meg of RAM, which contains lots of crazy BIOS tables
err = create_caps_to_cnode(0, X86_32_START_KERNEL_PHYS,
RegionType_PlatformData, &spawn_state, bootinfo);
assert(err_is_ok(err));
#endif
/* Walk multiboot MMAP structure, and create appropriate caps for memory */
char *mmap_addr = MBADDR_ASSTRING(glbl_core_data->mmap_addr);
genpaddr_t last_end_addr = 0;
char *clean_mmap_addr;
uint32_t clean_mmap_length;
cleanup_bios_regions(mmap_addr, &clean_mmap_addr, &clean_mmap_length);
for(char *m = mmap_addr; m < mmap_addr + glbl_core_data->mmap_length;) {
struct multiboot_mmap *mmap = (struct multiboot_mmap * SAFE)TC(m);
debug(SUBSYS_STARTUP, "MMAP %llx--%llx Type %"PRIu32"\n",
mmap->base_addr, mmap->base_addr + mmap->length,
mmap->type);
#if 0
// XXX: Remove intersecting regions
bool skip = false;
for(int i = 0; i < bootinfo->regions_length; i++) {
struct mem_region *r = &bootinfo->regions[i];
// Remove intersecting regions (earlier additions take precedence)
if((r->base + (1 << r->bits) >= mmap->base_addr
&& r->base + (1 << r->bits) <= mmap->base_addr + mmap->length)
|| (r->base >= mmap->base_addr
&& r->base <= mmap->base_addr + mmap->length)) {
skip = true;
break;
}
}
if(skip) {
continue;
}
#endif
if (last_end_addr >= init_alloc_addr
&& mmap->base_addr > last_end_addr) {
/* we have a gap between regions. add this as a physaddr range */
debug(SUBSYS_STARTUP, "physical address range %llx--%llx\n",
last_end_addr, mmap->base_addr);
err = create_caps_to_cnode(last_end_addr,
mmap->base_addr - last_end_addr,
RegionType_PhyAddr, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
if (mmap->type == MULTIBOOT_MEM_TYPE_RAM) {
genpaddr_t base_addr = mmap->base_addr;
genpaddr_t end_addr = base_addr + mmap->length;
// only map RAM which is greater than init_alloc_addr
if (end_addr > local_phys_to_gen_phys(init_alloc_addr)) {
if (base_addr < local_phys_to_gen_phys(init_alloc_addr)) {
base_addr = local_phys_to_gen_phys(init_alloc_addr);
}
#ifndef CONFIG_PAE
if(base_addr >= X86_32_PADDR_SPACE_SIZE) {
printk(LOG_NOTE, "skipping RAM [%llx--%llx] out of "
"mappable space\n", base_addr, end_addr);
last_end_addr = mmap->base_addr + mmap->length;
m += mmap->size + 4;
continue;
}
if(end_addr > X86_32_PADDR_SPACE_SIZE) {
printk(LOG_NOTE, "shortening RAM [%llx--%llx] to mappable "
"space [0--%llx]\n", base_addr, end_addr,
X86_32_PADDR_SPACE_LIMIT);
end_addr = X86_32_PADDR_SPACE_SIZE;
}
#endif
#ifndef __scc__
// XXX: Do not create ram caps for memory the kernel cannot
// address to prevent kernel objects from being created there
if(base_addr >= PADDR_SPACE_LIMIT) {
last_end_addr = mmap->base_addr + mmap->length;
m += mmap->size + 4;
continue;
}
if (end_addr > PADDR_SPACE_LIMIT) {
end_addr = PADDR_SPACE_LIMIT;
}
#endif
debug(SUBSYS_STARTUP, "RAM %llx--%llx\n", base_addr, end_addr);
assert(end_addr >= base_addr);
err = create_caps_to_cnode(base_addr, end_addr - base_addr,
RegionType_Empty, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
} else if (mmap->base_addr > local_phys_to_gen_phys(init_alloc_addr)) {
/* XXX: The multiboot spec just says that mapping types other than
* RAM are "reserved", but GRUB always maps the ACPI tables as type
* 3, and things like the IOAPIC tend to show up as type 2 or 4,
* so we map all these regions as platform data
*/
debug(SUBSYS_STARTUP, "platform %llx--%llx\n", mmap->base_addr,
mmap->base_addr + mmap->length);
assert(mmap->base_addr > local_phys_to_gen_phys(init_alloc_addr));
err = create_caps_to_cnode(mmap->base_addr, mmap->length,
RegionType_PlatformData, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
last_end_addr = mmap->base_addr + mmap->length;
m += mmap->size + 4;
}
// Assert that we have some physical address space
assert(last_end_addr != 0);
if (last_end_addr < X86_32_PADDR_SPACE_SIZE) {
/*
* FIXME: adding the full range results in too many caps to add
* to the cnode (and we can't handle such big caps in user-space
* yet anyway) so instead we limit it to something much smaller
*/
genpaddr_t size = X86_32_PADDR_SPACE_SIZE - last_end_addr;
const genpaddr_t phys_region_limit = 1ULL << 32; // PCI implementation limit
if (last_end_addr > phys_region_limit) {
size = 0; // end of RAM is already too high!
} else if (last_end_addr + size > phys_region_limit) {
size = phys_region_limit - last_end_addr;
}
debug(SUBSYS_STARTUP, "end physical address range %llx--%llx\n",
last_end_addr, last_end_addr + size);
err = create_caps_to_cnode(last_end_addr, size,
RegionType_PhyAddr, &spawn_state, bootinfo);
assert(err_is_ok(err));
}
}
/**
* \brief Cleanup the last cap copy for an object and the object itself
*/
static errval_t
cleanup_last(struct cte *cte, struct cte *ret_ram_cap)
{
errval_t err;
TRACE_CAP_MSG("cleaning up last copy", cte);
struct capability *cap = &cte->cap;
assert(!has_copies(cte));
if (cte->mdbnode.remote_copies) {
printk(LOG_WARN, "cleanup_last but remote_copies is set\n");
}
if (ret_ram_cap && ret_ram_cap->cap.type != ObjType_Null) {
return SYS_ERR_SLOT_IN_USE;
}
struct RAM ram = { .bits = 0 };
size_t len = sizeof(struct RAM) / sizeof(uintptr_t) + 1;
if (!has_descendants(cte) && !has_ancestors(cte)) {
// List all RAM-backed capabilities here
// NB: ObjType_PhysAddr and ObjType_DevFrame caps are *not* RAM-backed!
switch(cap->type) {
case ObjType_RAM:
ram.base = cap->u.ram.base;
ram.bits = cap->u.ram.bits;
break;
case ObjType_Frame:
ram.base = cap->u.frame.base;
ram.bits = cap->u.frame.bits;
break;
case ObjType_CNode:
ram.base = cap->u.cnode.cnode;
ram.bits = cap->u.cnode.bits + OBJBITS_CTE;
break;
case ObjType_Dispatcher:
// Convert to genpaddr
ram.base = local_phys_to_gen_phys(mem_to_local_phys((lvaddr_t)cap->u.dispatcher.dcb));
ram.bits = OBJBITS_DISPATCHER;
break;
default:
// Handle VNodes here
if(type_is_vnode(cap->type)) {
ram.base = get_address(cap);
ram.bits = vnode_objbits(cap->type);
}
break;
}
}
err = cleanup_copy(cte);
if (err_is_fail(err)) {
return err;
}
if(ram.bits > 0) {
// Send back as RAM cap to monitor
if (ret_ram_cap) {
if (dcb_current != monitor_ep.u.endpoint.listener) {
printk(LOG_WARN, "sending fresh ram cap to non-monitor?\n");
}
assert(ret_ram_cap->cap.type == ObjType_Null);
ret_ram_cap->cap.u.ram = ram;
ret_ram_cap->cap.type = ObjType_RAM;
err = mdb_insert(ret_ram_cap);
TRACE_CAP_MSG("reclaimed", ret_ram_cap);
assert(err_is_ok(err));
// note: this is a "success" code!
err = SYS_ERR_RAM_CAP_CREATED;
}
else if (monitor_ep.type && monitor_ep.u.endpoint.listener != 0) {
#ifdef TRACE_PMEM_CAPS
struct cte ramcte;
memset(&ramcte, 0, sizeof(ramcte));
ramcte.cap.u.ram = ram;
ramcte.cap.type = ObjType_RAM;
TRACE_CAP_MSG("reclaimed", ret_ram_cap);
#endif
// XXX: This looks pretty ugly. We need an interface.
err = lmp_deliver_payload(&monitor_ep, NULL,
(uintptr_t *)&ram,
len, false);
}
else {
printk(LOG_WARN, "dropping ram cap base %08"PRIxGENPADDR" bits %"PRIu8"\n", ram.base, ram.bits);
}
if (err_no(err) == SYS_ERR_LMP_BUF_OVERFLOW) {
printk(LOG_WARN, "dropped ram cap base %08"PRIxGENPADDR" bits %"PRIu8"\n", ram.base, ram.bits);
err = SYS_ERR_OK;
} else {
assert(err_is_ok(err));
}
}
return err;
}
/*
* Mark phase of revoke mark & sweep
*/
static void caps_mark_revoke_copy(struct cte *cte)
{
errval_t err;
err = caps_try_delete(cte);
if (err_is_fail(err)) {
// this should not happen as there is a copy of the cap
panic("error while marking/deleting cap copy for revoke:"
" 0x%"PRIuERRV"\n", err);
}
}
void ipi_notify_init(void)
{
my_arch_id = apic_get_id();
// Publish the address of the notify page in the global kernel state
global->notify[my_arch_id] = local_phys_to_gen_phys(mem_to_local_phys((lvaddr_t)my_notify_page));
}
