uintptr_t OS::heap_max() {
// Before the memory map is populated
if (UNLIKELY(memory_map().empty()))
return heap_max_;
// After memory map is populated
return memory_map().at(heap_begin).addr_end();
}
EFI_STATUS
print_memory_map(void)
{
EFI_MEMORY_DESCRIPTOR *buf;
UINTN desc_size;
UINT32 desc_version;
UINTN size, map_key, mapping_size;
EFI_MEMORY_DESCRIPTOR *desc;
EFI_STATUS err = EFI_SUCCESS;
int i = 0;
err = memory_map(&buf, &size, &map_key, &desc_size, &desc_version);
if (err != EFI_SUCCESS)
return err;
Print(L"Memory Map Size: %d\n", size);
Print(L"Map Key: %d\n", map_key);
Print(L"Descriptor Version: %d\n", desc_version);
Print(L"Descriptor Size: %d\n\n", desc_size);
desc = buf;
while ((void *)desc < (void *)buf + size) {
mapping_size = desc->NumberOfPages * PAGE_SIZE;
Print(L"[#%02d] Type: %s Attr: 0x%x\n", i, memory_type_to_str(desc->Type), desc->Attribute);
Print(L" Phys: %016llx-%016llx\n", desc->PhysicalStart, desc->PhysicalStart + mapping_size);
Print(L" Virt: %016llx-%016llx\n\n", desc->VirtualStart, desc->VirtualStart + mapping_size);
desc = (void *)desc + desc_size;
i++;
}
uefi_call_wrapper(BS->FreePool, 1, buf);
return err;
}
static inline int zcb_alloc (
struct conn_info *conn_info,
const char *path_to_file,
size_t size,
void **addr)
{
struct zcb_mapped *zcb_mapped;
unsigned int res;
zcb_mapped = malloc (sizeof (struct zcb_mapped));
if (zcb_mapped == NULL) {
return (-1);
}
res = memory_map (
path_to_file,
size,
addr);
if (res == -1) {
free (zcb_mapped);
return (-1);
}
list_init (&zcb_mapped->list);
zcb_mapped->addr = *addr;
zcb_mapped->size = size;
list_add_tail (&zcb_mapped->list, &conn_info->zcb_mapped_list_head);
return (0);
}
static
void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
{
if (order == 0) {
if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
/* small table */
ht->tbl_mmap = calloc(ht->max_nr_buckets,
sizeof(*ht->tbl_mmap));
assert(ht->tbl_mmap);
return;
}
/* large table */
ht->tbl_mmap = memory_map(ht->max_nr_buckets
* sizeof(*ht->tbl_mmap));
memory_populate(ht->tbl_mmap,
ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
} else if (order > ht->min_alloc_buckets_order) {
/* large table */
unsigned long len = 1UL << (order - 1);
assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
memory_populate(ht->tbl_mmap + len,
len * sizeof(*ht->tbl_mmap));
}
/* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
}
/**
* emalloc - Allocate memory with a strict alignment requirement
* @size: size in bytes of the requested allocation
* @align: the required alignment of the allocation
* @addr: a pointer to the allocated address on success
*
* If we cannot satisfy @align we return 0.
*/
EFI_STATUS emalloc(UINTN size, UINTN align, EFI_PHYSICAL_ADDRESS *addr)
{
UINTN map_size, map_key, desc_size;
EFI_MEMORY_DESCRIPTOR *map_buf;
UINTN d, map_end;
UINT32 desc_version;
EFI_STATUS err;
UINTN nr_pages = EFI_SIZE_TO_PAGES(size);
err = memory_map(&map_buf, &map_size, &map_key,
&desc_size, &desc_version);
if (err != EFI_SUCCESS)
goto fail;
d = (UINTN)map_buf;
map_end = (UINTN)map_buf + map_size;
for (; d < map_end; d += desc_size) {
EFI_MEMORY_DESCRIPTOR *desc;
EFI_PHYSICAL_ADDRESS start, end, aligned;
desc = (EFI_MEMORY_DESCRIPTOR *)d;
if (desc->Type != EfiConventionalMemory)
continue;
if (desc->NumberOfPages < nr_pages)
continue;
start = desc->PhysicalStart;
end = start + (desc->NumberOfPages << EFI_PAGE_SHIFT);
/* Low-memory is super-precious! */
if (end <= 1 << 20)
continue;
if (start < 1 << 20) {
size -= (1 << 20) - start;
start = (1 << 20);
}
aligned = (start + align -1) & ~(align -1);
if ((aligned + size) <= end) {
err = allocate_pages(AllocateAddress, EfiLoaderData,
nr_pages, &aligned);
if (err == EFI_SUCCESS) {
*addr = aligned;
break;
}
}
}
if (d == map_end)
err = EFI_OUT_OF_RESOURCES;
free_pool(map_buf);
fail:
return err;
}
void mbc1_rom_bank_number_write(uint16_t address, uint8_t data)
{
uint8_t lower = data & ROM_BANK_NUMBER_LOWER_MASK;
if (lower == 0)
lower = 1;
rom1_bank_number &= ~ROM_BANK_NUMBER_LOWER_MASK;
rom1_bank_number |= lower;
memory_map((char *)gb_get_cart_path(), (void *)rom1, rom1_bank_number * ROM1_BANK_SIZE, ROM1_BANK_SIZE);
}
void mbc1_ram_rom_bank_number_write(uint16_t address, uint8_t data)
{
if (!mode_select) {
uint8_t upper = data << ROM_BANK_NUMBER_UPPER_SHIFT;
rom1_bank_number &= ~ROM_BANK_NUMBER_UPPER_MASK;
rom1_bank_number |= upper;
memory_map((char *)gb_get_cart_path(), (void *)rom1, rom1_bank_number * ROM1_BANK_SIZE, ROM1_BANK_SIZE);
} else {
ram_bank_number = data;
}
}
void process_init(void) {
// Map the thread map
uintptr_t vaddr = MEMORY_PROCESS_MAP_VADDR;
uint16_t pflags = PAGE_FLAG_GLOBAL | PAGE_FLAG_WRITEABLE;
uintptr_t offset;
for (offset = 0; offset < PROCESS_MAP_SIZE; offset += 0x1000)
memory_map(vaddr + offset, frame_alloc(), pflags);
// Clear the thread map
memset((void *) vaddr, 0, PROCESS_MAP_SIZE);
}
static void _process_create_thread_map(uint32_t pid) {
// Map thread map
uintptr_t thread_map = MEMORY_THREAD_MAP_VADDR + pid * THREAD_MAP_SIZE;
uint16_t pflags = PAGE_FLAG_WRITEABLE | PAGE_FLAG_GLOBAL;
size_t offset;
for (offset = 0; offset < THREAD_MAP_SIZE; offset += 0x1000)
memory_map(thread_map + offset, frame_alloc(), pflags);
// Clear thread map
memset((void *) thread_map, 0, THREAD_MAP_SIZE);
}
static EFI_STATUS print_memory_map(void)
{
EFI_MEMORY_DESCRIPTOR *buf;
UINTN desc_size;
UINT32 desc_version;
UINTN size, map_key;
EFI_MEMORY_DESCRIPTOR *desc;
EFI_STATUS err;
int i;
err = memory_map(&buf, &size, &map_key,
&desc_size, &desc_version);
if (err != EFI_SUCCESS)
return err;
Print(L"System Memory Map\n");
Print(L"System Memory Map Size: %d\n", size);
Print(L"Descriptor Version: %d\n", desc_version);
Print(L"Descriptor Size: %d\n", desc_size);
desc = buf;
i = 0;
while ((void *)desc < (void *)buf + size) {
UINTN mapping_size;
mapping_size = desc->NumberOfPages * PAGE_SIZE;
Print(L"[#%.2d] Type: %s\n", i,
memory_type_to_str(desc->Type));
Print(L" Attr: 0x%016llx\n", desc->Attribute);
Print(L" Phys: [0x%016llx - 0x%016llx]\n",
desc->PhysicalStart,
desc->PhysicalStart + mapping_size);
Print(L" Virt: [0x%016llx - 0x%016llx]",
desc->VirtualStart,
desc->VirtualStart + mapping_size);
Print(L"\n");
desc = (void *)desc + desc_size;
i++;
}
free_pool(buf);
return err;
}
void pass2( void )
{
memory_check();
process_section_list();
allocate();
update_linker_information();
output_object();
if( mflag )
memory_map();
}
cs_error_t
coroipcc_zcb_alloc (
hdb_handle_t handle,
void **buffer,
size_t size,
size_t header_size)
{
struct ipc_instance *ipc_instance;
void *buf = NULL;
char path[PATH_MAX];
unsigned int res;
mar_req_coroipcc_zc_alloc_t req_coroipcc_zc_alloc;
coroipc_response_header_t res_coroipcs_zc_alloc;
size_t map_size;
struct iovec iovec;
struct coroipcs_zc_header *hdr;
res = hdb_error_to_cs (hdb_handle_get (&ipc_hdb, handle, (void **)&ipc_instance));
if (res != CS_OK) {
return (res);
}
map_size = size + header_size + sizeof (struct coroipcs_zc_header);
res = memory_map (path, "corosync_zerocopy-XXXXXX", &buf, map_size);
assert (res != -1);
req_coroipcc_zc_alloc.header.size = sizeof (mar_req_coroipcc_zc_alloc_t);
req_coroipcc_zc_alloc.header.id = ZC_ALLOC_HEADER;
req_coroipcc_zc_alloc.map_size = map_size;
strcpy (req_coroipcc_zc_alloc.path_to_file, path);
iovec.iov_base = (void *)&req_coroipcc_zc_alloc;
iovec.iov_len = sizeof (mar_req_coroipcc_zc_alloc_t);
res = coroipcc_msg_send_reply_receive (
handle,
&iovec,
1,
&res_coroipcs_zc_alloc,
sizeof (coroipc_response_header_t));
hdr = (struct coroipcs_zc_header *)buf;
hdr->map_size = map_size;
*buffer = ((char *)buf) + sizeof (struct coroipcs_zc_header);
hdb_handle_put (&ipc_hdb, handle);
return (res);
}
void OS::legacy_boot()
{
// Fetch CMOS memory info (unfortunately this is maximally 10^16 kb)
auto mem = x86::CMOS::meminfo();
if (OS::memory_end_ == 0)
{
//uintptr_t low_memory_size = mem.base.total * 1024;
INFO2("* Low memory: %i Kib", mem.base.total);
uintptr_t high_memory_size = mem.extended.total * 1024;
INFO2("* High memory (from cmos): %i Kib", mem.extended.total);
OS::memory_end_ = 0x100000 + high_memory_size - 1;
}
auto& memmap = memory_map();
// No guarantees without multiboot, but we assume standard memory layout
memmap.assign_range({0x0009FC00, 0x0009FFFF,
"EBDA"});
memmap.assign_range({0x000A0000, 0x000FFFFF,
"VGA/ROM"});
}
void stack_resize(stack_t *stack, uintptr_t new_len, process_t *process) {
// Check address space
if (UNLIKELY(process->addr_space != memory_space_get()))
PANIC("Failed trying to resize a stack for a process while not being in "
"its address space.");
// Greater than maximum length?
if (UNLIKELY(new_len > STACK_LENGTH_MAX))
PANIC("Failed trying to increase a stack's size over the maximum stack "
"size.");
// Size increased or decreased?
if (new_len > stack->length) { // Increased
// Map region
uintptr_t reg_end = stack->address - stack->length;
uintptr_t reg_addr;
uint16_t flags = PAGE_FLAG_WRITEABLE | PAGE_FLAG_USER;
for (reg_addr = stack->address - new_len; reg_addr < reg_end; reg_addr += PAGE_SIZE) {
uintptr_t phys = frame_alloc();
memory_map(reg_addr, phys, flags);
}
} else if (new_len < stack->length) {
// Unmap region
uintptr_t reg_end = stack->address - new_len;
uintptr_t reg_addr;
for (reg_addr = stack->address - stack->length; reg_addr < reg_end; reg_addr += 0x1000) {
uintptr_t phys = memory_physical(reg_addr);
memory_unmap(reg_addr);
frame_free(phys);
}
}
// Set new size
stack->length = new_len;
}
void mbc1_rom0_init()
{
memory_map((char *)gb_get_bios_path(), (void *)rom0, 0, BIOS_SIZE);
memory_map((char *)gb_get_cart_path(), (void *)&rom0[BIOS_SIZE], BIOS_SIZE, ROM0_BANK_SIZE - BIOS_SIZE);
}
void mbc1_lock_bios()
{
memory_map((char *)gb_get_cart_path(), (void *)rom0, 0, BIOS_SIZE);
}
void mbc1_rom_bank_number_init()
{
rom1_bank_number = 1;
memory_map((char *)gb_get_cart_path(), (void *)rom1, rom1_bank_number * ROM1_BANK_SIZE, ROM1_BANK_SIZE);
}
void rom_init(char *rom_path)
{
memory_map((char *)gb_get_bios_path(), (void *)rom, 0, BIOS_SIZE);
memory_map((char *)gb_get_cart_path(), (void *)&rom[BIOS_SIZE], BIOS_SIZE, ROM_SIZE - BIOS_SIZE);
}
void OS::start(char* _cmdline, uintptr_t mem_size)
{
// Initialize stdout handlers
OS::add_stdout(&OS::default_stdout);
PROFILE("");
// Print a fancy header
CAPTION("#include<os> // Literally");
void* esp = get_cpu_esp();
MYINFO("Stack: %p", esp);
/// STATMAN ///
/// initialize on page 7, 2 pages in size
Statman::get().init(0x6000, 0x3000);
OS::cmdline = _cmdline;
// setup memory and heap end
OS::memory_end_ = mem_size;
OS::heap_max_ = OS::memory_end_;
// Call global ctors
PROFILE("Global constructors");
__libc_init_array();
PROFILE("Memory map");
// Assign memory ranges used by the kernel
auto& memmap = memory_map();
MYINFO("Assigning fixed memory ranges (Memory map)");
memmap.assign_range({0x500, 0x5fff, "solo5", "solo5"});
memmap.assign_range({0x6000, 0x8fff, "Statman", "Statistics"});
memmap.assign_range({0xA000, 0x9fbff, "Stack", "Kernel / service main stack"});
memmap.assign_range({(uintptr_t)&_LOAD_START_, (uintptr_t)&_end,
"ELF", "Your service binary including OS"});
Expects(::heap_begin and heap_max_);
// @note for security we don't want to expose this
memmap.assign_range({(uintptr_t)&_end + 1, ::heap_begin - 1,
"Pre-heap", "Heap randomization area"});
uintptr_t span_max = std::numeric_limits<std::ptrdiff_t>::max();
uintptr_t heap_range_max_ = std::min(span_max, heap_max_);
MYINFO("Assigning heap");
memmap.assign_range({::heap_begin, heap_range_max_,
"Heap", "Dynamic memory", heap_usage });
MYINFO("Printing memory map");
for (const auto &i : memmap)
INFO2("* %s",i.second.to_string().c_str());
extern void __platform_init();
__platform_init();
MYINFO("Booted at monotonic_ns=%lld walltime_ns=%lld",
solo5_clock_monotonic(), solo5_clock_wall());
Solo5_manager::init();
// We don't need a start or stop function in solo5.
Timers::init(
// timer start function
[] (std::chrono::microseconds) {},
// timer stop function
[] () {});
// Some tests are asserting there is at least one timer that is always ON
// (the RTC calibration timer). Let's fake some timer so those tests pass.
Timers::oneshot(std::chrono::hours(1000000), [] (auto) {});
Timers::ready();
}
/*
* External API
*/
cs_error_t
coroipcc_service_connect (
const char *socket_name,
unsigned int service,
size_t request_size,
size_t response_size,
size_t dispatch_size,
hdb_handle_t *handle)
{
int request_fd;
struct sockaddr_un address;
cs_error_t res;
struct ipc_instance *ipc_instance;
#if _POSIX_THREAD_PROCESS_SHARED < 1
key_t semkey = 0;
union semun semun;
#endif
int sys_res;
mar_req_setup_t req_setup;
mar_res_setup_t res_setup;
char control_map_path[PATH_MAX];
char request_map_path[PATH_MAX];
char response_map_path[PATH_MAX];
char dispatch_map_path[PATH_MAX];
res = hdb_error_to_cs (hdb_handle_create (&ipc_hdb,
sizeof (struct ipc_instance), handle));
if (res != CS_OK) {
return (res);
}
res = hdb_error_to_cs (hdb_handle_get (&ipc_hdb, *handle, (void **)&ipc_instance));
if (res != CS_OK) {
return (res);
}
res_setup.error = CS_ERR_LIBRARY;
#if defined(COROSYNC_SOLARIS)
request_fd = socket (PF_UNIX, SOCK_STREAM, 0);
#else
request_fd = socket (PF_LOCAL, SOCK_STREAM, 0);
#endif
if (request_fd == -1) {
return (CS_ERR_LIBRARY);
}
#ifdef SO_NOSIGPIPE
socket_nosigpipe (request_fd);
#endif
memset (&address, 0, sizeof (struct sockaddr_un));
address.sun_family = AF_UNIX;
#if defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN)
address.sun_len = SUN_LEN(&address);
#endif
#if defined(COROSYNC_LINUX)
sprintf (address.sun_path + 1, "%s", socket_name);
#else
sprintf (address.sun_path, "%s/%s", SOCKETDIR, socket_name);
#endif
sys_res = connect (request_fd, (struct sockaddr *)&address,
COROSYNC_SUN_LEN(&address));
if (sys_res == -1) {
res = CS_ERR_TRY_AGAIN;
goto error_connect;
}
sys_res = memory_map (
control_map_path,
"control_buffer-XXXXXX",
(void *)&ipc_instance->control_buffer,
8192);
if (sys_res == -1) {
res = CS_ERR_LIBRARY;
goto error_connect;
}
sys_res = memory_map (
request_map_path,
"request_buffer-XXXXXX",
(void *)&ipc_instance->request_buffer,
request_size);
if (sys_res == -1) {
res = CS_ERR_LIBRARY;
goto error_request_buffer;
}
sys_res = memory_map (
response_map_path,
"response_buffer-XXXXXX",
(void *)&ipc_instance->response_buffer,
response_size);
if (sys_res == -1) {
res = CS_ERR_LIBRARY;
goto error_response_buffer;
}
sys_res = circular_memory_map (
dispatch_map_path,
"dispatch_buffer-XXXXXX",
(void *)&ipc_instance->dispatch_buffer,
dispatch_size);
if (sys_res == -1) {
res = CS_ERR_LIBRARY;
goto error_dispatch_buffer;
}
#if _POSIX_THREAD_PROCESS_SHARED > 0
sem_init (&ipc_instance->control_buffer->sem_request_or_flush_or_exit, 1, 0);
sem_init (&ipc_instance->control_buffer->sem_request, 1, 0);
sem_init (&ipc_instance->control_buffer->sem_response, 1, 0);
sem_init (&ipc_instance->control_buffer->sem_dispatch, 1, 0);
#else
{
int i;
/*
* Allocate a semaphore segment
*/
while (1) {
semkey = random();
ipc_instance->euid = geteuid ();
if ((ipc_instance->control_buffer->semid
= semget (semkey, 4, IPC_CREAT|IPC_EXCL|0600)) != -1) {
break;
}
/*
* EACCESS can be returned as non root user when opening a different
* users semaphore.
*
* EEXIST can happen when we are a root or nonroot user opening
* an existing shared memory segment for which we have access
*/
if (errno != EEXIST && errno != EACCES) {
res = CS_ERR_LIBRARY;
goto error_exit;
}
}
for (i = 0; i < 4; i++) {
semun.val = 0;
sys_res = semctl (ipc_instance->control_buffer->semid, i, SETVAL, semun);
if (sys_res != 0) {
res = CS_ERR_LIBRARY;
goto error_exit;
}
}
}
#endif
/*
* Initialize IPC setup message
*/
req_setup.service = service;
strcpy (req_setup.control_file, control_map_path);
strcpy (req_setup.request_file, request_map_path);
strcpy (req_setup.response_file, response_map_path);
strcpy (req_setup.dispatch_file, dispatch_map_path);
req_setup.control_size = 8192;
req_setup.request_size = request_size;
req_setup.response_size = response_size;
req_setup.dispatch_size = dispatch_size;
#if _POSIX_THREAD_PROCESS_SHARED < 1
req_setup.semkey = semkey;
#endif
res = socket_send (request_fd, &req_setup, sizeof (mar_req_setup_t));
if (res != CS_OK) {
goto error_exit;
}
res = socket_recv (request_fd, &res_setup, sizeof (mar_res_setup_t));
if (res != CS_OK) {
goto error_exit;
}
ipc_instance->fd = request_fd;
if (res_setup.error == CS_ERR_TRY_AGAIN) {
res = res_setup.error;
goto error_exit;
}
ipc_instance->control_size = 8192;
ipc_instance->request_size = request_size;
ipc_instance->response_size = response_size;
ipc_instance->dispatch_size = dispatch_size;
pthread_mutex_init (&ipc_instance->mutex, NULL);
hdb_handle_put (&ipc_hdb, *handle);
return (res_setup.error);
error_exit:
#if _POSIX_THREAD_PROCESS_SHARED < 1
if (ipc_instance->control_buffer->semid > 0)
semctl (ipc_instance->control_buffer->semid, 0, IPC_RMID);
#endif
memory_unmap (ipc_instance->dispatch_buffer, dispatch_size);
error_dispatch_buffer:
memory_unmap (ipc_instance->response_buffer, response_size);
error_response_buffer:
memory_unmap (ipc_instance->request_buffer, request_size);
error_request_buffer:
memory_unmap (ipc_instance->control_buffer, 8192);
error_connect:
close (request_fd);
hdb_handle_destroy (&ipc_hdb, *handle);
hdb_handle_put (&ipc_hdb, *handle);
return (res);
}
void OS::start(uint32_t boot_magic, uint32_t boot_addr) {
atexit(default_exit);
default_stdout_handlers();
// Print a fancy header
FILLINE('=');
CAPTION("#include<os> // Literally\n");
FILLINE('=');
auto esp = get_cpu_esp();
MYINFO ("Stack: 0x%x", esp);
Expects (esp < 0xA0000 and esp > 0x0 and "Stack location OK");
MYINFO("Boot args: 0x%x (multiboot magic), 0x%x (bootinfo addr)",
boot_magic, boot_addr);
MYINFO("Max mem (from linker): %i MiB", reinterpret_cast<size_t>(&_MAX_MEM_MIB_));
if (boot_magic == MULTIBOOT_BOOTLOADER_MAGIC) {
OS::multiboot(boot_magic, boot_addr);
} else {
// Fetch CMOS memory info (unfortunately this is maximally 10^16 kb)
auto mem = cmos::meminfo();
low_memory_size_ = mem.base.total * 1024;
INFO2("* Low memory: %i Kib", mem.base.total);
high_memory_size_ = mem.extended.total * 1024;
// Use memsize provided by Make / linker unless CMOS knows this is wrong
decltype(high_memory_size_) hardcoded_mem = reinterpret_cast<size_t>(&_MAX_MEM_MIB_ - 0x100000) << 20;
if (mem.extended.total == 0xffff or hardcoded_mem < mem.extended.total) {
high_memory_size_ = hardcoded_mem;
INFO2("* High memory (from linker): %i Kib", high_memory_size_ / 1024);
} else {
INFO2("* High memory (from cmos): %i Kib", mem.extended.total);
}
}
MYINFO("Assigning fixed memory ranges (Memory map)");
auto& memmap = memory_map();
// @ Todo: The first ~600k of memory is free for use. What can we put there?
memmap.assign_range({0x0009FC00, 0x0009FFFF,
"EBDA", "Extended BIOS data area"});
memmap.assign_range({0x000A0000, 0x000FFFFF,
"VGA/ROM", "Memory mapped video memory"});
memmap.assign_range({(uintptr_t)&_LOAD_START_, (uintptr_t)&_end,
"ELF", "Your service binary including OS"});
// @note for security we don't want to expose this
memmap.assign_range({(uintptr_t)&_end + 1, heap_begin - 1,
"Pre-heap", "Heap randomization area (not for use))"});
memmap.assign_range({0x8000, 0x9fff, "Statman", "Statistics"});
memmap.assign_range({0xA000, 0x9fbff, "Kernel / service main stack"});
// Create ranges for heap and the remaining address space
// @note : since the maximum size of a span is unsigned (ptrdiff_t) we may need more than one
uintptr_t addr_max = std::numeric_limits<std::size_t>::max();
uintptr_t span_max = std::numeric_limits<std::ptrdiff_t>::max();
// Give the rest of physical memory to heap
heap_max_ = ((0x100000 + high_memory_size_) & 0xffff0000) - 1;
// ...Unless it's more than the maximum for a range
// @note : this is a stupid way to limit the heap - we'll change it, but not until
// we have a good solution.
heap_max_ = std::min(span_max, heap_max_);
memmap.assign_range({heap_begin, heap_max_,
"Heap", "Dynamic memory", heap_usage });
uintptr_t unavail_start = 0x100000 + high_memory_size_;
size_t interval = std::min(span_max, addr_max - unavail_start) - 1;
uintptr_t unavail_end = unavail_start + interval;
while (unavail_end < addr_max){
INFO2("* Unavailable memory: 0x%x - 0x%x", unavail_start, unavail_end);
memmap.assign_range({unavail_start, unavail_end,
"N/A", "Reserved / outside physical range" });
unavail_start = unavail_end + 1;
interval = std::min(span_max, addr_max - unavail_start);
// Increment might wrapped around
if (unavail_start > unavail_end + interval or unavail_start + interval == addr_max){
INFO2("* Last chunk of memory: 0x%x - 0x%x", unavail_start, addr_max);
memmap.assign_range({unavail_start, addr_max,
"N/A", "Reserved / outside physical range" });
break;
}
unavail_end += interval;
}
MYINFO("Printing memory map");
for (const auto &i : memory_map())
INFO2("* %s",i.second.to_string().c_str());
// Set up interrupt and exception handlers
IRQ_manager::init();
// read ACPI tables
hw::ACPI::init();
// setup APIC, APIC timer, SMP etc.
hw::APIC::init();
// enable interrupts
INFO("BSP", "Enabling interrupts");
IRQ_manager::enable_interrupts();
// Initialize the Interval Timer
hw::PIT::init();
// Initialize PCI devices
PCI_manager::init();
// Print registered devices
hw::Devices::print_devices();
// Estimate CPU frequency
MYINFO("Estimating CPU-frequency");
INFO2("|");
INFO2("+--(10 samples, %f sec. interval)",
(hw::PIT::frequency() / _cpu_sampling_freq_divider_).count());
INFO2("|");
// TODO: Debug why actual measurments sometimes causes problems. Issue #246.
cpu_mhz_ = hw::PIT::CPU_frequency();
INFO2("+--> %f MHz", cpu_mhz_.count());
// cpu_mhz must be known before we can start timer system
/// initialize timers hooked up to APIC timer
Timers::init(
// timer start function
hw::APIC_Timer::oneshot,
// timer stop function
hw::APIC_Timer::stop);
// initialize BSP APIC timer
hw::APIC_Timer::init(
[] {
// set final interrupt handler
hw::APIC_Timer::set_handler(Timers::timers_handler);
// signal that kernel is done with everything
Service::ready();
// signal ready
// NOTE: this executes the first timers, so we
// don't want to run this before calling Service ready
Timers::ready();
});
// Realtime/monotonic clock
RTC::init();
booted_at_ = RTC::now();
// sleep statistics
os_cycles_hlt = &Statman::get().create(
Stat::UINT64, std::string("cpu0.cycles_hlt")).get_uint64();
os_cycles_total = &Statman::get().create(
Stat::UINT64, std::string("cpu0.cycles_total")).get_uint64();
// Trying custom initialization functions
MYINFO("Calling custom initialization functions");
for (auto init : custom_init_) {
INFO2("* Calling %s", init.name_);
try{
init.func_();
} catch(std::exception& e){
MYINFO("Exception thrown when calling custom init: %s", e.what());
} catch(...){
MYINFO("Unknown exception when calling custom initialization function");
}
}
// Everything is ready
MYINFO("Starting %s", Service::name().c_str());
FILLINE('=');
// initialize random seed based on cycles since start
srand(cycles_since_boot() & 0xFFFFFFFF);
// begin service start
Service::start(Service::command_line());
event_loop();
}
void OS::start(uint32_t boot_magic, uint32_t boot_addr)
{
OS::cmdline = Service::binary_name();
// Initialize stdout handlers
if(os_default_stdout) {
OS::add_stdout(&OS::default_stdout);
}
PROFILE("OS::start");
// Print a fancy header
CAPTION("#include<os> // Literally");
MYINFO("Stack: %p", get_cpu_esp());
MYINFO("Boot magic: 0x%x, addr: 0x%x", boot_magic, boot_addr);
// Call global ctors
PROFILE("Global constructors");
__libc_init_array();
// PAGING //
PROFILE("Enable paging");
extern void __arch_init_paging();
__arch_init_paging();
// BOOT METHOD //
PROFILE("Multiboot / legacy");
OS::memory_end_ = 0;
// Detect memory limits etc. depending on boot type
if (boot_magic == MULTIBOOT_BOOTLOADER_MAGIC) {
OS::multiboot(boot_addr);
} else {
if (is_softreset_magic(boot_magic) && boot_addr != 0)
OS::resume_softreset(boot_addr);
OS::legacy_boot();
}
assert(OS::memory_end_ != 0);
// Give the rest of physical memory to heap
OS::heap_max_ = OS::memory_end_;
/// STATMAN ///
PROFILE("Statman");
/// initialize on page 9, 8 pages in size
Statman::get().init(0x8000, 0x8000);
PROFILE("Memory map");
// Assign memory ranges used by the kernel
auto& memmap = memory_map();
MYINFO("Assigning fixed memory ranges (Memory map)");
memmap.assign_range({0x8000, 0xffff, "Statman"});
#if defined(ARCH_x86_64)
/**
* TODO: Map binary parts using mem::map instead of assigning ranges directly
* e.g. the .text segment is now mapped individually by __arch_init_paging
*/
memmap.assign_range({0x1000, 0x6fff, "Pagetables"});
memmap.assign_range({0x10000, 0x9d3ff, "Stack"});
#elif defined(ARCH_i686)
memmap.assign_range({0x10000, 0x9d3ff, "Stack"});
#endif
assert(::heap_begin != 0x0 and OS::heap_max_ != 0x0);
// @note for security we don't want to expose this
memmap.assign_range({(uintptr_t)&_end, ::heap_begin - 1,
"Pre-heap"});
uintptr_t span_max = std::numeric_limits<std::ptrdiff_t>::max();
uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_);
MYINFO("Assigning heap");
memmap.assign_range({::heap_begin, heap_range_max_,
"Dynamic memory", heap_usage });
MYINFO("Virtual memory map");
for (const auto &i : memmap)
INFO2("%s",i.second.to_string().c_str());
PROFILE("Platform init");
extern void __platform_init();
__platform_init();
PROFILE("RTC init");
// Realtime/monotonic clock
RTC::init();
}
static void *huge_move_expand(struct thread_cache *cache, void *old_addr, size_t old_size, size_t new_size) {
struct arena *arena;
void *new_addr = huge_chunk_alloc(cache, new_size, CHUNK_SIZE, &arena);
if (unlikely(!new_addr)) {
return NULL;
}
bool gap = true;
if (unlikely(memory_remap_fixed(old_addr, old_size, new_addr, new_size))) {
memcpy(new_addr, old_addr, old_size);
if (purge_ratio >= 0) {
memory_decommit(old_addr, old_size);
}
gap = false;
} else {
// Attempt to fill the virtual memory hole. The kernel should provide a flag for preserving
// the old mapping to avoid the possibility of this failing and creating fragmentation.
//
// https://lkml.org/lkml/2014/10/2/624
void *extra = memory_map(old_addr, old_size, false);
if (likely(extra)) {
if (unlikely(extra != old_addr)) {
memory_unmap(extra, old_size);
} else {
gap = false;
}
}
}
struct extent_node key;
key.addr = old_addr;
struct arena *old_arena = get_huge_arena(old_addr);
extent_tree *huge = acquire_huge(old_arena);
struct extent_node *node = extent_tree_ad_search(huge, &key);
assert(node);
extent_tree_ad_remove(huge, node);
node->addr = new_addr;
node->size = new_size;
if (arena != old_arena) {
release_huge(old_arena);
huge = acquire_huge(arena);
}
extent_tree_ad_insert(huge, node);
release_huge(arena);
if (!gap) {
if (arena != old_arena && old_arena) {
mutex_lock(&old_arena->mutex);
}
chunk_free(get_recycler(old_arena), old_addr, old_size);
if (arena != old_arena && old_arena) {
mutex_unlock(&old_arena->mutex);
}
}
maybe_unlock_arena(arena);
return new_addr;
}
int coroipcs_handler_dispatch (
int fd,
int revent,
void *context)
{
mar_req_setup_t *req_setup;
struct conn_info *conn_info = (struct conn_info *)context;
int res;
char buf = 0;
if (ipc_thread_exiting (conn_info)) {
return conn_info_destroy (conn_info);
}
/*
* If an error occurs, request exit
*/
if (revent & (POLLERR|POLLHUP)) {
ipc_disconnect (conn_info);
return (0);
}
/*
* Read the header and process it
*/
if (conn_info->service == SOCKET_SERVICE_INIT && (revent & POLLIN)) {
pthread_attr_t thread_attr;
/*
* Receive in a nonblocking fashion the request
* IF security invalid, send ERR_SECURITY, otherwise
* send OK
*/
res = req_setup_recv (conn_info);
if (res != CS_OK && res != CS_ERR_LIBRARY) {
req_setup_send (conn_info, res);
}
if (res != CS_OK) {
return (0);
}
pthread_mutex_init (&conn_info->mutex, NULL);
req_setup = (mar_req_setup_t *)conn_info->setup_msg;
/*
* Is the service registered ?
* Has service init function ?
*/
if (api->service_available (req_setup->service) == 0 ||
api->init_fn_get (req_setup->service) == NULL) {
req_setup_send (conn_info, CS_ERR_NOT_EXIST);
ipc_disconnect (conn_info);
return (0);
}
#if _POSIX_THREAD_PROCESS_SHARED < 1
conn_info->semkey = req_setup->semkey;
#endif
res = memory_map (
req_setup->control_file,
req_setup->control_size,
(void *)&conn_info->control_buffer);
if (res == -1) {
goto send_setup_response;
}
conn_info->control_size = req_setup->control_size;
res = memory_map (
req_setup->request_file,
req_setup->request_size,
(void *)&conn_info->request_buffer);
if (res == -1) {
goto send_setup_response;
}
conn_info->request_size = req_setup->request_size;
res = memory_map (
req_setup->response_file,
req_setup->response_size,
(void *)&conn_info->response_buffer);
if (res == -1) {
goto send_setup_response;
}
conn_info->response_size = req_setup->response_size;
res = circular_memory_map (
req_setup->dispatch_file,
req_setup->dispatch_size,
(void *)&conn_info->dispatch_buffer);
if (res == -1) {
goto send_setup_response;
}
conn_info->dispatch_size = req_setup->dispatch_size;
send_setup_response:
if (res == 0) {
req_setup_send (conn_info, CS_OK);
} else {
req_setup_send (conn_info, CS_ERR_LIBRARY);
ipc_disconnect (conn_info);
return (0);
}
conn_info->service = req_setup->service;
conn_info->refcount = 0;
conn_info->setup_bytes_read = 0;
#if _POSIX_THREAD_PROCESS_SHARED < 1
conn_info->control_buffer->semid = semget (conn_info->semkey, 3, 0600);
#endif
conn_info->pending_semops = 0;
/*
* ipc thread is the only reference at startup
*/
conn_info->refcount = 1;
conn_info->state = CONN_STATE_THREAD_ACTIVE;
conn_info->private_data = api->malloc (api->private_data_size_get (conn_info->service));
memset (conn_info->private_data, 0,
api->private_data_size_get (conn_info->service));
api->init_fn_get (conn_info->service) (conn_info);
/* create stats objects */
coroipcs_init_conn_stats (conn_info);
pthread_attr_init (&thread_attr);
/*
* IA64 needs more stack space then other arches
*/
#if defined(__ia64__)
pthread_attr_setstacksize (&thread_attr, 400000);
#else
pthread_attr_setstacksize (&thread_attr, 200000);
#endif
pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_JOINABLE);
res = pthread_create (&conn_info->thread,
&thread_attr,
pthread_ipc_consumer,
conn_info);
pthread_attr_destroy (&thread_attr);
/*
* Security check - disallow multiple configurations of
* the ipc connection
*/
if (conn_info->service == SOCKET_SERVICE_INIT) {
conn_info->service = SOCKET_SERVICE_SECURITY_VIOLATION;
}
} else
if (revent & POLLIN) {
coroipcs_refcount_inc (conn_info);
res = recv (fd, &buf, 1, MSG_NOSIGNAL);
if (res == 1) {
switch (buf) {
case MESSAGE_REQ_CHANGE_EUID:
if (priv_change (conn_info) == -1) {
ipc_disconnect (conn_info);
}
break;
default:
res = 0;
break;
}
}
#if defined(COROSYNC_SOLARIS) || defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN)
/* On many OS poll never return POLLHUP or POLLERR.
* EOF is detected when recvmsg return 0.
*/
if (res == 0) {
ipc_disconnect (conn_info);
coroipcs_refcount_dec (conn_info);
return (0);
}
#endif
coroipcs_refcount_dec (conn_info);
}
if (revent & POLLOUT) {
int psop = conn_info->pending_semops;
int i;
assert (psop != 0);
for (i = 0; i < psop; i++) {
res = send (conn_info->fd, &buf, 1, MSG_NOSIGNAL);
if (res != 1) {
return (0);
} else {
conn_info->pending_semops -= 1;
}
}
if (conn_info->poll_state == POLL_STATE_INOUT) {
conn_info->poll_state = POLL_STATE_IN;
api->poll_dispatch_modify (conn_info->fd, POLLIN|POLLNVAL);
}
}
return (0);
}
