int paging_init(uint32_t mem_size)
{
uint64_t mem_end = (uint64_t)mem_size * 1024ULL;
frames_count = (uint32_t)(mem_end / 0x1000ULL);
frames = (uint32_t*)static_alloc(BIT_INDEX(frames_count));
memset(frames, 0, BIT_INDEX(frames_count));
/* Create kernel page directory.
*/
kernel_directory = static_alloc(sizeof(*kernel_directory));
memset(kernel_directory, 0, sizeof(*kernel_directory));
/* Identity map pages up to heap address. We make the first page non-present so that NULL-pointer dereferences cause
* a page fault.
*/
frame_alloc(page_get(0, kernel_directory, true), 0);
uint32_t i = 0;
for (i = PAGE_SIZE; i < heap_addr; i += PAGE_SIZE) {
frame_alloc(page_get(i, kernel_directory, true), PAGE_FLAGS_PRESENT);
}
/* Map pages for the heap.
*/
for (; i < HEAP_ADDRESS + HEAP_SIZE_INIT; i += PAGE_SIZE) {
page_get(i, kernel_directory, true);
}
/* Set page fault handler.
*/
set_interrupt_handler(ISR_PAGE_FAULT, page_fault_handler);
page_directory_load(kernel_directory);
page_enable();
return 0;
}
struct thread *fault_page(struct thread *image) {
uint32_t cr2;
/* Get faulting address from register CR2 */
cr2 = cpu_get_cr2();
/* If in kernelspace, panic */
if ((image->cs & 0x3) == 0) { /* i.e. if it was kernelmode */
debug_printf("page fault at %x, ip = %x, frame %x, esp = %x\n",
cr2, image->eip, page_get(cr2), image->esp);
debug_panic("page fault exception");
}
if (cr2 >= image->stack && cr2 < image->stack + SEGSZ) {
/* allocate stack */
mem_alloc(cr2 & ~0xFFF, PAGESZ, PF_PRES | PF_RW | PF_USER);
return image;
}
else {
/* fault */
debug_printf("%d: %s: page fault at %x, ip = %x, frame %x\n",
image->proc->pid, image->proc->name, cr2, image->eip, page_get(cr2));
debug_printf("user stack: %x - %x\n", image->stack, image->stack + SEGSZ);
debug_printf("user stack dump: (ebp = %x)\n", image->ebp);
debug_dumpi((void*) image->useresp, 30);
process_freeze(image->proc);
return thread_send(image, image->proc->pid, PORT_PAGE, NULL);
}
}
static struct page *__alloc_pages_at(size_t addr, unsigned long n, struct page_zone *zone)
{
unsigned long flags;
struct page *page;
struct page *end;
struct page *p = NULL;
spin_lock_irq(&zone->lock, &flags);
end = page_struct(addr + (n * PAGE_SIZE));
for (page = page_struct(addr); page < end; page++) {
if (page->count)
goto out;
}
for (page = page_struct(addr); page < end; page++) {
page_get(page);
}
zone->num_free -= n;
p = page_struct(addr);
out:
spin_unlock_irq(&zone->lock, flags);
return p;
}
/* Big area management (size > PAGE_SIZE / 2) */
static void *big_malloc (int nb_pages)
{
page_descr_t *main_descr, *pool_head, *pool_descr;
main_descr = main_descr_get();
pool_head = pool_head_get(POOL_MAX);
pool_descr = page_descr_get();
if (pool_descr == NULL) {
MEMOPS_PRINTF("%s: cannot get pool descr\n", __func__);
return NULL;
}
pool_descr->addr = page_get(nb_pages);
if (pool_descr->addr == NULL) {
page_descr_put(pool_descr);
MEMOPS_PRINTF("%s: cannot get page\n", __func__);
return NULL;
}
if (page_cache_add_page(pool_descr, POOL_MAX) < 0) {
page_put(pool_descr->addr, nb_pages);
page_descr_put(pool_descr);
MEMOPS_PRINTF("%s: cannot get add page to cache\n", __func__);
return NULL;
}
pool_descr->prev = pool_head->prev;
pool_descr->next = pool_head;
pool_descr->nb = nb_pages;
pool_head->prev->next = pool_descr;
pool_head->prev = pool_descr;
return pool_descr->addr;
}
static inline struct slab_header *get_slab_header_from_page(struct page *pg)
{
struct slab_header *header;
header = (struct slab_header *)pg->free_base;
pg->free_base = pg->free_base + SLAB_HEADER_SIZE;
pg->free_size -= SLAB_HEADER_SIZE;
page_get(pg);
return header;
}
/* Global entry points */
int page_descrs_init (void)
{
page_descr_t *main_descr, *page_descr, *pool_head, *cache_head;
int i;
/* Allocate first descriptor page */
malloc_base = page_get(1);
if (malloc_base == NULL) {
set_errno(ENOMEM);
MEMOPS_PRINTF("%s: cannot get main descriptor\n", __func__);
return -1;
}
/* Init free slots in this page */
free_slots_init(malloc_base, sizeof(page_descr_t), PAGE_SIZE);
/* Init main descriptor */
page_descr = malloc_base;
main_descr = main_descr_get();
main_descr->addr = page_descr;
main_descr->nb = 0;
main_descr->next = page_descr;
main_descr->prev = page_descr;
page_descr->nb = 1;
page_descr->addr = page_descr + 2;
page_descr->next = main_descr;
page_descr->prev = main_descr;
/* Init pool lists heads */
for (i = 0; i <= POOL_MAX; i++) {
pool_head = page_descr_get();
if (pool_head == NULL) {
page_put(malloc_base, 1);
malloc_base = NULL;
MEMOPS_PRINTF("%s: cannot get pool descriptor %d\n", __func__, i);
return -1;
}
pool_head->prev = pool_head;
pool_head->next = pool_head;
pool_head->addr = NULL;
}
/* Init page caches lists heads */
for (i = 0; i < CACHE_MAX; i++) {
cache_head = page_descr_get();
if (cache_head == NULL) {
page_put(malloc_base, 1);
malloc_base = NULL;
MEMOPS_PRINTF("%s: cannot get page cache descriptor %d\n",
__func__, i);
return -1;
}
cache_head->prev = cache_head;
cache_head->next = cache_head;
}
return 0;
}
struct pde*
pgdir_create(void) {
struct pde* pgdir = page_get();
memset(pgdir, 0, PG_SIZE);
for (size_t i = PDX(UTOP); i < TBL_SIZE; ++i)
pgdir[i] = kpd[i];
return pgdir;
}
static void *pool_malloc (int pool_idx)
{
page_descr_t *main_descr, *pool_head, *pool_descr;
free_slot_t *first_free, *next_free;
main_descr = main_descr_get();
pool_head = pool_head_get(pool_idx);
pool_descr = pool_head->addr;
if (pool_descr == NULL || pool_descr->addr == NULL) {
pool_descr = descr_find_free(pool_head, NULL);
if (pool_descr == NULL) {
pool_descr = page_descr_get();
if (pool_descr == NULL) {
MEMOPS_PRINTF("%s: cannot get pool descr\n", __func__);
return NULL;
}
pool_descr->addr = page_get(1);
if (pool_descr->addr == NULL) {
MEMOPS_PRINTF("%s: cannot allocate new page\n", __func__);
page_descr_put(pool_descr);
return NULL;
}
if (page_cache_add_page(pool_descr, pool_idx) < 0) {
MEMOPS_PRINTF("%s: cannot add new page to cache\n", __func__);
page_put(pool_descr->addr, 1);
page_descr_put(pool_descr);
return NULL;
}
free_slots_init(pool_descr->addr,
1 << (MIN_ELEM_BITS + pool_idx), PAGE_SIZE);
pool_descr->nb = 0;
pool_descr->prev = pool_head->prev;
pool_descr->next = pool_head;
pool_head->prev->next = pool_descr;
pool_head->prev = pool_descr;
pool_head->nb++;
}
pool_head->addr = pool_descr;
}
first_free = pool_descr->addr;
next_free = first_free->next;
// memset(first_free, 0, 1 << (MIN_ELEM_BITS + pool_idx));
pool_descr->addr = next_free;
if (pool_descr->nb == 0)
pool_head->nb--;
pool_descr->nb++;
return first_free;
}
static page_descr_t *page_descr_get (void)
{
page_descr_t *main_descr, *head_descr, *page_descr;
free_slot_t *first_free;
main_descr = main_descr_get();
head_descr = main_descr->addr;
first_free = head_descr->addr;
if (first_free == NULL) {
/* Find a page with free descriptors */
head_descr = descr_find_free(main_descr, NULL);
if (head_descr != NULL) {
/* Get the first free slot */
first_free = head_descr->addr;
} else {
/* Allocate a new page */
head_descr = page_get(1);
if (head_descr == NULL) {
MEMOPS_PRINTF("%s: cannot get new head descriptor\n",
__func__);
return NULL;
}
/* Initialise free slots */
free_slots_init(head_descr, sizeof(page_descr_t), PAGE_SIZE);
/* Initialise page head descriptor */
head_descr->addr = head_descr + 1;
head_descr->nb = 0;
head_descr->next = main_descr;
head_descr->prev = main_descr->prev;
/* Update main descriptor */
main_descr->prev->next = head_descr;
main_descr->prev = head_descr;
main_descr->nb++;
first_free = head_descr->addr;
}
main_descr->addr = head_descr;
}
head_descr->addr = first_free->next;
if (head_descr->nb == 0)
main_descr->nb--;
head_descr->nb++;
page_descr = (page_descr_t *)first_free;
page_descr->prev = NULL;
page_descr->next = NULL;
page_descr->addr = NULL;
page_descr->nb = 0;
return page_descr;
}
struct page *__alloc_pages(unsigned long n, struct page_zone *zone)
{
unsigned long flags;
struct page *pages;
struct page *p;
spin_lock_irq(&zone->lock, &flags);
pages = find_contig_pages(n, zone);
if (!pages) {
goto alloc_pages_out;
}
for (p = pages; p < pages + n; p++) {
page_get(p);
}
zone->num_free -= n;
alloc_pages_out:
spin_unlock_irq(&zone->lock, flags);
return pages;
}
void fault_page(struct thread *image) {
uint32_t cr2;
/* Get faulting address from register CR2 */
cr2 = cpu_get_cr2();
/* If in kernelspace, panic */
if ((image->cs & 0x3) == 0) { /* i.e. if it was kernelmode */
debug_printf("page fault at %x, ip = %x, frame %x, esp = %x\n",
cr2, image->eip, page_get(cr2), image->esp);
debug_panic("page fault exception");
}
/* fault */
thread_save(image);
image->fault_addr = cr2;
image->fault = FV_PAGE;
image->state = TS_PAUSED;
// add to fault queue
fault_push(image);
}
void kcall(struct thread *image) {
if (image->flags & TF_USER) {
// perform syscall
// save user state
image->usr_eip = image->eip;
image->usr_esp = image->useresp;
// switch to system mode
image->ss = 0x21;
image->ds = 0x21;
image->cs = 0x19;
image->flags &= ~TF_USER;
// restore system state
image->eip = image->sys_eip;
image->useresp = image->sys_esp;
return;
}
switch (image->eax) {
case KCALL_SPAWN: {
int id = thread_new();
if (id == -1) {
// failure to create new thread
image->eax = -1;
break;
}
struct thread *thread = thread_get(id);
struct t_info *state = (void*) image->ebx;
// initialize thread state
thread->useresp = state->regs.esp;
thread->esp = (uintptr_t) &thread->num;
thread->ss = 0x21;
thread->ds = 0x21;
thread->cs = 0x19;
thread->eflags = image->eflags;
thread->eip = state->regs.eip;
thread->ebp = state->regs.ebp;
thread->esi = state->regs.esi;
thread->edi = state->regs.edi;
thread->edx = state->regs.edx;
thread->ecx = state->regs.ecx;
thread->ebx = state->regs.ebx;
thread->eax = state->regs.eax;
// add to scheduler queue
thread->state = TS_QUEUED;
schedule_push(thread);
image->eax = id;
break;
}
case KCALL_REAP: {
struct thread *target = thread_get(image->ebx);
if (target->state != TS_PAUSED) {
image->eax = TE_STATE;
}
else {
if (image->ecx) {
save_info((void*) image->ecx, target);
}
image->eax = 0;
thread_kill(target);
}
break;
}
case KCALL_GETTID: {
image->eax = image->id;
break;
}
case KCALL_YIELD: {
thread_save(image);
schedule_push(image);
image->state = TS_QUEUED;
break;
}
case KCALL_PAUSE: {
struct thread *target = thread_get(image->ebx);
if (image->ebx == (uint32_t) -1) target = image;
if (!target) {
image->eax = TE_EXIST;
}
else switch (target->state) {
case TS_RUNNING:
// pause (normal, from running)
thread_save(target);
target->state = TS_PAUSED;
image->eax = 0;
break;
case TS_QUEUED:
// pause (normal, from queued)
schedule_remv(target);
target->state = TS_PAUSED;
image->eax = 0;
break;
case TS_WAITING:
// pause (waiting)
event_remv(target->id, target->event);
target->state = TS_PAUSEDW;
image->eax = 0;
break;
default:
// invalid state transition
image->eax = TE_STATE;
break;
}
break;
}
case KCALL_RESUME: {
struct thread *target = thread_get(image->ebx);
if (!target) {
image->eax = TE_EXIST;
}
else switch (target->state) {
case TS_PAUSED:
// resume thread by scheduling
schedule_push(target);
target->state = TS_QUEUED;
image->eax = 0;
break;
case TS_PAUSEDW:
// resume thread by entering wait queue
event_wait(target->id, target->event);
target->state = TS_WAITING;
image->eax = 0;
break;
default:
image->eax = TE_STATE;
break;
}
break;
}
case KCALL_GETSTATE: {
struct thread *target = thread_get(image->ebx);
if (image->ebx == (uint32_t) -1) target = image;
if (!target) {
image->eax = TE_EXIST;
}
else if (target->state != TS_PAUSED && target->state != TS_PAUSEDW && target != image) {
image->eax = TE_STATE;
}
else {
save_info((void*) image->ecx, target);
image->eax = 0;
}
break;
}
case KCALL_SETSTATE: {
struct thread *target = thread_get(image->ebx);
if (image->ebx == (uint32_t) -1) target = image;
if (!target) {
image->eax = TE_EXIST;
}
else switch (target->state) {
case TS_PAUSED:
case TS_PAUSEDW:
case TS_RUNNING: {
struct t_info *src = (void*) image->ecx;
if (src->flags & TF_DEAD) {
// kill thread
if (target->state == TS_RUNNING) {
thread_save(target);
target->state = TS_PAUSED;
}
// notify reaper
dead_push(target);
}
if (target->pctx != src->pctx) {
// change paging contexts
pctx_load(src->pctx);
}
// save thread state
target->pctx = src->pctx;
target->flags = src->flags;
target->fault = src->fault;
if (target->state != TS_RUNNING) {
// save register state
target->edi = src->regs.edi;
target->esi = src->regs.esi;
target->ebp = src->regs.ebp;
target->useresp = src->regs.esp;
target->ebx = src->regs.ebx;
target->edx = src->regs.edx;
target->ecx = src->regs.ecx;
target->eax = src->regs.eax;
target->eip = src->regs.eip;
target->eflags = src->regs.eflags;
// save MMX/SSE state
if (!target->fxdata) target->fxdata = heap_alloc(512);
memcpy(target->fxdata, &src->regs.fxdata[0], 512);
}
target->usr_eip = src->usr_ip;
target->usr_esp = src->usr_sp;
image->eax = 0;
break;
}
default:
image->eax = TE_STATE;
break;
}
break;
}
case KCALL_GETDEAD: {
if (dead_peek()) {
struct thread *dead = dead_pull();
image->eax = dead->id;
}
else {
thread_save(image);
image->state = TS_PAUSED;
dead_wait(image);
}
break;
}
case KCALL_GETFAULT: {
if (fault_peek()) {
struct thread *fault = fault_pull();
image->eax = fault->id;
}
else {
thread_save(image);
image->state = TS_PAUSED;
fault_wait(image);
}
break;
}
case KCALL_WAIT: {
image->eax = event_wait(image->id, image->ebx);
break;
}
case KCALL_RESET: {
if (image->ebx < 240) {
extern int irqstate[EV_COUNT];
irqstate[image->ebx] = 0;
irq_unmask(image->ebx);
}
image->eax = 0;
break;
}
case KCALL_SYSRET: {
// save system state
image->sys_eip = image->eip;
image->sys_esp = image->useresp;
// perform return value swap-in
image->eax = image->ebp;
// switch to usermode
image->ss = 0x33;
image->ds = 0x33;
image->cs = 0x2B;
image->flags |= TF_USER;
// restore user state
image->eip = image->usr_eip;
image->useresp = image->usr_esp;
break;
}
case KCALL_NEWPCTX: {
image->eax = pctx_new();
break;
}
case KCALL_FREEPCTX: {
image->eax = pctx_free(image->ebx);
break;
}
case KCALL_SETFRAME: {
page_set(image->ebx, page_fmt(image->ecx, page_get(image->ebx)));
image->eax = 0;
break;
}
case KCALL_SETFLAGS: {
page_set(image->ebx, page_fmt(page_ufmt(page_get(image->ebx)), image->ecx));
image->eax = 0;
break;
}
case KCALL_GETFRAME: {
uint32_t off = image->ebx & 0xFFF;
uint32_t page = image->ebx & ~0xFFF;
image->eax = page_ufmt(page_get(page)) | off;
break;
}
case KCALL_GETFLAGS: {
image->eax = page_get(image->ebx) & PF_MASK;
break;
}
case KCALL_NEWFRAME: {
uint64_t frame = frame_new();
image->eax = frame & 0xFFFFFFFF;
image->ebx = frame >> 32ULL;
break;
}
case KCALL_FREEFRAME: {
frame_free(image->ebx);
image->eax = 0;
break;
}
case KCALL_TAKEFRAME: {
image->eax = 1;
break;
}
default: {
debug_printf("warning: unimplemented kcall %d\n", image->eax);
image->eax = -1;
break;
}
}
}