void pg_miss(void) { FAR struct tcb_s *ftcb = (FAR struct tcb_s*)g_readytorun.head; FAR struct tcb_s *wtcb; /* Sanity checking * * ASSERT if the currently executing task is the page fill worker thread. * The page fill worker thread is how the page fault is resolved and * all logic associated with the page fill worker must be "locked" and * always present in memory. */ pglldbg("Blocking TCB: %p PID: %d\n", ftcb, ftcb->pid); DEBUGASSERT(g_pgworker != ftcb->pid); /* Block the currently executing task * - Call up_block_task() to block the task at the head of the ready- * to-run list. This should cause an interrupt level context switch * to the next highest priority task. * - The blocked task will be marked with state TSTATE_WAIT_PAGEFILL * and will be retained in the g_waitingforfill prioritized task list. */ up_block_task(ftcb, TSTATE_WAIT_PAGEFILL); /* Boost the page fill worker thread priority. * - Check the priority of the task at the head of the g_waitingforfill * list. If the priority of that task is higher than the current * priority of the page fill worker thread, then boost the priority * of the page fill worker thread to that priority. */ wtcb = sched_gettcb(g_pgworker); DEBUGASSERT(wtcb != NULL); if (wtcb->sched_priority < ftcb->sched_priority) { /* Reprioritize the page fill worker thread */ pgllvdbg("New worker priority. %d->%d\n", wtcb->sched_priority, ftcb->sched_priority); sched_setpriority(wtcb, ftcb->sched_priority); } /* Signal the page fill worker thread. * - Is there a page fill pending? If not then signal the worker * thread to start working on the queued page fill requests. */ if (!g_pftcb) { pglldbg("Signaling worker. PID: %d\n", g_pgworker); kill(g_pgworker, SIGWORK); } }
static inline void pg_fillcomplete(void) { /* Call up_unblocktask(g_pftcb) to make the task that just * received the fill ready-to-run. */ pglldbg("Restarting TCB: %p\n", g_pftcb); up_unblock_task(g_pftcb); }
static void pg_callback(FAR struct tcb_s *tcb, int result) { /* Verify that g_pftcb is non-NULL */ pgllvdbg("g_pftcb: %p\n", g_pftcb); if (g_pftcb) { FAR struct tcb_s *htcb = (FAR struct tcb_s *)g_waitingforfill.head; FAR struct tcb_s *wtcb = sched_gettcb(g_pgworker); /* Find the higher priority between the task waiting for the fill to * complete in g_pftcb and the task waiting at the head of the * g_waitingforfill list. That will be the priority of he highest * priority task waiting for a fill. */ int priority = g_pftcb->sched_priority; if (htcb && priority < htcb->sched_priority) { priority = htcb->sched_priority; } /* If this higher priority is higher than current page fill worker * thread, then boost worker thread's priority to that level. Thus, * the page fill worker thread will always run at the priority of * the highest priority task that is waiting for a fill. */ if (priority > wtcb->sched_priority) { pgllvdbg("New worker priority. %d->%d\n", wtcb->sched_priority, priority); sched_setpriority(wtcb, priority); } /* Save the page fill result (don't permit the value -EBUSY) */ if (result == -EBUSY) { result = -ENOSYS; } g_fillresult = result; } /* Signal the page fill worker thread (in any event) */ pglldbg("Signaling worker. PID: %d\n", g_pgworker); kill(g_pgworker, SIGWORK); }
int up_fillpage(FAR struct tcb_s *tcb, FAR void *vpage, up_pgcallback_t pg_callback) { pglldbg("TCB: %p vpage: %d far: %08x\n", tcb, vpage, tcb->xcp.far); DEBUGASSERT(tcb->xcp.far >= PG_PAGED_VBASE && tcb->xcp.far < PG_PAGED_VEND); #if defined(CONFIG_PAGING_BINPATH) # error "File system-based paging must always be implemented with blocking calls" #elif defined(CONFIG_PAGING_M25PX) || defined(CONFIG_PAGING_AT45DB) # error "SPI FLASH paging must always be implemented with blocking calls" #else # warning "Not implemented" #endif return -ENOSYS; }
int pg_worker(int argc, char *argv[]) { irqstate_t flags; /* Loop forever -- Notice that interrupts will be disable at all times that * this thread runs. That is so that we con't lose signals or have * asynchronous page faults. * * All interrupt logic as well as all page fill worker thread logic must * be locked in memory. Therefore, keeping interrupts disabled here * should prevent any concurrent page faults. Any page faults or page * fill completions should occur while this thread sleeps. */ pglldbg("Started\n"); flags = irqsave(); for (;;) { /* Wait awhile. We will wait here until either the configurable timeout * elapses or until we are awakened by a signal (which terminates the * usleep with an EINTR error). Note that interrupts will be re-enabled * while this task sleeps. * * The timeout is a failsafe that will handle any cases where a single * is lost (that would really be a bug and shouldn't happen!) and also * supports timeouts for case of non-blocking, asynchronous fills. */ usleep(CONFIG_PAGING_WORKPERIOD); /* The page fill worker thread will be awakened on one of three conditions: * * - When signaled by pg_miss(), the page fill worker thread will be awakenend, * - if CONFIG_PAGING_BLOCKINGFILL is not defined, from pg_callback() * after completing a page fill, or * - On a configurable timeout expires with no activity. * * Interrupts are still disabled. */ #ifndef CONFIG_PAGING_BLOCKINGFILL /* For the non-blocking up_fillpage(), the page fill worker thread will detect * that the page fill is complete when it is awakened with g_pftcb non-NULL * and fill completion status from pg_callback. */ if (g_pftcb != NULL) { /* If it is a real page fill completion event, then the result of the page * fill will be in g_fillresult and will not be equal to -EBUSY. */ if (g_fillresult != -EBUSY) { /* Any value other than OK, brings the system down */ ASSERT(g_fillresult == OK); /* Handle the successful page fill complete event by restarting the * task that was blocked waiting for this page fill. */ pglldbg("Restarting TCB: %p\n", g_pftcb); up_unblock_task(g_pftcb);; /* Yes .. Start the next asynchronous fill. Check the return * value to see a fill was actually started (false means that * no fill was started). */ pgllvdbg("Calling pg_startfill\n"); if (!pg_startfill()) { /* No fill was started. This can mean only that all queued * page fill actions have and been completed and there is * nothing more to do. */ pgllvdbg("Call pg_alldone()\n"); pg_alldone(); } } /* If a configurable timeout period expires with no page fill completion * event, then declare a failure. */ #ifdef CONFIG_PAGING_TIMEOUT_TICKS else { lldbg("Timeout!\n"); ASSERT(clock_systimer() - g_starttime < CONFIG_PAGING_TIMEOUT_TICKS); } #endif } /* Otherwise, this might be a page fill initiation event. When * awakened from pg_miss(), no fill will be in progress and * g_pftcb will be NULL. */ else { /* Are there tasks blocked and waiting for a fill? If so, * pg_startfill() will start the asynchronous fill (and set * g_pftcb). */ pgllvdbg("Calling pg_startfill\n"); (void)pg_startfill(); } #else /* Are there tasks blocked and waiting for a fill? Loop until all * pending fills have been processed. */ for (;;) { /* Yes .. Start the fill and block until the fill completes. * Check the return value to see a fill was actually performed. * (false means that no fill was perforemd). */ pgllvdbg("Calling pg_startfill\n"); if (!pg_startfill()) { /* Break out of the loop -- there is nothing more to do */ break; } /* Handle the page fill complete event by restarting the * task that was blocked waiting for this page fill. In the * non-blocking fill case, the page fill worker thread will * know that the page fill is complete when pg_startfill() * returns true. */ pgllvdbg("Restarting TCB: %p\n", g_pftcb); up_unblock_task(g_pftcb);; } /* All queued fills have been processed */ pgllvdbg("Call pg_alldone()\n"); pg_alldone(); #endif } return OK; /* To keep some compilers happy */ }
static inline bool pg_startfill(void) { FAR void *vpage; int result; /* Remove the TCB at the head of the g_waitfor fill list and check if there * is any task waiting for a page fill. pg_dequeue will handle this (plus * some cornercases) and will true if the next page TCB was successfully * dequeued. */ if (pg_dequeue()) { /* Call up_allocpage(tcb, &vpage). This architecture-specific function will * set aside page in memory and map to virtual address (vpage). If all * available pages are in-use (the typical case), this function will select * a page in-use, un-map it, and make it available. */ pgllvdbg("Call up_allocpage(%p)\n", g_pftcb); result = up_allocpage(g_pftcb, &vpage); DEBUGASSERT(result == OK); /* Start the fill. The exact way that the fill is started depends upon * the nature of the architecture-specific up_fillpage() function -- Is it * a blocking or a non-blocking call? */ #ifdef CONFIG_PAGING_BLOCKINGFILL /* If CONFIG_PAGING_BLOCKINGFILL is defined, then up_fillpage is blocking * call. In this case, up_fillpage() will accept only (1) a reference to * the TCB that requires the fill. Architecture-specific context information * within the TCB will be sufficient to perform the fill. And (2) the * (virtual) address of the allocated page to be filled. The resulting * status of the fill will be provided by return value from up_fillpage(). */ pgllvdbg("Call up_fillpage(%p)\n", g_pftcb); result = up_fillpage(g_pftcb, vpage); DEBUGASSERT(result == OK); #else /* If CONFIG_PAGING_BLOCKINGFILL is defined, then up_fillpage is non-blocking * call. In this case up_fillpage() will accept an additional argument: The page * fill worker thread will provide a callback function, pg_callback. * * Calling up_fillpage will start an asynchronous page fill. pg_callback * ill be called when the page fill is finished (or an error occurs). This * This callback will probably from interrupt level. */ pgllvdbg("Call up_fillpage(%p)\n", g_pftcb); result = up_fillpage(g_pftcb, vpage, pg_callback); DEBUGASSERT(result == OK); /* Save the time that the fill was started. These will be used to check for * timeouts. */ #ifdef CONFIG_PAGING_TIMEOUT_TICKS g_starttime = clock_systimer(); #endif /* Return and wait to be signaled for the next event -- the fill completion * event. While the fill is in progress, other tasks may execute. If * another page fault occurs during this time, the faulting task will be * blocked, its TCB will be added (in priority order) to g_waitingforfill * and the priority of the page worker task may be boosted. But no action * will be taken until the current page fill completes. NOTE: The IDLE task * must also be fully locked in memory. The IDLE task cannot be blocked. It * the case where all tasks are blocked waiting for a page fill, the IDLE * task must still be available to run. */ #endif /* CONFIG_PAGING_BLOCKINGFILL */ return true; } pglldbg("Queue empty\n"); return false; }
static inline bool pg_dequeue(void) { /* Loop until either (1) the TCB of a task that requires a fill is found, OR * (2) the g_watingforfill list becomes empty. */ do { /* Remove the TCB from the head of the list (if any) */ g_pftcb = (FAR struct tcb_s *)dq_remfirst((dq_queue_t*)&g_waitingforfill); pgllvdbg("g_pftcb: %p\n", g_pftcb); if (g_pftcb != NULL) { /* Call the architecture-specific function up_checkmapping() to see if * the page fill still needs to be performed. In certain conditions, * the page fault may occur on several threads for the same page and * be queues multiple times. In this corner case, the blocked task will * simply be restarted. */ if (!up_checkmapping(g_pftcb)) { /* This page needs to be filled. pg_miss bumps up * the priority of the page fill worker thread as each * TCB is added to the g_waitingforfill list. So we * may need to also drop the priority of the worker * thread as the next TCB comes off of the list. * * If wtcb->sched_priority > CONFIG_PAGING_DEFPRIO, * then the page fill worker thread is executing at * an elevated priority that may be reduced. * * If wtcb->sched_priority > g_pftcb->sched_priority * then the page fill worker thread is executing at * a higher priority than is appropriate for this * fill (this priority can get re-boosted by pg_miss() * if a new higher priority fill is required). */ FAR struct tcb_s *wtcb = (FAR struct tcb_s *)g_readytorun.head; if (wtcb->sched_priority > CONFIG_PAGING_DEFPRIO && wtcb->sched_priority > g_pftcb->sched_priority) { /* Don't reduce the priority of the page fill * worker thread lower than the configured * minimum. */ int priority = g_pftcb->sched_priority; if (priority < CONFIG_PAGING_DEFPRIO) { priority = CONFIG_PAGING_DEFPRIO; } /* Reduce the priority of the page fill worker thread */ pgllvdbg("New worker priority. %d->%d\n", wtcb->sched_priority, priority); sched_setpriority(wtcb, priority); } /* Return with g_pftcb holding the pointer to * the TCB associated with task that requires the page fill. */ return true; } /* The page need by this task has already been mapped into the * virtual address space -- just restart it. */ pglldbg("Restarting TCB: %p\n", g_pftcb); up_unblock_task(g_pftcb); } } while (g_pftcb != NULL); return false; }
uint32_t *arm_dataabort(uint32_t *regs, uint32_t dfar, uint32_t dfsr) { DFAR struct tcb_s *tcb = (DFAR struct tcb_s *)g_readytorun.head; uint32_t *savestate; /* Save the saved processor context in current_regs where it can be accessed * for register dumps and possibly context switching. */ savestate = (uint32_t*)current_regs; current_regs = regs; /* In the NuttX on-demand paging implementation, only the read-only, .text * section is paged. However, the ARM compiler generated PC-relative data * fetches from within the .text sections. Also, it is customary to locate * read-only data (.rodata) within the same section as .text so that it * does not require copying to RAM. Misses in either of these case should * cause a data abort. * * We are only interested in data aborts due to page translations faults. * Sections should already be in place and permissions should already be * be set correctly (to read-only) so any other data abort reason is a * fatal error. */ pglldbg("DFSR: %08x DFAR: %08x\n", dfsr, dfar); if ((dfsr & FSR_MASK) != FSR_PAGE) { goto segfault; } /* Check the (virtual) address of data that caused the data abort. When * the exception occurred, this address was provided in the DFAR register. * (It has not yet been saved in the register context save area). */ pgllvdbg("VBASE: %08x VEND: %08x\n", PG_PAGED_VBASE, PG_PAGED_VEND); if (dfar < PG_PAGED_VBASE || dfar >= PG_PAGED_VEND) { goto segfault; } /* Save the offending data address as the fault address in the TCB of * the currently task. This fault address is also used by the prefetch * abort handling; this will allow common paging logic for both * prefetch and data aborts. */ tcb->xcp.dfar = regs[REG_R15]; /* Call pg_miss() to schedule the page fill. A consequences of this * call are: * * (1) The currently executing task will be blocked and saved on * on the g_waitingforfill task list. * (2) An interrupt-level context switch will occur so that when * this function returns, it will return to a different task, * most likely the page fill worker thread. * (3) The page fill worker task has been signalled and should * execute immediately when we return from this exception. */ pg_miss(); /* Restore the previous value of current_regs. NULL would indicate that * we are no longer in an interrupt handler. It will be non-NULL if we * are returning from a nested interrupt. */ current_regs = savestate; return regs; segfault: lldbg("Data abort. PC: %08x DFAR: %08x DFSR: %08x\n", regs[REG_PC], dfar, dfsr); PANIC(); return regs; /* To keep the compiler happy */ }
uint32_t *arm_prefetchabort(uint32_t *regs, uint32_t ifar, uint32_t ifsr) { uint32_t *savestate; /* Save the saved processor context in current_regs where it can be accessed * for register dumps and possibly context switching. */ savestate = (uint32_t *)current_regs; current_regs = regs; /* Get the (virtual) address of instruction that caused the prefetch abort. * When the exception occurred, this address was provided in the lr register * and this value was saved in the context save area as the PC at the * REG_R15 index. * * Check to see if this miss address is within the configured range of * virtual addresses. */ pglldbg("VADDR: %08x VBASE: %08x VEND: %08x\n", regs[REG_PC], PG_PAGED_VBASE, PG_PAGED_VEND); if (regs[REG_R15] >= PG_PAGED_VBASE && regs[REG_R15] < PG_PAGED_VEND) { /* Save the offending PC as the fault address in the TCB of the currently * executing task. This value is, of course, already known in regs[REG_R15], * but saving it in this location will allow common paging logic for both * prefetch and data aborts. */ struct tcb_s *tcb = this_task(); tcb->xcp.far = regs[REG_R15]; /* Call pg_miss() to schedule the page fill. A consequences of this * call are: * * (1) The currently executing task will be blocked and saved on * on the g_waitingforfill task list. * (2) An interrupt-level context switch will occur so that when * this function returns, it will return to a different task, * most likely the page fill worker thread. * (3) The page fill worker task has been signalled and should * execute immediately when we return from this exception. */ pg_miss(); /* Restore the previous value of current_regs. NULL would indicate that * we are no longer in an interrupt handler. It will be non-NULL if we * are returning from a nested interrupt. */ current_regs = savestate; } else { lldbg("Prefetch abort. PC: %08x IFAR: %08x IFSR: %08x\n", regs[REG_PC], ifar, ifsr); PANIC(); } return regs; }
int up_fillpage(FAR struct tcb_s *tcb, FAR void *vpage) { #if defined(CONFIG_PAGING_BINPATH) ssize_t nbytes; off_t offset; off_t pos; #elif defined(CONFIG_PAGING_M25PX) || defined(CONFIG_PAGING_AT45DB) ssize_t nbytes; off_t offset; #endif pglldbg("TCB: %p vpage: %p far: %08x\n", tcb, vpage, tcb->xcp.far); DEBUGASSERT(tcb->xcp.far >= PG_PAGED_VBASE && tcb->xcp.far < PG_PAGED_VEND); /* If BINPATH is defined, then it is the full path to a file on a mounted file * system. In this case initialization will be deferred until the first * time that up_fillpage() is called. Are we initialized? */ #if defined(CONFIG_PAGING_BINPATH) /* Perform initialization of the paging source device (if necessary) */ lpc31_initsrc(); /* Create an offset into the binary image that corresponds to the * virtual address. File offset 0 corresponds to PG_LOCKED_VBASE. */ offset = (off_t)tcb->xcp.far - PG_LOCKED_VBASE; /* Seek to that position */ pos = lseek(g_pgsrc.fd, offset, SEEK_SET); DEBUGASSERT(pos != (off_t)-1); /* And read the page data from that offset */ nbytes = read(g_pgsrc.fd, vpage, PAGESIZE); DEBUGASSERT(nbytes == PAGESIZE); return OK; #elif defined(CONFIG_PAGING_M25PX) || defined(CONFIG_PAGING_AT45DB) /* !CONFIG_PAGING_BINPATH */ /* Perform initialization of the paging source device (if necessary) */ lpc31_initsrc(); /* Create an offset into the binary image that corresponds to the * virtual address. File offset 0 corresponds to PG_LOCKED_VBASE. */ offset = (off_t)tcb->xcp.far - PG_LOCKED_VBASE + CONFIG_EA3131_PAGING_BINOFFSET; /* Read the page at the correct offset into the SPI FLASH device */ nbytes = MTD_READ(g_pgsrc.mtd, offset, PAGESIZE, (FAR uint8_t *)vpage); DEBUGASSERT(nbytes == PAGESIZE); return OK; #else /* !CONFIG_PAGING_BINPATH && !CONFIG_PAGING_M25PX && !CONFIG_PAGING_AT45DB */ # warning "Not implemented" return -ENOSYS; #endif /* !CONFIG_PAGING_BINPATH && !CONFIG_PAGING_M25PX && !CONFIG_PAGING_AT45DB */ }