static void pth_thread_refl_fault(struct uthread *uthread, unsigned int trap_nr,
unsigned int err, unsigned long aux)
{
struct pthread_tcb *pthread = (struct pthread_tcb*)uthread;
pthread->state = PTH_BLK_SYSC;
mcs_pdr_lock(&queue_lock);
threads_active--;
TAILQ_REMOVE(&active_queue, pthread, tq_next);
mcs_pdr_unlock(&queue_lock);
/* TODO: RISCV/x86 issue! (0 is divby0, 14 is PF, etc) */
#if defined(__i386__) || defined(__x86_64__)
switch(trap_nr) {
case 0:
handle_div_by_zero(uthread, err, aux);
break;
case 13:
handle_gp_fault(uthread, err, aux);
break;
case 14:
handle_page_fault(uthread, err, aux);
break;
default:
printf("Pthread has unhandled fault: %d, err: %d, aux: %p\n",
trap_nr, err, aux);
/* Note that uthread.c already copied out our ctx into the uth
* struct */
print_user_context(&uthread->u_ctx);
printf("Turn on printx to spew unhandled, malignant trap info\n");
exit(-1);
}
#else
#error "Handling hardware faults is currently only supported on x86"
#endif
}
/*
* We give a *copy* of the faultinfo in the regs to segv.
* This must be done, since nesting SEGVs could overwrite
* the info in the regs. A pointer to the info then would
* give us bad data!
*/
unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user, void *sc)
{
struct siginfo si;
void *catcher;
int err;
int is_write = FAULT_WRITE(fi);
unsigned long address = FAULT_ADDRESS(fi);
if(!is_user && (address >= start_vm) && (address < end_vm)){
flush_tlb_kernel_vm();
return(0);
}
else if(current->mm == NULL)
panic("Segfault with no mm");
if (SEGV_IS_FIXABLE(&fi) || SEGV_MAYBE_FIXABLE(&fi))
err = handle_page_fault(address, ip, is_write, is_user, &si.si_code);
else {
err = -EFAULT;
/* A thread accessed NULL, we get a fault, but CR2 is invalid.
* This code is used in __do_copy_from_user() of TT mode. */
address = 0;
}
catcher = current->thread.fault_catcher;
if(!err)
return(0);
else if(catcher != NULL){
current->thread.fault_addr = (void *) address;
do_longjmp(catcher, 1);
}
else if(current->thread.fault_addr != NULL)
panic("fault_addr set but no fault catcher");
else if(!is_user && arch_fixup(ip, sc))
return(0);
if(!is_user)
panic("Kernel mode fault at addr 0x%lx, ip 0x%lx",
address, ip);
if (err == -EACCES) {
si.si_signo = SIGBUS;
si.si_errno = 0;
si.si_code = BUS_ADRERR;
si.si_addr = (void __user *)address;
current->thread.arch.faultinfo = fi;
force_sig_info(SIGBUS, &si, current);
} else if (err == -ENOMEM) {
printk("VM: killing process %s\n", current->comm);
do_exit(SIGKILL);
} else {
BUG_ON(err != -EFAULT);
si.si_signo = SIGSEGV;
si.si_addr = (void __user *) address;
current->thread.arch.faultinfo = fi;
force_sig_info(SIGSEGV, &si, current);
}
return(0);
}
void isr_handler(registers_t* regs)
{
if(exception_depth++ >= 3){
printf("Exception depth exceeded 3");
while(1);
}
if((regs->int_no < 32) && (isFatal(regs->int_no))){
terminal_bluescreen();
char* name = interrupt_name(regs->int_no);
printf("\t\t\t\t\t\tSOMETHING JUST WENT VERY WRONG\n\nI'd like to");
printf(" interject for a moment. What you're referring to as:\n\n\t");
printf("Interrupt %d\n\nIs in fact, a fatal exception, ", (int) regs->int_no);
printf("or as I've recently taken to calling it:\n\n");
printf("\t%s with error code %p\n", name, (void *) regs->err_code);
//Halt the machine
while(1);
}
if(regs->int_no >= 32){
//Handle our IRQs
//interrupt numbers 0-31 are reserved by intel, so subtract 32
//to get the isa irq code. 32 is an arbitary number decided in the
//PIC initialization code
int isa_irq = (regs->int_no - 32);
switch(isa_irq){
case 0:
//Programmable interval timer interrupt
break;
case 1:
//Keyboard interrupt
handle_keyboard_interrupt();
break;
}
PIC_sendEOI(regs->int_no);
}else{
//It is an exception we can (and should) handle
switch(regs->int_no){
case 14:
handle_page_fault(regs->err_code);
break;
}
}
exception_depth = 0;
return;
}
static void
handle_fault_store(struct hw_trapframe *state)
{
if(in_kernel(state))
{
print_trapframe(state);
panic("Store Page Fault in the Kernel at %p!", state->badvaddr);
}
set_current_ctx_hw(&per_cpu_info[core_id()], state);
if(handle_page_fault(current, state->badvaddr, PROT_WRITE))
unhandled_trap(state, "Store Page Fault");
}
static void
handle_fault_fetch(struct hw_trapframe *state)
{
if(in_kernel(state))
{
print_trapframe(state);
panic("Instruction Page Fault in the Kernel at %p!", state->epc);
}
set_current_ctx_hw(&per_cpu_info[core_id()], state);
#warning "returns EAGAIN if you should reflect the fault"
if(handle_page_fault(current, state->epc, PROT_EXEC))
unhandled_trap(state, "Instruction Page Fault");
}
static void
handle_fault_load(struct hw_trapframe *state)
{
if(in_kernel(state))
{
print_trapframe(state);
panic("Load Page Fault in the Kernel at %p!", state->badvaddr);
}
set_current_ctx_hw(&per_cpu_info[core_id()], state);
#warning "returns EAGAIN if you should reflect the fault"
if(handle_page_fault(current, state->badvaddr, PROT_READ))
unhandled_trap(state, "Load Page Fault");
}
/*
* Check an async_tlb structure to see if a deferred fault is waiting,
* and if so pass it to the page-fault code.
*/
static void handle_async_page_fault(struct pt_regs *regs,
struct async_tlb *async)
{
if (async->fault_num) {
/*
* Clear async->fault_num before calling the page-fault
* handler so that if we re-interrupt before returning
* from the function we have somewhere to put the
* information from the new interrupt.
*/
int fault_num = async->fault_num;
async->fault_num = 0;
handle_page_fault(regs, fault_num, async->is_fault,
async->address, async->is_write);
}
}
static unsigned long maybe_map(unsigned long virt, int is_write)
{
pte_t pte;
int err;
void *phys = um_virt_to_phys(current, virt, &pte);
int dummy_code;
if(IS_ERR(phys) || (is_write && !pte_write(pte))) {
err = handle_page_fault(virt, 0, is_write, 1, &dummy_code);
if(err)
return(0);
phys = um_virt_to_phys(current, virt, NULL);
}
return((unsigned long) phys);
}
static pte_t *maybe_map(unsigned long virt, int is_write)
{
pte_t *pte = virt_to_pte(current->mm, virt);
int err, dummy_code;
if ((pte == NULL) || !pte_present(*pte) ||
(is_write && !pte_write(*pte))) {
err = handle_page_fault(virt, 0, is_write, 1, &dummy_code);
if (err)
return NULL;
pte = virt_to_pte(current->mm, virt);
}
if (!pte_present(*pte))
pte = NULL;
return pte;
}
static void thread0_thread_refl_fault(struct uthread *uth,
struct user_context *ctx)
{
unsigned int trap_nr = __arch_refl_get_nr(ctx);
unsigned int err = __arch_refl_get_err(ctx);
unsigned long aux = __arch_refl_get_aux(ctx);
assert(ctx->type == ROS_HW_CTX);
switch (trap_nr) {
case HW_TRAP_PAGE_FAULT:
if (!handle_page_fault(uth, err, aux))
refl_error(uth, trap_nr, err, aux);
break;
default:
refl_error(uth, trap_nr, err, aux);
}
}
void handle_trap(struct interrupt_frame *frame)
{
unsigned int address;
int trap_cause = __builtin_nyuzi_read_control_reg(CR_TRAP_CAUSE);
switch (trap_cause & 0xf)
{
case TT_PAGE_FAULT:
case TT_ILLEGAL_STORE:
address = __builtin_nyuzi_read_control_reg(CR_TRAP_ADDR);
enable_interrupts();
if (!handle_page_fault(address, (trap_cause & 0x10) != 0))
{
// Jump to user_copy fault handler if set
if (fault_handler[current_hw_thread()] != 0)
frame->pc = fault_handler[current_hw_thread()];
else
bad_fault(frame);
}
disable_interrupts();
break;
case TT_SYSCALL:
// Enable interrupts
address = __builtin_nyuzi_read_control_reg(CR_TRAP_ADDR);
enable_interrupts();
frame->gpr[0] = handle_syscall(frame->gpr[0], frame->gpr[1],
frame->gpr[2], frame->gpr[3],
frame->gpr[4], frame->gpr[5]);
frame->pc += 4; // Next instruction
disable_interrupts();
break;
case TT_INTERRUPT:
handle_interrupt(frame);
break;
default:
bad_fault(frame);
}
}
/*
* This routine effectively re-issues asynchronous page faults
* when we are returning to user space.
*/
void do_async_page_fault(struct pt_regs *regs)
{
struct async_tlb *async = ¤t->thread.dma_async_tlb;
/*
* Clear thread flag early. If we re-interrupt while processing
* code here, we will reset it and recall this routine before
* returning to user space.
*/
clear_thread_flag(TIF_ASYNC_TLB);
if (async->fault_num) {
/*
* Clear async->fault_num before calling the page-fault
* handler so that if we re-interrupt before returning
* from the function we have somewhere to put the
* information from the new interrupt.
*/
int fault_num = async->fault_num;
async->fault_num = 0;
handle_page_fault(regs, fault_num, async->is_fault,
async->address, async->is_write);
}
}
static void handle_fault_fetch(trapframe_t* tf)
{
if (handle_page_fault(tf->badvaddr, PROT_EXEC) != 0)
segfault(tf, tf->badvaddr, "fetch");
}
int
vm_event(vm_t* vm, seL4_MessageInfo_t tag)
{
seL4_Word label;
seL4_Word length;
label = seL4_MessageInfo_get_label(tag);
length = seL4_MessageInfo_get_length(tag);
switch (label) {
case SEL4_PFIPC_LABEL: {
int err;
fault_t* fault;
fault = vm->fault;
err = new_fault(fault);
assert(!err);
do {
err = handle_page_fault(vm, fault);
if (err) {
return -1;
}
} while (!fault_handled(fault));
}
break;
case SEL4_EXCEPT_IPC_LABEL: {
int err;
assert(length == SEL4_EXCEPT_IPC_LENGTH);
err = handle_syscall(vm, length);
assert(!err);
if (!err) {
seL4_MessageInfo_t reply;
reply = seL4_MessageInfo_new(0, 0, 0, 0);
seL4_Reply(reply);
}
}
break;
case SEL4_USER_EXCEPTION_LABEL: {
seL4_Word ip;
int err;
assert(length == SEL4_USER_EXCEPTION_LENGTH);
ip = seL4_GetMR(0);
err = handle_exception(vm, ip);
assert(!err);
if (!err) {
seL4_MessageInfo_t reply;
reply = seL4_MessageInfo_new(0, 0, 0, 0);
seL4_Reply(reply);
}
}
break;
case SEL4_VGIC_MAINTENANCE_LABEL: {
int idx;
int err;
assert(length == SEL4_VGIC_MAINTENANCE_LENGTH);
idx = seL4_GetMR(EXCEPT_IPC_SYS_MR_R0);
/* Currently not handling spurious IRQs */
assert(idx >= 0);
err = handle_vgic_maintenance(vm, idx);
assert(!err);
if (!err) {
seL4_MessageInfo_t reply;
reply = seL4_MessageInfo_new(0, 0, 0, 0);
seL4_Reply(reply);
}
}
break;
case SEL4_VCPU_FAULT_LABEL: {
seL4_MessageInfo_t reply;
seL4_UserContext regs;
seL4_CPtr tcb;
uint32_t hsr;
int err;
assert(length == SEL4_VCPU_FAULT_LENGTH);
hsr = seL4_GetMR(EXCEPT_IPC_SYS_MR_R0);
/* Increment the PC and ignore the fault */
tcb = vm_get_tcb(vm);
err = seL4_TCB_ReadRegisters(tcb, false, 0,
sizeof(regs) / sizeof(regs.pc), ®s);
assert(!err);
switch (hsr) {
case HSR_WFI:
case HSR_WFE:
regs.pc += (regs.cpsr & BIT(5)) ? 2 : 4;
err = seL4_TCB_WriteRegisters(tcb, false, 0,
sizeof(regs) / sizeof(regs.pc), ®s);
assert(!err);
reply = seL4_MessageInfo_new(0, 0, 0, 0);
seL4_Reply(reply);
return 0;
default:
printf("Unhandled VCPU fault from [%s]: HSR 0x%08x\n", vm->name, hsr);
print_ctx_regs(®s);
return -1;
}
}
break;
default:
/* What? Why are we here? What just happened? */
printf("Unknown fault from [%s]: label=0x%x length=0x%x\n",
vm->name, label, length);
return -1;
}
return 0;
}
/*
* This routine handles page faults. It determines the address, and the
* problem, and then passes it handle_page_fault() for normal DTLB and
* ITLB issues, and for DMA or SN processor faults when we are in user
* space. For the latter, if we're in kernel mode, we just save the
* interrupt away appropriately and return immediately. We can't do
* page faults for user code while in kernel mode.
*/
void do_page_fault(struct pt_regs *regs, int fault_num,
unsigned long address, unsigned long write)
{
int is_page_fault;
/* This case should have been handled by do_page_fault_ics(). */
BUG_ON(write & ~1);
#if CHIP_HAS_TILE_DMA()
/*
* If it's a DMA fault, suspend the transfer while we're
* handling the miss; we'll restart after it's handled. If we
* don't suspend, it's possible that this process could swap
* out and back in, and restart the engine since the DMA is
* still 'running'.
*/
if (fault_num == INT_DMATLB_MISS ||
fault_num == INT_DMATLB_ACCESS ||
fault_num == INT_DMATLB_MISS_DWNCL ||
fault_num == INT_DMATLB_ACCESS_DWNCL) {
__insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
while (__insn_mfspr(SPR_DMA_USER_STATUS) &
SPR_DMA_STATUS__BUSY_MASK)
;
}
#endif
/* Validate fault num and decide if this is a first-time page fault. */
switch (fault_num) {
case INT_ITLB_MISS:
case INT_DTLB_MISS:
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_MISS:
case INT_DMATLB_MISS_DWNCL:
#endif
#if CHIP_HAS_SN_PROC()
case INT_SNITLB_MISS:
case INT_SNITLB_MISS_DWNCL:
#endif
is_page_fault = 1;
break;
case INT_DTLB_ACCESS:
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_ACCESS:
case INT_DMATLB_ACCESS_DWNCL:
#endif
is_page_fault = 0;
break;
default:
panic("Bad fault number %d in do_page_fault", fault_num);
}
#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
if (EX1_PL(regs->ex1) != USER_PL) {
struct async_tlb *async;
switch (fault_num) {
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_MISS:
case INT_DMATLB_ACCESS:
case INT_DMATLB_MISS_DWNCL:
case INT_DMATLB_ACCESS_DWNCL:
async = ¤t->thread.dma_async_tlb;
break;
#endif
#if CHIP_HAS_SN_PROC()
case INT_SNITLB_MISS:
case INT_SNITLB_MISS_DWNCL:
async = ¤t->thread.sn_async_tlb;
break;
#endif
default:
async = NULL;
}
if (async) {
/*
* No vmalloc check required, so we can allow
* interrupts immediately at this point.
*/
local_irq_enable();
set_thread_flag(TIF_ASYNC_TLB);
if (async->fault_num != 0) {
panic("Second async fault %d;"
" old fault was %d (%#lx/%ld)",
fault_num, async->fault_num,
address, write);
}
BUG_ON(fault_num == 0);
async->fault_num = fault_num;
async->is_fault = is_page_fault;
async->is_write = write;
async->address = address;
return;
}
}
#endif
handle_page_fault(regs, fault_num, is_page_fault, address, write);
}
/*
* stress_userfaultfd_oomable()
* stress userfaultfd system call, this
* is an OOM-able child process that the
* parent can restart
*/
static int stress_userfaultfd_oomable(
const args_t *args,
const size_t userfaultfd_bytes)
{
const size_t page_size = args->page_size;
size_t sz;
uint8_t *data;
void *zero_page = NULL;
int fd = -1, fdinfo = -1, status, rc = EXIT_SUCCESS, count = 0;
const unsigned int uffdio_copy = 1 << _UFFDIO_COPY;
const unsigned int uffdio_zeropage = 1 << _UFFDIO_ZEROPAGE;
pid_t pid;
struct uffdio_api api;
struct uffdio_register reg;
context_t c;
bool do_poll = true;
char filename[PATH_MAX];
/* Child clone stack */
static uint8_t stack[STACK_SIZE];
const ssize_t stack_offset =
stress_get_stack_direction() * (STACK_SIZE - 64);
uint8_t *stack_top = stack + stack_offset;
sz = userfaultfd_bytes & ~(page_size - 1);
if (posix_memalign(&zero_page, page_size, page_size)) {
pr_err("%s: zero page allocation failed\n", args->name);
return EXIT_NO_RESOURCE;
}
data = mmap(NULL, sz, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (data == MAP_FAILED) {
rc = EXIT_NO_RESOURCE;
pr_err("%s: mmap failed\n", args->name);
goto free_zeropage;
}
/* Get userfault fd */
if ((fd = shim_userfaultfd(0)) < 0) {
if (errno == ENOSYS) {
pr_inf("%s: stressor will be skipped, "
"userfaultfd not supported\n",
args->name);
rc = EXIT_NOT_IMPLEMENTED;
goto unmap_data;
}
rc = exit_status(errno);
pr_err("%s: userfaultfd failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
goto unmap_data;
}
(void)snprintf(filename, sizeof(filename), "/proc/%d/fdinfo/%d",
getpid(), fd);
fdinfo = open(filename, O_RDONLY);
if (stress_set_nonblock(fd) < 0)
do_poll = false;
/* API sanity check */
(void)memset(&api, 0, sizeof(api));
api.api = UFFD_API;
api.features = 0;
if (ioctl(fd, UFFDIO_API, &api) < 0) {
pr_err("%s: ioctl UFFDIO_API failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
rc = EXIT_FAILURE;
goto unmap_data;
}
if (api.api != UFFD_API) {
pr_err("%s: ioctl UFFDIO_API API check failed\n",
args->name);
rc = EXIT_FAILURE;
goto unmap_data;
}
/* Register fault handling mode */
(void)memset(®, 0, sizeof(reg));
reg.range.start = (unsigned long)data;
reg.range.len = sz;
reg.mode = UFFDIO_REGISTER_MODE_MISSING;
if (ioctl(fd, UFFDIO_REGISTER, ®) < 0) {
pr_err("%s: ioctl UFFDIO_REGISTER failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
rc = EXIT_FAILURE;
goto unmap_data;
}
/* OK, so do we have copy supported? */
if ((reg.ioctls & uffdio_copy) != uffdio_copy) {
pr_err("%s: ioctl UFFDIO_REGISTER did not support _UFFDIO_COPY\n",
args->name);
rc = EXIT_FAILURE;
goto unmap_data;
}
/* OK, so do we have zeropage supported? */
if ((reg.ioctls & uffdio_zeropage) != uffdio_zeropage) {
pr_err("%s: ioctl UFFDIO_REGISTER did not support _UFFDIO_ZEROPAGE\n",
args->name);
rc = EXIT_FAILURE;
goto unmap_data;
}
/* Set up context for child */
c.args = args;
c.data = data;
c.sz = sz;
c.page_size = page_size;
c.parent = getpid();
/*
* We need to clone the child and share the same VM address space
* as parent so we can perform the page fault handling
*/
pid = clone(stress_userfaultfd_child, align_stack(stack_top),
SIGCHLD | CLONE_FILES | CLONE_FS | CLONE_SIGHAND | CLONE_VM, &c);
if (pid < 0) {
pr_err("%s: fork failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
goto unreg;
}
/* Parent */
do {
struct uffd_msg msg;
ssize_t ret;
/* check we should break out before we block on the read */
if (!g_keep_stressing_flag)
break;
/*
* polled wait exercises userfaultfd_poll
* in the kernel, but only works if fd is NONBLOCKing
*/
if (do_poll) {
struct pollfd fds[1];
(void)memset(fds, 0, sizeof fds);
fds[0].fd = fd;
fds[0].events = POLLIN;
/* wait for 1 second max */
ret = poll(fds, 1, 1000);
if (ret == 0)
continue; /* timed out, redo the poll */
if (ret < 0) {
if (errno == EINTR)
continue;
if (errno != ENOMEM) {
pr_fail_err("poll userfaultfd");
if (!g_keep_stressing_flag)
break;
}
/*
* poll ran out of free space for internal
* fd tables, so give up and block on the
* read anyway
*/
goto do_read;
}
/* No data, re-poll */
if (!(fds[0].revents & POLLIN))
continue;
if (LIKELY(fdinfo > -1) &&
UNLIKELY(count++ >= COUNT_MAX)) {
ret = lseek(fdinfo, 0, SEEK_SET);
if (ret == 0) {
char buffer[4096];
ret = read(fdinfo, buffer, sizeof(buffer));
(void)ret;
}
count = 0;
}
}
do_read:
if ((ret = read(fd, &msg, sizeof(msg))) < 0) {
if (errno == EINTR)
continue;
pr_fail_err("read userfaultfd");
if (!g_keep_stressing_flag)
break;
continue;
}
/* We only expect a page fault event */
if (msg.event != UFFD_EVENT_PAGEFAULT) {
pr_fail_err("userfaultfd msg not pagefault event");
continue;
}
/* We only expect a write fault */
if (!(msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)) {
pr_fail_err("userfaultfd msg not write page fault event");
continue;
}
/* Go handle the page fault */
if (handle_page_fault(args, fd, (uint8_t *)(ptrdiff_t)msg.arg.pagefault.address,
zero_page, data, data + sz, page_size) < 0)
break;
inc_counter(args);
} while (keep_stressing());
/* Run it over, zap child */
(void)kill(pid, SIGKILL);
if (shim_waitpid(pid, &status, 0) < 0) {
pr_dbg("%s: waitpid failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
}
unreg:
if (ioctl(fd, UFFDIO_UNREGISTER, ®) < 0) {
pr_err("%s: ioctl UFFDIO_UNREGISTER failed, errno = %d (%s)\n",
args->name, errno, strerror(errno));
rc = EXIT_FAILURE;
goto unmap_data;
}
unmap_data:
(void)munmap(data, sz);
free_zeropage:
free(zero_page);
if (fdinfo > -1)
(void)close(fdinfo);
if (fd > -1)
(void)close(fd);
return rc;
}
static void handle_fault_store(trapframe_t* tf)
{
if (handle_page_fault(tf->badvaddr, PROT_WRITE) != 0)
segfault(tf, tf->badvaddr, "store");
}
static void handle_fault_load(trapframe_t* tf)
{
if (handle_page_fault(tf->badvaddr, PROT_READ) != 0)
segfault(tf, tf->badvaddr, "load");
}
void handle_fault_store(trapframe_t* tf)
{
tf->badvaddr = read_csr(sbadaddr);
if (handle_page_fault(tf->badvaddr, PROT_WRITE) != 0)
segfault(tf, tf->badvaddr, "store");
}
/*
* This routine handles page faults. It determines the address, and the
* problem, and then passes it handle_page_fault() for normal DTLB and
* ITLB issues, and for DMA or SN processor faults when we are in user
* space. For the latter, if we're in kernel mode, we just save the
* interrupt away appropriately and return immediately. We can't do
* page faults for user code while in kernel mode.
*/
void do_page_fault(struct pt_regs *regs, int fault_num,
unsigned long address, unsigned long write)
{
int is_page_fault;
#ifdef CONFIG_KPROBES
/*
* This is to notify the fault handler of the kprobes. The
* exception code is redundant as it is also carried in REGS,
* but we pass it anyhow.
*/
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
regs->faultnum, SIGSEGV) == NOTIFY_STOP)
return;
#endif
#ifdef __tilegx__
/*
* We don't need early do_page_fault_ics() support, since unlike
* Pro we don't need to worry about unlocking the atomic locks.
* There is only one current case in GX where we touch any memory
* under ICS other than our own kernel stack, and we handle that
* here. (If we crash due to trying to touch our own stack,
* we're in too much trouble for C code to help out anyway.)
*/
if (write & ~1) {
unsigned long pc = write & ~1;
if (pc >= (unsigned long) __start_unalign_asm_code &&
pc < (unsigned long) __end_unalign_asm_code) {
struct thread_info *ti = current_thread_info();
/*
* Our EX_CONTEXT is still what it was from the
* initial unalign exception, but now we've faulted
* on the JIT page. We would like to complete the
* page fault however is appropriate, and then retry
* the instruction that caused the unalign exception.
* Our state has been "corrupted" by setting the low
* bit in "sp", and stashing r0..r3 in the
* thread_info area, so we revert all of that, then
* continue as if this were a normal page fault.
*/
regs->sp &= ~1UL;
regs->regs[0] = ti->unalign_jit_tmp[0];
regs->regs[1] = ti->unalign_jit_tmp[1];
regs->regs[2] = ti->unalign_jit_tmp[2];
regs->regs[3] = ti->unalign_jit_tmp[3];
write &= 1;
} else {
pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n",
current->comm, current->pid, pc, address);
show_regs(regs);
do_group_exit(SIGKILL);
return;
}
}
#else
/* This case should have been handled by do_page_fault_ics(). */
BUG_ON(write & ~1);
#endif
#if CHIP_HAS_TILE_DMA()
/*
* If it's a DMA fault, suspend the transfer while we're
* handling the miss; we'll restart after it's handled. If we
* don't suspend, it's possible that this process could swap
* out and back in, and restart the engine since the DMA is
* still 'running'.
*/
if (fault_num == INT_DMATLB_MISS ||
fault_num == INT_DMATLB_ACCESS ||
fault_num == INT_DMATLB_MISS_DWNCL ||
fault_num == INT_DMATLB_ACCESS_DWNCL) {
__insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
while (__insn_mfspr(SPR_DMA_USER_STATUS) &
SPR_DMA_STATUS__BUSY_MASK)
;
}
#endif
/* Validate fault num and decide if this is a first-time page fault. */
switch (fault_num) {
case INT_ITLB_MISS:
case INT_DTLB_MISS:
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_MISS:
case INT_DMATLB_MISS_DWNCL:
#endif
is_page_fault = 1;
break;
case INT_DTLB_ACCESS:
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_ACCESS:
case INT_DMATLB_ACCESS_DWNCL:
#endif
is_page_fault = 0;
break;
default:
panic("Bad fault number %d in do_page_fault", fault_num);
}
#if CHIP_HAS_TILE_DMA()
if (!user_mode(regs)) {
struct async_tlb *async;
switch (fault_num) {
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_MISS:
case INT_DMATLB_ACCESS:
case INT_DMATLB_MISS_DWNCL:
case INT_DMATLB_ACCESS_DWNCL:
async = ¤t->thread.dma_async_tlb;
break;
#endif
default:
async = NULL;
}
if (async) {
/*
* No vmalloc check required, so we can allow
* interrupts immediately at this point.
*/
local_irq_enable();
set_thread_flag(TIF_ASYNC_TLB);
if (async->fault_num != 0) {
panic("Second async fault %d;"
" old fault was %d (%#lx/%ld)",
fault_num, async->fault_num,
address, write);
}
BUG_ON(fault_num == 0);
async->fault_num = fault_num;
async->is_fault = is_page_fault;
async->is_write = write;
async->address = address;
return;
}
}
#endif
handle_page_fault(regs, fault_num, is_page_fault, address, write);
}
/*
* We give a *copy* of the faultinfo in the regs to segv.
* This must be done, since nesting SEGVs could overwrite
* the info in the regs. A pointer to the info then would
* give us bad data!
*/
unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user,
struct uml_pt_regs *regs)
{
struct siginfo si;
jmp_buf *catcher;
int err;
int is_write = FAULT_WRITE(fi);
unsigned long address = FAULT_ADDRESS(fi);
if (!is_user && (address >= start_vm) && (address < end_vm)) {
flush_tlb_kernel_vm();
return 0;
}
else if (current->mm == NULL) {
show_regs(container_of(regs, struct pt_regs, regs));
panic("Segfault with no mm");
}
if (SEGV_IS_FIXABLE(&fi) || SEGV_MAYBE_FIXABLE(&fi))
err = handle_page_fault(address, ip, is_write, is_user,
&si.si_code);
else {
err = -EFAULT;
/*
* A thread accessed NULL, we get a fault, but CR2 is invalid.
* This code is used in __do_copy_from_user() of TT mode.
* XXX tt mode is gone, so maybe this isn't needed any more
*/
address = 0;
}
catcher = current->thread.fault_catcher;
if (!err)
return 0;
else if (catcher != NULL) {
current->thread.fault_addr = (void *) address;
UML_LONGJMP(catcher, 1);
}
else if (current->thread.fault_addr != NULL)
panic("fault_addr set but no fault catcher");
else if (!is_user && arch_fixup(ip, regs))
return 0;
if (!is_user) {
show_regs(container_of(regs, struct pt_regs, regs));
panic("Kernel mode fault at addr 0x%lx, ip 0x%lx",
address, ip);
}
if (err == -EACCES) {
si.si_signo = SIGBUS;
si.si_errno = 0;
si.si_code = BUS_ADRERR;
si.si_addr = (void __user *)address;
current->thread.arch.faultinfo = fi;
force_sig_info(SIGBUS, &si, current);
} else if (err == -ENOMEM) {
printk(KERN_INFO "VM: killing process %s\n", current->comm);
do_exit(SIGKILL);
} else {
BUG_ON(err != -EFAULT);
si.si_signo = SIGSEGV;
si.si_addr = (void __user *) address;
current->thread.arch.faultinfo = fi;
force_sig_info(SIGSEGV, &si, current);
}
return 0;
}
void handle_fault_load(trapframe_t* tf)
{
tf->badvaddr = read_csr(sbadaddr);
if (handle_page_fault(tf->badvaddr, PROT_READ) != 0)
segfault(tf, tf->badvaddr, "load");
}
