int nk_thread_run(nk_thread_id_t t)
{
nk_thread_t * newthread = (nk_thread_t*)t;
printk("Trying to execute thread %p (tid %lu)", newthread,newthread->tid);
printk("RUN: Function: %llu\n", newthread->fun);
printk("RUN: Bound_CPU: %llu\n", newthread->bound_cpu);
thread_setup_init_stack(newthread, newthread->fun, newthread->input);
nk_enqueue_thread_on_runq(newthread, newthread->bound_cpu);
#ifdef NAUT_CONFIG_DEBUG_THREADS
if (newthread->bound_cpu == CPU_ANY) {
SCHED_DEBUG("Running thread (%p, tid=%u) on [ANY CPU]\n", newthread, newthread->tid);
} else {
SCHED_DEBUG("Newthread thread (%p, tid=%u) on cpu %u\n", newthread, newthread->tid, newthread->bound_cpu);
}
#endif
#ifdef NAUT_CONFIG_KICK_SCHEDULE
// kick it
// this really should not fire on CPU_ANY....
if (newthread->bound_cpu != my_cpu_id()) {
apic_ipi(per_cpu_get(apic),
nk_get_nautilus_info()->sys.cpus[newthread->bound_cpu]->lapic_id,
APIC_NULL_KICK_VEC);
}
#endif
return 0;
}
int
null_excp_handler (excp_entry_t * excp,
excp_vec_t vector,
addr_t fault_addr)
{
cpu_id_t cpu_id = cpu_info_ready ? my_cpu_id() : 0xffffffff;
/* TODO: this should be based on scheduler initialization, not CPU */
unsigned tid = cpu_info_ready ? get_cur_thread()->tid : 0xffffffff;
printk("\n+++ UNHANDLED EXCEPTION +++\n");
if (vector < 32) {
printk("[%s] (0x%x) error=0x%x <%s>\n RIP=%p (core=%u, thread=%u)\n",
excp_codes[vector][EXCP_NAME],
vector,
excp->error_code,
excp_codes[vector][EXCP_MNEMONIC],
(void*)excp->rip,
cpu_id, tid);
} else {
printk("[Unknown Exception] (vector=0x%x)\n RIP=(%p) (core=%u)\n",
vector,
(void*)excp->rip,
cpu_id);
}
printk("RDTSC at null exception is %llu\n", rdtsc());
struct nk_regs * r = (struct nk_regs*)((char*)excp - 128);
nk_print_regs(r);
backtrace(r->rbp);
panic("+++ HALTING +++\n");
return 0;
}
static int
error_int_handler (excp_entry_t * excp, excp_vec_t v)
{
struct apic_dev * apic = per_cpu_get(apic);
char * s = "[Unknown Error]";
uint8_t i = 0;
uint32_t err = 0;
apic_write(apic, APIC_REG_ESR, 0);
err = apic_read(apic, APIC_REG_ESR);
apic_do_eoi();
apic->err_int_cnt++;
err &= 0xff;
APIC_WARN("Error interrupt recieved from local APIC (ID=0x%x) on Core %u (error=0x%x):\n",
per_cpu_get(apic)->id, my_cpu_id(), err);
while (err) {
if (err & 0x1) {
s = (char*)apic_err_codes[i];
APIC_WARN("\t%s\n", s);
}
++i;
err >>= 1;
}
return 0;
}
static int
thermal_int_handler (excp_entry_t * excp, excp_vec_t v)
{
panic("Received a thermal interrupt from the LAPIC (0x%x) on core %u (Should be masked)\n",
per_cpu_get(apic)->id,
my_cpu_id());
return 0;
}
static int
dummy_int_handler (excp_entry_t * excp, excp_vec_t v)
{
panic("Received an interrupt from an Extended LVT vector on LAPIC (0x%x) on core %u (Should be masked)\n",
per_cpu_get(apic)->id,
my_cpu_id());
return 0;
}
static int
spur_int_handler (excp_entry_t * excp, excp_vec_t v)
{
APIC_WARN("APIC (ID=0x%x) Received Spurious Interrupt on core %u\n",
per_cpu_get(apic)->id,
my_cpu_id());
struct apic_dev * a = per_cpu_get(apic);
a->spur_int_cnt++;
/* we don't need to EOI here */
return 0;
}
int
null_irq_handler (excp_entry_t * excp,
excp_vec_t vector)
{
printk("[Unhandled IRQ] (vector=0x%x)\n RIP=(%p) (core=%u)\n",
vector,
(void*)excp->rip,
my_cpu_id());
struct nk_regs * r = (struct nk_regs*)((char*)excp - 128);
nk_print_regs(r);
backtrace(r->rbp);
panic("+++ HALTING +++\n");
return 0;
}
inline nk_thread_t*
nk_dequeue_thread_from_runq (nk_thread_t * t)
{
nk_thread_queue_t * q = t->cur_run_q;
nk_queue_entry_t * elm = NULL;
nk_thread_t * ret = NULL;
/* bail if the run queue doesn't exist */
if (!q) {
ERROR_PRINT("Attempt to dequeue thread not on run queue (cpu=%u)\n", my_cpu_id());
return NULL;
}
elm = nk_dequeue_entry_atomic(q, &(t->runq_node));
ret = container_of(elm, nk_thread_t, runq_node);
t->status = NK_THR_SUSPENDED;
t->cur_run_q = NULL;
return ret;
}
static void
apic_dump (struct apic_dev * apic)
{
char buf[128];
APIC_DEBUG("DUMP (LOGICAL CPU #%u):\n", my_cpu_id());
APIC_DEBUG(
" ID: 0x%08x (id=%d)\n",
apic_read(apic, APIC_REG_ID),
APIC_GET_ID(apic_read(apic, APIC_REG_ID))
);
APIC_DEBUG(
" VER: 0x%08x (version=0x%x, max_lvt=%d)\n",
apic_read(apic, APIC_REG_LVR),
APIC_LVR_VER(apic_read(apic, APIC_REG_LVR)),
APIC_LVR_MAX(apic_read(apic, APIC_REG_LVR))
);
APIC_DEBUG(
" BASE ADDR: %p\n",
apic->base_addr
);
if (nk_is_amd() && amd_has_ext_lvt(apic)) {
APIC_DEBUG(
" EXT (AMD-only): 0x%08x (Ext LVT Count=%u, Ext APIC ID=%u, Specific EOI=%u, Int Enable Reg=%u)\n",
apic_read(apic, APIC_REG_EXFR),
APIC_EXFR_GET_LVT(apic_read(apic, APIC_REG_EXFR)),
APIC_EXFR_GET_XAIDC(apic_read(apic, APIC_REG_EXFR)),
APIC_EXFR_GET_SNIC(apic_read(apic, APIC_REG_EXFR)),
APIC_EXFR_GET_INC(apic_read(apic, APIC_REG_EXFR))
);
int i;
for (i = 0; i < APIC_EXFR_GET_LVT(apic_read(apic, APIC_REG_EXFR)); i++) {
APIC_DEBUG(
" EXT-LVT[%u]: 0x%08x (%s)\n",
i,
apic_read(apic, APIC_REG_EXTLVT(i)),
lvt_stringify(apic_read(apic, APIC_REG_EXTLVT(i)), buf)
);
}
}
APIC_DEBUG(
" ESR: 0x%08x (Error Status Reg, non-zero is bad)\n",
apic_read(apic, APIC_REG_ESR)
);
APIC_DEBUG(
" SVR: 0x%08x (Spurious vector=%d, %s, %s)\n",
apic_read(apic, APIC_REG_SPIV),
apic_read(apic, APIC_REG_SPIV) & APIC_SPIV_VEC_MASK,
(apic_read(apic, APIC_REG_SPIV) & APIC_SPIV_SW_ENABLE)
? "APIC IS ENABLED"
: "APIC IS DISABLED",
(apic_read(apic, APIC_REG_SPIV) & APIC_SPIV_CORE_FOCUS)
? "Core Focusing Disabled"
: "Core Focusing Enabled"
);
/*
* Local Vector Table
*/
APIC_DEBUG(" Local Vector Table Entries:\n");
char * timer_mode;
if (apic_read(apic, APIC_REG_LVTT) & APIC_TIMER_PERIODIC) {
timer_mode = "Periodic";
} else if (apic_read(apic, APIC_REG_LVTT) & APIC_TIMER_TSCDLINE) {
timer_mode = "TSC-Deadline";
} else {
timer_mode = "One-shot";
}
APIC_DEBUG(" LVT[0] Timer: 0x%08x (mode=%s, %s)\n",
apic_read(apic, APIC_REG_LVTT),
timer_mode,
lvt_stringify(apic_read(apic, APIC_REG_LVTT), buf)
);
APIC_DEBUG(" LVT[1] Thermal: 0x%08x (%s)\n",
apic_read(apic, APIC_REG_LVTTHMR),
lvt_stringify(apic_read(apic, APIC_REG_LVTTHMR), buf)
);
APIC_DEBUG(" LVT[2] Perf Cnt: 0x%08x (%s)\n",
apic_read(apic, APIC_REG_LVTPC),
lvt_stringify(apic_read(apic, APIC_REG_LVTPC), buf)
);
APIC_DEBUG(" LVT[3] LINT0 Pin: 0x%08x (%s)\n",
apic_read(apic, APIC_REG_LVT0),
lvt_stringify(apic_read(apic, APIC_REG_LVT0), buf)
);
APIC_DEBUG(" LVT[4] LINT1 Pin: 0x%08x (%s)\n",
apic_read(apic, APIC_REG_LVT1),
lvt_stringify(apic_read(apic, APIC_REG_LVT1), buf)
);
APIC_DEBUG(" LVT[5] Error: 0x%08x (%s)\n",
apic_read(apic, APIC_REG_LVTERR),
lvt_stringify(apic_read(apic, APIC_REG_LVTERR), buf)
);
/*
* APIC timer configuration registers
*/
APIC_DEBUG(" Local APIC Timer:\n");
APIC_DEBUG(" DCR (Divide Config Reg): 0x%08x\n",
apic_read(apic, APIC_REG_TMDCR)
);
APIC_DEBUG(" ICT (Initial Count Reg): 0x%08x\n",
apic_read(apic, APIC_REG_TMICT)
);
APIC_DEBUG(" CCT (Current Count Reg): 0x%08x\n",
apic_read(apic, APIC_REG_TMCCT)
);
/*
* Logical APIC addressing mode registers
*/
APIC_DEBUG(" Logical Addressing Mode Information:\n");
APIC_DEBUG(" LDR (Logical Dest Reg): 0x%08x (id=%d)\n",
apic_read(apic, APIC_REG_LDR),
GET_APIC_LOGICAL_ID(apic_read(apic, APIC_REG_LDR))
);
APIC_DEBUG(" DFR (Dest Format Reg): 0x%08x (%s)\n",
apic_read(apic, APIC_REG_DFR),
(apic_read(apic, APIC_REG_DFR) == APIC_DFR_FLAT) ? "FLAT" : "CLUSTER"
);
/*
* Task/processor/arbitration priority registers
*/
APIC_DEBUG(" Task/Processor/Arbitration Priorities:\n");
APIC_DEBUG(" TPR (Task Priority Reg): 0x%08x\n",
apic_read(apic, APIC_REG_TPR)
);
APIC_DEBUG(" PPR (Processor Priority Reg): 0x%08x\n",
apic_read(apic, APIC_REG_PPR)
);
APIC_DEBUG(" APR (Arbitration Priority Reg): 0x%08x\n",
apic_read(apic, APIC_REG_APR)
);
/*
* ISR/IRR
*/
APIC_DEBUG(" IRR/ISR:\n");
APIC_DEBUG(" IRR (Interrupt Request Reg): 0x%08x\n",
apic_read(apic, APIC_GET_IRR(0)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(1)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(2)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(3)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(4)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(5)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(6)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_IRR(7)));
APIC_DEBUG(" ISR (In-Service Reg): 0x%08x\n",
apic_read(apic, APIC_GET_ISR(0)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(1)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(2)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(3)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(4)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(5)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(6)));
APIC_DEBUG(" 0x%08x\n",
apic_read(apic, APIC_GET_ISR(7)));
}
int
nk_thread_start (nk_thread_fun_t fun,
void *input,
void **output,
uint8_t is_detached,
nk_stack_size_t stack_size,
nk_thread_id_t *tid,
int cpu,
int rt_type,
rt_constraints *rt_constraints,
uint64_t rt_deadline)
#endif
{
nk_thread_id_t newtid = NULL;
nk_thread_t * newthread = NULL;
/* put it on the current CPU */
if (cpu == CPU_ANY) {
cpu = my_cpu_id();
}
if (nk_thread_create(fun, input, output, is_detached, stack_size, &newtid, cpu) < 0) {
ERROR_PRINT("Could not create thread\n");
return -1;
}
newthread = (nk_thread_t*)newtid;
if (tid) {
*tid = newtid;
}
thread_setup_init_stack(newthread, fun, input);
#ifdef NAUT_CONFIG_USE_RT_SCHEDULER
rt_thread *rt = rt_thread_init(rt_type, rt_constraints, rt_deadline, newthread);
RT_THREAD_DEBUG("rt_deadline is %llu\n", rt->deadline);
struct sys_info *sys = per_cpu_get(system);
if (sys->cpus[cpu]->rt_sched)
{
if (rt_admit(sys->cpus[cpu]->rt_sched, rt))
{
if (rt_type == PERIODIC || rt_type == SPORADIC)
{
enqueue_thread(sys->cpus[cpu]->rt_sched->runnable, rt);
RT_THREAD_DEBUG("THREAD DEADLINE ON RUN QUEUE IS: %llu\n", sys->cpus[cpu]->rt_sched->runnable->threads[0]->deadline);
}
else
{
enqueue_thread(sys->cpus[cpu]->rt_sched->aperiodic, rt);
}
} else
{
RT_THREAD_DEBUG("FAILED TO START THREAD. ADMISSION CONTROL DENYING ENTRY.\n");
}
}
nk_schedule();
#else
nk_enqueue_thread_on_runq(newthread, cpu);
#endif
#ifdef NAUT_CONFIG_DEBUG_THREADS
if (cpu == CPU_ANY) {
SCHED_DEBUG("Started thread (%p, tid=%u) on [ANY CPU]\n", newthread, newthread->tid);
} else {
SCHED_DEBUG("Started thread (%p, tid=%u) on cpu %u\n", newthread, newthread->tid, cpu);
}
#endif
#ifdef NAUT_CONFIG_KICK_SCHEDULE
// kick it
if (cpu != my_cpu_id()) {
apic_ipi(per_cpu_get(apic),
nk_get_nautilus_info()->sys.cpus[cpu]->lapic_id,
APIC_NULL_KICK_VEC);
}
#endif
return 0;
}
int
nk_thread_create (nk_thread_fun_t fun,
void * input,
void ** output,
uint8_t is_detached,
nk_stack_size_t stack_size,
nk_thread_id_t * tid,
int cpu)
{
nk_thread_t * t = NULL;
void * stack = NULL;
if (cpu == CPU_ANY) {
cpu = my_cpu_id();
}
#ifndef NAUT_CONFIG_THREAD_OPTIMIZE
ASSERT(cpu < per_cpu_get(system)->num_cpus);
if (cpu >= per_cpu_get(system)->num_cpus) {
ERROR_PRINT("thread create received invalid CPU id (%u)\n", cpu);
return -EINVAL;
}
#endif
t = malloc(sizeof(nk_thread_t));
#ifndef NAUT_CONFIG_THREAD_OPTIMIZE
ASSERT(t);
if (!t) {
ERROR_PRINT("Could not allocate thread struct\n");
return -EINVAL;
}
memset(t, 0, sizeof(nk_thread_t));
#endif
#ifndef NAUT_CONFIG_THREAD_OPTIMIZE
if (stack_size) {
stack = (void*)malloc(stack_size);
t->stack_size = stack_size;
} else {
stack = (void*)malloc(PAGE_SIZE);
t->stack_size = PAGE_SIZE;
}
#else
stack = malloc(PAGE_SIZE_4KB);
t->stack_size = PAGE_SIZE_4KB;
#endif
ASSERT(stack);
if (thread_init(t, stack, is_detached, cpu, get_cur_thread()) < 0) {
ERROR_PRINT("Could not initialize thread\n");
goto out_err1;
}
t->status = NK_THR_INIT;
t->fun = fun;
t->input = input;
t->output = output;
enqueue_thread_on_tlist(t);
if (tid) {
*tid = (nk_thread_id_t)t;
}
SCHED_DEBUG("Thread create creating new thread with t=%p, tid=%lu\n", t, t->tid);
return 0;
out_err1:
free(stack);
free(t);
return -1;
}
static void
thread_cleanup (void)
{
SCHED_DEBUG("Thread (%d) exiting on core %d\n", get_cur_thread()->tid, my_cpu_id());
nk_thread_exit(0);
}
static inline nk_thread_t * __always_inline
get_runnable_thread_myq (void)
{
cpu_id_t id = my_cpu_id();
return get_runnable_thread(id);
}
