PRIVATE void estimate_cpu_freq(void)
{
u64_t tsc_delta;
u64_t cpu_freq;
irq_hook_t calib_cpu;
/* set the probe, we use the legacy timer, IRQ 0 */
put_irq_handler(&calib_cpu, CLOCK_IRQ, calib_cpu_handler);
/* just in case we are in an SMP single cpu fallback mode */
BKL_UNLOCK();
/* set the PIC timer to get some time */
intr_enable();
/* loop for some time to get a sample */
while(probe_ticks < PROBE_TICKS) {
intr_enable();
}
intr_disable();
/* just in case we are in an SMP single cpu fallback mode */
BKL_LOCK();
/* remove the probe */
rm_irq_handler(&calib_cpu);
tsc_delta = sub64(tsc1, tsc0);
cpu_freq = mul64(div64u64(tsc_delta, PROBE_TICKS - 1), make64(system_hz, 0));
cpu_set_freq(cpuid, cpu_freq);
cpu_info[cpuid].freq = div64u(cpu_freq, 1000000);
BOOT_VERBOSE(cpu_print_freq(cpuid));
}
/*
* tell another cpu about a task to do and return only after the cpu acks that
* the task is finished. Also wait before it finishes task sent by another cpu
* to the same one.
*/
static void smp_schedule_sync(struct proc * p, unsigned task)
{
unsigned cpu = p->p_cpu;
unsigned mycpu = cpuid;
assert(cpu != mycpu);
/*
* if some other cpu made a request to the same cpu, wait until it is
* done before proceeding
*/
if (sched_ipi_data[cpu].flags != 0) {
BKL_UNLOCK();
while (sched_ipi_data[cpu].flags != 0) {
if (sched_ipi_data[mycpu].flags) {
BKL_LOCK();
smp_sched_handler();
BKL_UNLOCK();
}
}
BKL_LOCK();
}
sched_ipi_data[cpu].data = (u32_t) p;
sched_ipi_data[cpu].flags |= task;
__insn_barrier();
arch_send_smp_schedule_ipi(cpu);
/* wait until the destination cpu finishes its job */
BKL_UNLOCK();
while (sched_ipi_data[cpu].flags != 0) {
if (sched_ipi_data[mycpu].flags) {
BKL_LOCK();
smp_sched_handler();
BKL_UNLOCK();
}
}
BKL_LOCK();
}
void wait_for_APs_to_finish_booting(void)
{
unsigned n = 0;
int i;
/* check how many cpus are actually alive */
for (i = 0 ; i < ncpus ; i++) {
if (cpu_test_flag(i, CPU_IS_READY))
n++;
}
if (n != ncpus)
printf("WARNING only %d out of %d cpus booted\n", n, ncpus);
/* we must let the other CPUs to run in kernel mode first */
BKL_UNLOCK();
while (ap_cpus_booted != (n - 1))
arch_pause();
/* now we have to take the lock again as we continue execution */
BKL_LOCK();
}
PUBLIC void context_stop(struct proc * p)
{
u64_t tsc, tsc_delta;
u64_t * __tsc_ctr_switch = get_cpulocal_var_ptr(tsc_ctr_switch);
#ifdef CONFIG_SMP
unsigned cpu = cpuid;
/*
* This function is called only if we switch from kernel to user or idle
* or back. Therefore this is a perfect location to place the big kernel
* lock which will hopefully disappear soon.
*
* If we stop accounting for KERNEL we must unlock the BKL. If account
* for IDLE we must not hold the lock
*/
if (p == proc_addr(KERNEL)) {
u64_t tmp;
read_tsc_64(&tsc);
tmp = sub64(tsc, *__tsc_ctr_switch);
kernel_ticks[cpu] = add64(kernel_ticks[cpu], tmp);
p->p_cycles = add64(p->p_cycles, tmp);
BKL_UNLOCK();
} else {
u64_t bkl_tsc;
atomic_t succ;
read_tsc_64(&bkl_tsc);
/* this only gives a good estimate */
succ = big_kernel_lock.val;
BKL_LOCK();
read_tsc_64(&tsc);
bkl_ticks[cpu] = add64(bkl_ticks[cpu], sub64(tsc, bkl_tsc));
bkl_tries[cpu]++;
bkl_succ[cpu] += !(!(succ == 0));
p->p_cycles = add64(p->p_cycles, sub64(tsc, *__tsc_ctr_switch));
#ifdef CONFIG_SMP
/*
* Since at the time we got a scheduling IPI we might have been
* waiting for BKL already, we may miss it due to a similar IPI to
* the cpu which is already waiting for us to handle its. This
* results in a live-lock of these two cpus.
*
* Therefore we always check if there is one pending and if so,
* we handle it straight away so the other cpu can continue and
* we do not deadlock.
*/
smp_sched_handler();
#endif
}
#else
read_tsc_64(&tsc);
p->p_cycles = add64(p->p_cycles, sub64(tsc, *__tsc_ctr_switch));
#endif
tsc_delta = sub64(tsc, *__tsc_ctr_switch);
if(kbill_ipc) {
kbill_ipc->p_kipc_cycles =
add64(kbill_ipc->p_kipc_cycles, tsc_delta);
kbill_ipc = NULL;
}
if(kbill_kcall) {
kbill_kcall->p_kcall_cycles =
add64(kbill_kcall->p_kcall_cycles, tsc_delta);
kbill_kcall = NULL;
}
/*
* deduct the just consumed cpu cycles from the cpu time left for this
* process during its current quantum. Skip IDLE and other pseudo kernel
* tasks
*/
if (p->p_endpoint >= 0) {
#if DEBUG_RACE
make_zero64(p->p_cpu_time_left);
#else
/* if (tsc_delta < p->p_cpu_time_left) in 64bit */
if (ex64hi(tsc_delta) < ex64hi(p->p_cpu_time_left) ||
(ex64hi(tsc_delta) == ex64hi(p->p_cpu_time_left) &&
ex64lo(tsc_delta) < ex64lo(p->p_cpu_time_left)))
p->p_cpu_time_left = sub64(p->p_cpu_time_left, tsc_delta);
else {
make_zero64(p->p_cpu_time_left);
}
#endif
}
*__tsc_ctr_switch = tsc;
}
/*===========================================================================*
* kmain *
*===========================================================================*/
void kmain(kinfo_t *local_cbi)
{
/* Start the ball rolling. */
struct boot_image *ip; /* boot image pointer */
register struct proc *rp; /* process pointer */
register int i, j;
/* save a global copy of the boot parameters */
memcpy(&kinfo, local_cbi, sizeof(kinfo));
memcpy(&kmess, kinfo.kmess, sizeof(kmess));
#ifdef __arm__
/* We want to initialize serial before we do any output */
omap3_ser_init();
#endif
/* We can talk now */
printf("MINIX booting\n");
/* Kernel may use bits of main memory before VM is started */
kernel_may_alloc = 1;
assert(sizeof(kinfo.boot_procs) == sizeof(image));
memcpy(kinfo.boot_procs, image, sizeof(kinfo.boot_procs));
cstart();
BKL_LOCK();
DEBUGEXTRA(("main()\n"));
proc_init();
if(NR_BOOT_MODULES != kinfo.mbi.mods_count)
panic("expecting %d boot processes/modules, found %d",
NR_BOOT_MODULES, kinfo.mbi.mods_count);
/* Set up proc table entries for processes in boot image. */
for (i=0; i < NR_BOOT_PROCS; ++i) {
int schedulable_proc;
proc_nr_t proc_nr;
int ipc_to_m, kcalls;
sys_map_t map;
ip = &image[i]; /* process' attributes */
DEBUGEXTRA(("initializing %s... ", ip->proc_name));
rp = proc_addr(ip->proc_nr); /* get process pointer */
ip->endpoint = rp->p_endpoint; /* ipc endpoint */
make_zero64(rp->p_cpu_time_left);
if(i < NR_TASKS) /* name (tasks only) */
strlcpy(rp->p_name, ip->proc_name, sizeof(rp->p_name));
if(i >= NR_TASKS) {
/* Remember this so it can be passed to VM */
multiboot_module_t *mb_mod = &kinfo.module_list[i - NR_TASKS];
ip->start_addr = mb_mod->mod_start;
ip->len = mb_mod->mod_end - mb_mod->mod_start;
}
reset_proc_accounting(rp);
/* See if this process is immediately schedulable.
* In that case, set its privileges now and allow it to run.
* Only kernel tasks and the root system process get to run immediately.
* All the other system processes are inhibited from running by the
* RTS_NO_PRIV flag. They can only be scheduled once the root system
* process has set their privileges.
*/
proc_nr = proc_nr(rp);
schedulable_proc = (iskerneln(proc_nr) || isrootsysn(proc_nr) ||
proc_nr == VM_PROC_NR);
if(schedulable_proc) {
/* Assign privilege structure. Force a static privilege id. */
(void) get_priv(rp, static_priv_id(proc_nr));
/* Priviliges for kernel tasks. */
if(proc_nr == VM_PROC_NR) {
priv(rp)->s_flags = VM_F;
priv(rp)->s_trap_mask = SRV_T;
ipc_to_m = SRV_M;
kcalls = SRV_KC;
priv(rp)->s_sig_mgr = SELF;
rp->p_priority = SRV_Q;
rp->p_quantum_size_ms = SRV_QT;
}
else if(iskerneln(proc_nr)) {
/* Privilege flags. */
priv(rp)->s_flags = (proc_nr == IDLE ? IDL_F : TSK_F);
/* Allowed traps. */
priv(rp)->s_trap_mask = (proc_nr == CLOCK
|| proc_nr == SYSTEM ? CSK_T : TSK_T);
ipc_to_m = TSK_M; /* allowed targets */
kcalls = TSK_KC; /* allowed kernel calls */
}
/* Priviliges for the root system process. */
else {
assert(isrootsysn(proc_nr));
priv(rp)->s_flags= RSYS_F; /* privilege flags */
priv(rp)->s_trap_mask= SRV_T; /* allowed traps */
ipc_to_m = SRV_M; /* allowed targets */
kcalls = SRV_KC; /* allowed kernel calls */
priv(rp)->s_sig_mgr = SRV_SM; /* signal manager */
rp->p_priority = SRV_Q; /* priority queue */
rp->p_quantum_size_ms = SRV_QT; /* quantum size */
}
/* Fill in target mask. */
memset(&map, 0, sizeof(map));
if (ipc_to_m == ALL_M) {
for(j = 0; j < NR_SYS_PROCS; j++)
set_sys_bit(map, j);
}
fill_sendto_mask(rp, &map);
/* Fill in kernel call mask. */
for(j = 0; j < SYS_CALL_MASK_SIZE; j++) {
priv(rp)->s_k_call_mask[j] = (kcalls == NO_C ? 0 : (~0));
}
}
else {
/* Don't let the process run for now. */
RTS_SET(rp, RTS_NO_PRIV | RTS_NO_QUANTUM);
}
/* Arch-specific state initialization. */
arch_boot_proc(ip, rp);
/* scheduling functions depend on proc_ptr pointing somewhere. */
if(!get_cpulocal_var(proc_ptr))
get_cpulocal_var(proc_ptr) = rp;
/* Process isn't scheduled until VM has set up a pagetable for it. */
if(rp->p_nr != VM_PROC_NR && rp->p_nr >= 0) {
rp->p_rts_flags |= RTS_VMINHIBIT;
rp->p_rts_flags |= RTS_BOOTINHIBIT;
}
rp->p_rts_flags |= RTS_PROC_STOP;
rp->p_rts_flags &= ~RTS_SLOT_FREE;
DEBUGEXTRA(("done\n"));
}
/* update boot procs info for VM */
memcpy(kinfo.boot_procs, image, sizeof(kinfo.boot_procs));
#define IPCNAME(n) { \
assert((n) >= 0 && (n) <= IPCNO_HIGHEST); \
assert(!ipc_call_names[n]); \
ipc_call_names[n] = #n; \
}
arch_post_init();
IPCNAME(SEND);
IPCNAME(RECEIVE);
IPCNAME(SENDREC);
IPCNAME(NOTIFY);
IPCNAME(SENDNB);
IPCNAME(SENDA);
/* System and processes initialization */
memory_init();
DEBUGEXTRA(("system_init()... "));
system_init();
DEBUGEXTRA(("done\n"));
/* The bootstrap phase is over, so we can add the physical
* memory used for it to the free list.
*/
add_memmap(&kinfo, kinfo.bootstrap_start, kinfo.bootstrap_len);
#ifdef CONFIG_SMP
if (config_no_apic) {
BOOT_VERBOSE(printf("APIC disabled, disables SMP, using legacy PIC\n"));
smp_single_cpu_fallback();
} else if (config_no_smp) {
BOOT_VERBOSE(printf("SMP disabled, using legacy PIC\n"));
smp_single_cpu_fallback();
} else {
smp_init();
/*
* if smp_init() returns it means that it failed and we try to finish
* single CPU booting
*/
bsp_finish_booting();
}
#else
/*
* if configured for a single CPU, we are already on the kernel stack which we
* are going to use everytime we execute kernel code. We finish booting and we
* never return here
*/
bsp_finish_booting();
#endif
NOT_REACHABLE;
}
/*===========================================================================*
* main *
*===========================================================================*/
PUBLIC int main(void)
{
/* Start the ball rolling. */
struct boot_image *ip; /* boot image pointer */
register struct proc *rp; /* process pointer */
register int i, j;
size_t argsz; /* size of arguments passed to crtso on stack */
BKL_LOCK();
/* Global value to test segment sanity. */
magictest = MAGICTEST;
DEBUGEXTRA(("main()\n"));
proc_init();
/* Set up proc table entries for processes in boot image. The stacks
* of the servers have been added to the data segment by the monitor, so
* the stack pointer is set to the end of the data segment.
*/
for (i=0; i < NR_BOOT_PROCS; ++i) {
int schedulable_proc;
proc_nr_t proc_nr;
int ipc_to_m, kcalls;
sys_map_t map;
ip = &image[i]; /* process' attributes */
DEBUGEXTRA(("initializing %s... ", ip->proc_name));
rp = proc_addr(ip->proc_nr); /* get process pointer */
ip->endpoint = rp->p_endpoint; /* ipc endpoint */
make_zero64(rp->p_cpu_time_left);
strncpy(rp->p_name, ip->proc_name, P_NAME_LEN); /* set process name */
reset_proc_accounting(rp);
/* See if this process is immediately schedulable.
* In that case, set its privileges now and allow it to run.
* Only kernel tasks and the root system process get to run immediately.
* All the other system processes are inhibited from running by the
* RTS_NO_PRIV flag. They can only be scheduled once the root system
* process has set their privileges.
*/
proc_nr = proc_nr(rp);
schedulable_proc = (iskerneln(proc_nr) || isrootsysn(proc_nr));
if(schedulable_proc) {
/* Assign privilege structure. Force a static privilege id. */
(void) get_priv(rp, static_priv_id(proc_nr));
/* Priviliges for kernel tasks. */
if(iskerneln(proc_nr)) {
/* Privilege flags. */
priv(rp)->s_flags = (proc_nr == IDLE ? IDL_F : TSK_F);
/* Allowed traps. */
priv(rp)->s_trap_mask = (proc_nr == CLOCK
|| proc_nr == SYSTEM ? CSK_T : TSK_T);
ipc_to_m = TSK_M; /* allowed targets */
kcalls = TSK_KC; /* allowed kernel calls */
}
/* Priviliges for the root system process. */
else if(isrootsysn(proc_nr)) {
priv(rp)->s_flags= RSYS_F; /* privilege flags */
priv(rp)->s_trap_mask= SRV_T; /* allowed traps */
ipc_to_m = SRV_M; /* allowed targets */
kcalls = SRV_KC; /* allowed kernel calls */
priv(rp)->s_sig_mgr = SRV_SM; /* signal manager */
rp->p_priority = SRV_Q; /* priority queue */
rp->p_quantum_size_ms = SRV_QT; /* quantum size */
}
/* Priviliges for ordinary process. */
else {
NOT_REACHABLE;
}
/* Fill in target mask. */
memset(&map, 0, sizeof(map));
if (ipc_to_m == ALL_M) {
for(j = 0; j < NR_SYS_PROCS; j++)
set_sys_bit(map, j);
}
fill_sendto_mask(rp, &map);
/* Fill in kernel call mask. */
for(j = 0; j < SYS_CALL_MASK_SIZE; j++) {
priv(rp)->s_k_call_mask[j] = (kcalls == NO_C ? 0 : (~0));
}
}
else {
/* Don't let the process run for now. */
RTS_SET(rp, RTS_NO_PRIV | RTS_NO_QUANTUM);
}
rp->p_memmap[T].mem_vir = ABS2CLICK(ip->memmap.text_vaddr);
rp->p_memmap[T].mem_phys = ABS2CLICK(ip->memmap.text_paddr);
rp->p_memmap[T].mem_len = ABS2CLICK(ip->memmap.text_bytes);
rp->p_memmap[D].mem_vir = ABS2CLICK(ip->memmap.data_vaddr);
rp->p_memmap[D].mem_phys = ABS2CLICK(ip->memmap.data_paddr);
rp->p_memmap[D].mem_len = ABS2CLICK(ip->memmap.data_bytes);
rp->p_memmap[S].mem_phys = ABS2CLICK(ip->memmap.data_paddr +
ip->memmap.data_bytes +
ip->memmap.stack_bytes);
rp->p_memmap[S].mem_vir = ABS2CLICK(ip->memmap.data_vaddr +
ip->memmap.data_bytes +
ip->memmap.stack_bytes);
rp->p_memmap[S].mem_len = 0;
/* Set initial register values. The processor status word for tasks
* is different from that of other processes because tasks can
* access I/O; this is not allowed to less-privileged processes
*/
rp->p_reg.pc = ip->memmap.entry;
rp->p_reg.psw = (iskerneln(proc_nr)) ? INIT_TASK_PSW : INIT_PSW;
/* Initialize the server stack pointer. Take it down three words
* to give crtso.s something to use as "argc", "argv" and "envp".
*/
if (isusern(proc_nr)) { /* user-space process? */
rp->p_reg.sp = (rp->p_memmap[S].mem_vir +
rp->p_memmap[S].mem_len) << CLICK_SHIFT;
argsz = 3 * sizeof(reg_t);
rp->p_reg.sp -= argsz;
phys_memset(rp->p_reg.sp -
(rp->p_memmap[S].mem_vir << CLICK_SHIFT) +
(rp->p_memmap[S].mem_phys << CLICK_SHIFT),
0, argsz);
}
/* scheduling functions depend on proc_ptr pointing somewhere. */
if(!get_cpulocal_var(proc_ptr))
get_cpulocal_var(proc_ptr) = rp;
/* If this process has its own page table, VM will set the
* PT up and manage it. VM will signal the kernel when it has
* done this; until then, don't let it run.
*/
if(ip->flags & PROC_FULLVM)
rp->p_rts_flags |= RTS_VMINHIBIT;
rp->p_rts_flags |= RTS_PROC_STOP;
rp->p_rts_flags &= ~RTS_SLOT_FREE;
alloc_segments(rp);
DEBUGEXTRA(("done\n"));
}
#define IPCNAME(n) { \
assert((n) >= 0 && (n) <= IPCNO_HIGHEST); \
assert(!ipc_call_names[n]); \
ipc_call_names[n] = #n; \
}
IPCNAME(SEND);
IPCNAME(RECEIVE);
IPCNAME(SENDREC);
IPCNAME(NOTIFY);
IPCNAME(SENDNB);
IPCNAME(SENDA);
/* Architecture-dependent initialization. */
DEBUGEXTRA(("arch_init()... "));
arch_init();
DEBUGEXTRA(("done\n"));
/* System and processes initialization */
DEBUGEXTRA(("system_init()... "));
system_init();
DEBUGEXTRA(("done\n"));
#ifdef CONFIG_SMP
if (config_no_apic) {
BOOT_VERBOSE(printf("APIC disabled, disables SMP, using legacy PIC\n"));
smp_single_cpu_fallback();
} else if (config_no_smp) {
BOOT_VERBOSE(printf("SMP disabled, using legacy PIC\n"));
smp_single_cpu_fallback();
} else {
smp_init();
/*
* if smp_init() returns it means that it failed and we try to finish
* single CPU booting
*/
bsp_finish_booting();
}
#else
/*
* if configured for a single CPU, we are already on the kernel stack which we
* are going to use everytime we execute kernel code. We finish booting and we
* never return here
*/
bsp_finish_booting();
#endif
NOT_REACHABLE;
return 1;
}
PUBLIC void context_stop(struct proc * p)
{
u64_t tsc, tsc_delta;
u64_t * __tsc_ctr_switch = get_cpulocal_var_ptr(tsc_ctr_switch);
#ifdef CONFIG_SMP
unsigned cpu = cpuid;
/*
* This function is called only if we switch from kernel to user or idle
* or back. Therefore this is a perfect location to place the big kernel
* lock which will hopefully disappear soon.
*
* If we stop accounting for KERNEL we must unlock the BKL. If account
* for IDLE we must not hold the lock
*/
if (p == proc_addr(KERNEL)) {
u64_t tmp;
read_tsc_64(&tsc);
tmp = sub64(tsc, *__tsc_ctr_switch);
kernel_ticks[cpu] = add64(kernel_ticks[cpu], tmp);
p->p_cycles = add64(p->p_cycles, tmp);
BKL_UNLOCK();
} else {
u64_t bkl_tsc;
atomic_t succ;
read_tsc_64(&bkl_tsc);
/* this only gives a good estimate */
succ = big_kernel_lock.val;
BKL_LOCK();
read_tsc_64(&tsc);
bkl_ticks[cpu] = add64(bkl_ticks[cpu], sub64(tsc, bkl_tsc));
bkl_tries[cpu]++;
bkl_succ[cpu] += !(!(succ == 0));
p->p_cycles = add64(p->p_cycles, sub64(tsc, *__tsc_ctr_switch));
}
#else
read_tsc_64(&tsc);
p->p_cycles = add64(p->p_cycles, sub64(tsc, *__tsc_ctr_switch));
#endif
tsc_delta = sub64(tsc, *__tsc_ctr_switch);
if(kbill_ipc) {
kbill_ipc->p_kipc_cycles =
add64(kbill_ipc->p_kipc_cycles, tsc_delta);
kbill_ipc = NULL;
}
if(kbill_kcall) {
kbill_kcall->p_kcall_cycles =
add64(kbill_kcall->p_kcall_cycles, tsc_delta);
kbill_kcall = NULL;
}
/*
* deduct the just consumed cpu cycles from the cpu time left for this
* process during its current quantum. Skip IDLE and other pseudo kernel
* tasks
*/
if (p->p_endpoint >= 0) {
#if DEBUG_RACE
make_zero64(p->p_cpu_time_left);
#else
/* if (tsc_delta < p->p_cpu_time_left) in 64bit */
if (ex64hi(tsc_delta) < ex64hi(p->p_cpu_time_left) ||
(ex64hi(tsc_delta) == ex64hi(p->p_cpu_time_left) &&
ex64lo(tsc_delta) < ex64lo(p->p_cpu_time_left)))
p->p_cpu_time_left = sub64(p->p_cpu_time_left, tsc_delta);
else {
make_zero64(p->p_cpu_time_left);
}
#endif
}
*__tsc_ctr_switch = tsc;
}