/*
* Called with interrupts disabled.
*/
void
processor_doshutdown(
processor_t processor)
{
thread_t old_thread, self = current_thread();
processor_t prev;
processor_set_t pset;
/*
* Get onto the processor to shutdown
*/
prev = thread_bind(processor);
thread_block(THREAD_CONTINUE_NULL);
assert(processor->state == PROCESSOR_SHUTDOWN);
#if CONFIG_DTRACE
if (dtrace_cpu_state_changed_hook)
(*dtrace_cpu_state_changed_hook)(processor->cpu_id, FALSE);
#endif
ml_cpu_down();
#if HIBERNATION
if (processor_avail_count < 2) {
hibernate_vm_lock();
hibernate_vm_unlock();
}
#endif
pset = processor->processor_set;
pset_lock(pset);
processor->state = PROCESSOR_OFF_LINE;
--pset->online_processor_count;
(void)hw_atomic_sub(&processor_avail_count, 1);
commpage_update_active_cpus();
SCHED(processor_queue_shutdown)(processor);
/* pset lock dropped */
/*
* Continue processor shutdown in shutdown context.
*
* We save the current context in machine_processor_shutdown in such a way
* that when this thread is next invoked it will return from here instead of
* from the machine_switch_context() in thread_invoke like a normal context switch.
*
* As such, 'old_thread' is neither the idle thread nor the current thread - it's whatever
* thread invoked back to this one. (Usually, it's another processor's idle thread.)
*
* TODO: Make this a real thread_run of the idle_thread, so we don't have to keep this in sync
* with thread_invoke.
*/
thread_bind(prev);
old_thread = machine_processor_shutdown(self, processor_offline, processor);
thread_dispatch(old_thread, self);
}
/*
* Adjust the Universal (Posix) time gradually.
*/
kern_return_t
host_adjust_time(
host_t host,
time_value_t newadj,
time_value_t *oldadj) /* OUT */
{
time_value_t oadj;
integer_t ndelta;
spl_t s;
if (host == HOST_NULL)
return (KERN_INVALID_HOST);
ndelta = (newadj.seconds * 1000000) + newadj.microseconds;
#if NCPUS > 1
thread_bind(current_thread(), master_processor);
mp_disable_preemption();
if (current_processor() != master_processor) {
mp_enable_preemption();
thread_block((void (*)(void)) 0);
} else {
mp_enable_preemption();
}
#endif /* NCPUS > 1 */
s = splclock();
oadj.seconds = timedelta / 1000000;
oadj.microseconds = timedelta % 1000000;
if (timedelta == 0) {
if (ndelta > bigadj)
tickdelta = 10 * tickadj;
else
tickdelta = tickadj;
}
if (ndelta % tickdelta)
ndelta = ndelta / tickdelta * tickdelta;
timedelta = ndelta;
splx(s);
#if NCPUS > 1
thread_bind(current_thread(), PROCESSOR_NULL);
#endif /* NCPUS > 1 */
*oldadj = oadj;
return (KERN_SUCCESS);
}
__private_extern__ kern_return_t
chudxnu_unbind_thread(thread_t thread, __unused int options)
{
if(thread == current_thread())
thread_bind(PROCESSOR_NULL);
return KERN_SUCCESS;
}
/*
* Called with interrupts disabled.
*/
void
processor_doshutdown(
processor_t processor)
{
thread_t old_thread, self = current_thread();
processor_t prev;
processor_set_t pset;
/*
* Get onto the processor to shutdown
*/
prev = thread_bind(processor);
thread_block(THREAD_CONTINUE_NULL);
assert(processor->state == PROCESSOR_SHUTDOWN);
#if CONFIG_DTRACE
if (dtrace_cpu_state_changed_hook)
(*dtrace_cpu_state_changed_hook)(processor->cpu_id, FALSE);
#endif
ml_cpu_down();
#if HIBERNATION
if (processor_avail_count < 2) {
hibernate_vm_lock();
hibernate_vm_unlock();
}
#endif
pset = processor->processor_set;
pset_lock(pset);
processor->state = PROCESSOR_OFF_LINE;
--pset->online_processor_count;
(void)hw_atomic_sub(&processor_avail_count, 1);
commpage_update_active_cpus();
SCHED(processor_queue_shutdown)(processor);
/* pset lock dropped */
/*
* Continue processor shutdown in shutdown context.
*/
thread_bind(prev);
old_thread = machine_processor_shutdown(self, processor_offline, processor);
thread_dispatch(old_thread, self);
}
__private_extern__
kern_return_t chudxnu_bind_current_thread(int cpu)
{
if(cpu>=0 && cpu<chudxnu_avail_cpu_count()) { /* make sure cpu # is sane */
thread_bind(current_thread(), processor_ptr[cpu]);
thread_block((void (*)(void)) 0);
return KERN_SUCCESS;
} else {
return KERN_FAILURE;
}
}
/*
* Set the Universal (Posix) time. Privileged call.
*/
kern_return_t
host_set_time(
host_t host,
time_value_t new_time)
{
spl_t s;
if (host == HOST_NULL)
return(KERN_INVALID_HOST);
#if NCPUS > 1
thread_bind(current_thread(), master_processor);
mp_disable_preemption();
if (current_processor() != master_processor) {
mp_enable_preemption();
thread_block((void (*)(void)) 0);
} else {
mp_enable_preemption();
}
#endif /* NCPUS > 1 */
s = splhigh();
time = new_time;
update_mapped_time(&time);
#if PTIME_MACH_RT
rtc_gettime_interrupts_disabled((tvalspec_t *)&last_utime_tick);
#endif /* PTIME_MACH_RT */
#if 0
(void)bbc_settime((time_value_t *)&time);
#endif
splx(s);
#if NCPUS > 1
thread_bind(current_thread(), PROCESSOR_NULL);
#endif /* NCPUS > 1 */
return (KERN_SUCCESS);
}
struct thread *thread_send(struct thread *image, pid_t target, portid_t port, struct msg *msg) {
struct process *p_targ;
struct thread *new_image;
/* find target process */
p_targ = process_get(target);
/* check process */
if (!p_targ || !p_targ->entry) {
return image;
}
/* create new thread */
new_image = thread_alloc();
thread_bind(new_image, p_targ);
new_image->ds = 0x23;
new_image->cs = 0x1B;
new_image->ss = 0x23;
new_image->eflags = 0;
new_image->useresp = new_image->stack + SEGSZ;
new_image->proc = p_targ;
new_image->eip = p_targ->entry;
/* set up registers in new thread */
new_image->ebx = 0;
new_image->ecx = (msg) ? msg->count : 0;
new_image->edx = port;
new_image->esi = (image) ? image->proc->pid : 0;
new_image->edi = 0;
new_image->msg = msg;
/* set new thread's user id */
new_image->user = (!image || p_targ->user) ? p_targ->user : image->user;
/* insert new thread into scheduler */
schedule_insert(new_image);
/* return new thread */
return new_image;
}
/*
* This method will bind a given thread to the requested CPU starting at the
* next time quantum. If the thread is the current thread, this method will
* force a thread_block(). The result is that if you call this method on the
* current thread, you will be on the requested CPU when this method returns.
*/
__private_extern__ kern_return_t
chudxnu_bind_thread(thread_t thread, int cpu, __unused int options)
{
processor_t proc = NULL;
if(cpu < 0 || (unsigned int)cpu >= real_ncpus) // sanity check
return KERN_FAILURE;
// temporary restriction until after phase 2 of the scheduler
if(thread != current_thread())
return KERN_FAILURE;
proc = cpu_to_processor(cpu);
/*
* Potentially racey, but mainly to prevent bind to shutdown
* processor.
*/
if(proc && !(proc->state == PROCESSOR_OFF_LINE) &&
!(proc->state == PROCESSOR_SHUTDOWN)) {
thread_bind(proc);
/*
* If we're trying to bind the current thread, and
* we're not on the target cpu, and not at interrupt
* context, block the current thread to force a
* reschedule on the target CPU.
*/
if(thread == current_thread() &&
!ml_at_interrupt_context() && cpu_number() != cpu) {
(void)thread_block(THREAD_CONTINUE_NULL);
}
return KERN_SUCCESS;
}
return KERN_FAILURE;
}
void
processor_doaction(
processor_t processor)
{
thread_t this_thread;
spl_t s;
register processor_set_t pset;
#if MACH_HOST
register processor_set_t new_pset;
register thread_t thread;
register thread_t prev_thread = THREAD_NULL;
thread_act_t thr_act;
boolean_t have_pset_ref = FALSE;
#endif /* MACH_HOST */
/*
* Get onto the processor to shutdown
*/
this_thread = current_thread();
thread_bind(this_thread, processor);
thread_block((void (*)(void)) 0);
pset = processor->processor_set;
#if MACH_HOST
/*
* If this is the last processor in the processor_set,
* stop all the threads first.
*/
pset_lock(pset);
if (pset->processor_count == 1) {
thread = (thread_t) queue_first(&pset->threads);
prev_thread = THREAD_NULL;
pset->ref_count++;
have_pset_ref = TRUE;
pset->empty = TRUE;
/*
* loop through freezing the processor set assignment
* and reference counting the threads;
*/
while (!queue_end(&pset->threads, (queue_entry_t) thread)) {
thread_reference(thread);
pset_unlock(pset);
/*
* Freeze the thread on the processor set.
* If it's moved, just release the reference.
* Get the next thread in the processor set list
* from the last one which was frozen.
*/
if( thread_stop_freeze(thread, pset) )
prev_thread = thread;
else
thread_deallocate(thread);
pset_lock(pset);
if( prev_thread != THREAD_NULL )
thread = (thread_t)queue_next(&prev_thread->pset_threads);
else
thread = (thread_t) queue_first(&pset->threads);
}
/*
* Remove the processor from the set so that when the threads
* are unstopped below the ones blocked in the kernel don't
* start running again.
*/
s = splsched();
processor_lock(processor);
pset_remove_processor(pset, processor);
/*
* Prevent race with another processor being added to the set
* See code after Restart_pset:
* while(new_pset->empty && new_pset->processor_count > 0)
*
* ... it tests for the condition where a new processor is
* added to the set while the last one is still being removed.
*/
pset->processor_count++; /* block new processors being added */
assert( pset->processor_count == 1 );
/*
* Release the thread assignment locks, unstop the threads and
* release the thread references which were taken above.
*/
thread = (thread_t) queue_first(&pset->threads);
while( !queue_empty( &pset->threads) && (thread != THREAD_NULL) ) {
prev_thread = thread;
if( queue_end(&pset->threads, (queue_entry_t) thread) )
thread = THREAD_NULL;
else
thread = (thread_t) queue_next(&prev_thread->pset_threads);
pset_unlock(pset);
thread_unfreeze(prev_thread);
thread_unstop(prev_thread);
thread_deallocate(prev_thread);
pset_lock(pset);
}
/*
* allow a processor to be added to the empty pset
*/
pset->processor_count--;
}
else {
/* not last processor in set */
#endif /* MACH_HOST */
/*
* At this point, it is ok to rm the processor from the pset.
*/
s = splsched();
processor_lock(processor);
pset_remove_processor(pset, processor);
#if MACH_HOST
}
pset_unlock(pset);
/*
* Copy the next pset pointer into a local variable and clear
* it because we are taking over its reference.
*/
new_pset = processor->processor_set_next;
processor->processor_set_next = PROCESSOR_SET_NULL;
if (processor->state == PROCESSOR_ASSIGN) {
Restart_pset:
/*
* Nasty problem: we want to lock the target pset, but
* we have to enable interrupts to do that which requires
* dropping the processor lock. While the processor
* is unlocked, it could be reassigned or shutdown.
*/
processor_unlock(processor);
splx(s);
/*
* Lock target pset and handle remove last / assign first race.
* Only happens if there is more than one action thread.
*/
pset_lock(new_pset);
while (new_pset->empty && new_pset->processor_count > 0) {
pset_unlock(new_pset);
while (*(volatile boolean_t *)&new_pset->empty &&
*(volatile int *)&new_pset->processor_count > 0)
/* spin */;
pset_lock(new_pset);
}
/*
* Finally relock the processor and see if something changed.
* The only possibilities are assignment to a different pset
* and shutdown.
*/
s = splsched();
processor_lock(processor);
if (processor->state == PROCESSOR_SHUTDOWN) {
pset_unlock(new_pset);
goto shutdown; /* will release pset reference */
}
if (processor->processor_set_next != PROCESSOR_SET_NULL) {
/*
* Processor was reassigned. Drop the reference
* we have on the wrong new_pset, and get the
* right one. Involves lots of lock juggling.
*/
processor_unlock(processor);
splx(s);
pset_unlock(new_pset);
pset_deallocate(new_pset);
s = splsched();
processor_lock(processor);
new_pset = processor->processor_set_next;
processor->processor_set_next = PROCESSOR_SET_NULL;
goto Restart_pset;
}
/*
* If the pset has been deactivated since the operation
* was requested, redirect to the default pset.
*/
if (!(new_pset->active)) {
pset_unlock(new_pset);
pset_deallocate(new_pset);
new_pset = &default_pset;
pset_lock(new_pset);
new_pset->ref_count++;
}
/*
* Do assignment, then wakeup anyone waiting for it.
* Finally context switch to have it take effect.
*/
pset_add_processor(new_pset, processor);
if (new_pset->empty) {
/*
* Set all the threads loose
*/
thread = (thread_t) queue_first(&new_pset->threads);
while (!queue_end(&new_pset->threads,(queue_entry_t)thread)) {
thr_act = thread_lock_act(thread);
thread_release(thread->top_act);
act_unlock_thread(thr_act);
thread = (thread_t) queue_next(&thread->pset_threads);
}
new_pset->empty = FALSE;
}
processor->processor_set_next = PROCESSOR_SET_NULL;
processor->state = PROCESSOR_RUNNING;
thread_wakeup((event_t)processor);
processor_unlock(processor);
splx(s);
pset_unlock(new_pset);
/*
* Clean up dangling references, and release our binding.
*/
pset_deallocate(new_pset);
if (have_pset_ref)
pset_deallocate(pset);
if (prev_thread != THREAD_NULL)
thread_deallocate(prev_thread);
thread_bind(this_thread, PROCESSOR_NULL);
thread_block((void (*)(void)) 0);
return;
}
shutdown:
#endif /* MACH_HOST */
/*
* Do shutdown, make sure we live when processor dies.
*/
if (processor->state != PROCESSOR_SHUTDOWN) {
printf("state: %d\n", processor->state);
panic("action_thread -- bad processor state");
}
processor_unlock(processor);
/*
* Clean up dangling references, and release our binding.
*/
#if MACH_HOST
if (new_pset != PROCESSOR_SET_NULL)
pset_deallocate(new_pset);
if (have_pset_ref)
pset_deallocate(pset);
if (prev_thread != THREAD_NULL)
thread_deallocate(prev_thread);
#endif /* MACH_HOST */
thread_bind(this_thread, PROCESSOR_NULL);
switch_to_shutdown_context(this_thread,
processor_doshutdown,
processor);
splx(s);
}
/*
* Now running in a thread. Create the rest of the kernel threads
* and the bootstrap task.
*/
void start_kernel_threads()
{
register int i;
/*
* Create the idle threads and the other
* service threads.
*/
for (i = 0; i < NCPUS; i++) {
if (machine_slot[i].is_cpu) {
thread_t th;
(void) thread_create(kernel_task, &th);
thread_bind(th, cpu_to_processor(i));
thread_start(th, idle_thread);
thread_doswapin(th);
(void) thread_resume(th);
}
}
(void) kernel_thread(kernel_task, reaper_thread, (char *) 0);
(void) kernel_thread(kernel_task, swapin_thread, (char *) 0);
(void) kernel_thread(kernel_task, sched_thread, (char *) 0);
#if NCPUS > 1
/*
* Create the shutdown thread.
*/
(void) kernel_thread(kernel_task, action_thread, (char *) 0);
/*
* Allow other CPUs to run.
*/
start_other_cpus();
#endif /* NCPUS > 1 */
/*
* Create the device service.
*/
device_service_create();
/*
* Initialize kernel task's creation time.
* When we created the kernel task in task_init, the mapped
* time was not yet available. Now, last thing before starting
* the user bootstrap, record the current time as the kernel
* task's creation time.
*/
record_time_stamp (&kernel_task->creation_time);
/*
* Start the user bootstrap.
*/
bootstrap_create();
#if XPR_DEBUG
xprinit(); /* XXX */
#endif /* XPR_DEBUG */
/*
* Become the pageout daemon.
*/
(void) spl0();
vm_pageout();
/*NOTREACHED*/
}
struct thread *init(struct multiboot *mboot, uint32_t mboot_magic) {
struct process *idle, *init;
struct module *module;
struct memory_map *mem_map;
size_t mem_map_count, i, addr;
uintptr_t boot_image_size;
void *boot_image;
struct elf32_ehdr *init_image;
struct elf32_ehdr *dl_image;
/* initialize debugging output */
debug_init();
debug_printf("Rhombus Operating System Kernel v0.8a\n");
/* check multiboot header */
if (mboot_magic != 0x2BADB002) {
debug_panic("bootloader is not multiboot compliant");
}
/* touch pages for the kernel heap */
for (i = KSPACE; i < KERNEL_HEAP_END; i += SEGSZ) {
page_touch(i);
}
/* identity map kernel boot frames */
for (i = KSPACE + KERNEL_BOOT; i < KSPACE + KERNEL_BOOT_END; i += PAGESZ) {
page_set(i, page_fmt(i - KSPACE, PF_PRES | PF_RW));
}
/* parse the multiboot memory map to find the size of memory */
mem_map = (void*) (mboot->mmap_addr + KSPACE);
mem_map_count = mboot->mmap_length / sizeof(struct memory_map);
for (i = 0; i < mem_map_count; i++) {
if (mem_map[i].type == 1 && mem_map[i].base_addr_low <= 0x100000) {
for (addr = 0; addr < mem_map[i].length_low; addr += PAGESZ) {
frame_add(mem_map[i].base_addr_low + addr);
}
}
}
/* bootstrap process 0 (idle) */
idle = process_alloc();
idle->space = cpu_get_cr3();
idle->user = 0;
/* fork process 1 (init) and switch */
init = process_clone(idle, NULL);
process_switch(init);
/* get multiboot module information */
if (mboot->mods_count < 3) {
if (mboot->mods_count < 2) {
if (mboot->mods_count < 1) {
debug_panic("no boot or init or dl modules found");
}
else {
debug_panic("no boot or dl modules found");
}
}
else {
debug_panic("no dl module found");
}
}
module = (void*) (mboot->mods_addr + KSPACE);
init_image = (void*) (module[0].mod_start + KSPACE);
boot_image = (void*) (module[1].mod_start + KSPACE);
dl_image = (void*) (module[2].mod_start + KSPACE);
boot_image_size = module[1].mod_end - module[1].mod_start;
/* move boot image to BOOT_IMAGE in userspace */
mem_alloc(BOOT_IMAGE, boot_image_size, PF_PRES | PF_USER | PF_RW);
memcpy((void*) BOOT_IMAGE, boot_image, boot_image_size);
/* bootstrap thread 0 in init */
thread_bind(init->thread[0], init);
init->thread[0]->useresp = init->thread[0]->stack + SEGSZ;
init->thread[0]->esp = (uintptr_t) &init->thread[0]->num;
init->thread[0]->ss = 0x23;
init->thread[0]->ds = 0x23;
init->thread[0]->cs = 0x1B;
init->thread[0]->eflags = cpu_get_eflags() | 0x3200; /* IF, IOPL = 3 */
/* bootstrap idle thread */
idle->thread[0] = &__idle_thread;
__idle_thread.proc = idle;
/* load dl */
if (elf_check_file(dl_image)) {
debug_panic("dl.so is not a valid ELF executable");
}
elf_load_file(dl_image);
/* execute init */
if (elf_check_file(init_image)) {
debug_panic("init is not a valid ELF executable");
}
elf_load_file(init_image);
init->thread[0]->eip = init_image->e_entry;
/* register system calls */
int_set_handler(SYSCALL_SEND, syscall_send);
int_set_handler(SYSCALL_DONE, syscall_done);
int_set_handler(SYSCALL_WHEN, syscall_when);
int_set_handler(SYSCALL_RIRQ, syscall_rirq);
int_set_handler(SYSCALL_ALSO, syscall_also);
int_set_handler(SYSCALL_STAT, syscall_stat);
int_set_handler(SYSCALL_PAGE, syscall_page);
int_set_handler(SYSCALL_PHYS, syscall_phys);
int_set_handler(SYSCALL_FORK, syscall_fork);
int_set_handler(SYSCALL_EXIT, syscall_exit);
int_set_handler(SYSCALL_STOP, syscall_stop);
int_set_handler(SYSCALL_WAKE, syscall_wake);
int_set_handler(SYSCALL_GPID, syscall_gpid);
int_set_handler(SYSCALL_TIME, syscall_time);
int_set_handler(SYSCALL_USER, syscall_user);
int_set_handler(SYSCALL_AUTH, syscall_auth);
int_set_handler(SYSCALL_PROC, syscall_proc);
int_set_handler(SYSCALL_KILL, syscall_kill);
int_set_handler(SYSCALL_VM86, syscall_vm86);
int_set_handler(SYSCALL_NAME, syscall_name);
int_set_handler(SYSCALL_REAP, syscall_reap);
/* register fault handlers */
int_set_handler(FAULT_DE, fault_float);
int_set_handler(FAULT_DB, fault_generic);
int_set_handler(FAULT_NI, fault_generic);
int_set_handler(FAULT_BP, fault_generic);
int_set_handler(FAULT_OF, fault_generic);
int_set_handler(FAULT_BR, fault_generic);
int_set_handler(FAULT_UD, fault_generic);
int_set_handler(FAULT_NM, fault_nomath);
int_set_handler(FAULT_DF, fault_double);
int_set_handler(FAULT_CO, fault_float);
int_set_handler(FAULT_TS, fault_generic);
int_set_handler(FAULT_NP, fault_generic);
int_set_handler(FAULT_SS, fault_generic);
int_set_handler(FAULT_GP, fault_gpf);
int_set_handler(FAULT_PF, fault_page);
int_set_handler(FAULT_MF, fault_float);
int_set_handler(FAULT_AC, fault_generic);
int_set_handler(FAULT_MC, fault_generic);
int_set_handler(FAULT_XM, fault_nomath);
/* start timer (for preemption) */
timer_set_freq(64);
/* initialize FPU/MMX/SSE */
cpu_init_fpu();
/* drop to usermode, scheduling the next thread */
debug_printf("dropping to usermode\n");
return thread_switch(NULL, schedule_next());
}
/*
* Now running in a thread. Kick off other services,
* invoke user bootstrap, enter pageout loop.
*/
static void
kernel_bootstrap_thread(void)
{
processor_t processor = current_processor();
#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
kernel_bootstrap_thread_log("idle_thread_create");
/*
* Create the idle processor thread.
*/
idle_thread_create(processor);
/*
* N.B. Do not stick anything else
* before this point.
*
* Start up the scheduler services.
*/
kernel_bootstrap_thread_log("sched_startup");
sched_startup();
/*
* Thread lifecycle maintenance (teardown, stack allocation)
*/
kernel_bootstrap_thread_log("thread_daemon_init");
thread_daemon_init();
/* Create kernel map entry reserve */
vm_kernel_reserved_entry_init();
/*
* Thread callout service.
*/
kernel_bootstrap_thread_log("thread_call_initialize");
thread_call_initialize();
/*
* Remain on current processor as
* additional processors come online.
*/
kernel_bootstrap_thread_log("thread_bind");
thread_bind(processor);
/*
* Initialize ipc thread call support.
*/
kernel_bootstrap_thread_log("ipc_thread_call_init");
ipc_thread_call_init();
/*
* Kick off memory mapping adjustments.
*/
kernel_bootstrap_thread_log("mapping_adjust");
mapping_adjust();
/*
* Create the clock service.
*/
kernel_bootstrap_thread_log("clock_service_create");
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
kth_started = 1;
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the physical copy window for processor 0
* This is required before starting kicking off IOKit.
*/
cpu_physwindow_init(0);
#endif
#if MACH_KDP
kernel_bootstrap_log("kdp_init");
kdp_init();
#endif
#if ALTERNATE_DEBUGGER
alternate_debugger_init();
#endif
#if KPC
kpc_init();
#endif
#if CONFIG_ECC_LOGGING
ecc_log_init();
#endif
#if KPERF
kperf_bootstrap();
#endif
#if HYPERVISOR
hv_support_init();
#endif
#if CONFIG_TELEMETRY
kernel_bootstrap_log("bootprofile_init");
bootprofile_init();
#endif
#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX
vmx_init();
#endif
#if (defined(__i386__) || defined(__x86_64__))
if (kdebug_serial) {
new_nkdbufs = 1;
if (trace_typefilter == 0)
trace_typefilter = 1;
}
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
if (trace_typefilter)
start_kern_tracing_with_typefilter(new_nkdbufs,
FALSE,
trace_typefilter);
else
start_kern_tracing(new_nkdbufs, FALSE);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
kernel_bootstrap_log("prng_init");
prng_cpu_init(master_cpu);
#ifdef IOKIT
PE_init_iokit();
#endif
assert(ml_get_interrupts_enabled() == FALSE);
(void) spllo(); /* Allow interruptions */
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the copy window for processor 0
* This also allocates window space for all other processors.
* However, this is dependent on the number of processors - so this call
* must be after IOKit has been started because IOKit performs processor
* discovery.
*/
cpu_userwindow_init(0);
#endif
#if (!defined(__i386__) && !defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
if (trace_typefilter)
start_kern_tracing_with_typefilter(new_nkdbufs, FALSE, trace_typefilter);
else
start_kern_tracing(new_nkdbufs, FALSE);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
/*
* Initialize the shared region module.
*/
vm_shared_region_init();
vm_commpage_init();
vm_commpage_text_init();
#if CONFIG_MACF
kernel_bootstrap_log("mac_policy_initmach");
mac_policy_initmach();
#endif
#if CONFIG_SCHED_SFI
kernel_bootstrap_log("sfi_init");
sfi_init();
#endif
/*
* Initialize the globals used for permuting kernel
* addresses that may be exported to userland as tokens
* using VM_KERNEL_ADDRPERM()/VM_KERNEL_ADDRPERM_EXTERNAL().
* Force the random number to be odd to avoid mapping a non-zero
* word-aligned address to zero via addition.
* Note: at this stage we can use the cryptographically secure PRNG
* rather than early_random().
*/
read_random(&vm_kernel_addrperm, sizeof(vm_kernel_addrperm));
vm_kernel_addrperm |= 1;
read_random(&buf_kernel_addrperm, sizeof(buf_kernel_addrperm));
buf_kernel_addrperm |= 1;
read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext));
vm_kernel_addrperm_ext |= 1;
vm_set_restrictions();
/*
* Start the user bootstrap.
*/
#ifdef MACH_BSD
bsd_init();
#endif
/*
* Get rid of segments used to bootstrap kext loading. This removes
* the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
*/
OSKextRemoveKextBootstrap();
serial_keyboard_init(); /* Start serial keyboard if wanted */
vm_page_init_local_q();
thread_bind(PROCESSOR_NULL);
/*
* Become the pageout daemon.
*/
vm_pageout();
/*NOTREACHED*/
}
kern_return_t
processor_start(
processor_t processor)
{
processor_set_t pset;
thread_t thread;
kern_return_t result;
spl_t s;
if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL)
return (KERN_INVALID_ARGUMENT);
if (processor == master_processor) {
processor_t prev;
prev = thread_bind(processor);
thread_block(THREAD_CONTINUE_NULL);
result = cpu_start(processor->cpu_id);
thread_bind(prev);
return (result);
}
s = splsched();
pset = processor->processor_set;
pset_lock(pset);
if (processor->state != PROCESSOR_OFF_LINE) {
pset_unlock(pset);
splx(s);
return (KERN_FAILURE);
}
processor->state = PROCESSOR_START;
pset_unlock(pset);
splx(s);
/*
* Create the idle processor thread.
*/
if (processor->idle_thread == THREAD_NULL) {
result = idle_thread_create(processor);
if (result != KERN_SUCCESS) {
s = splsched();
pset_lock(pset);
processor->state = PROCESSOR_OFF_LINE;
pset_unlock(pset);
splx(s);
return (result);
}
}
/*
* If there is no active thread, the processor
* has never been started. Create a dedicated
* start up thread.
*/
if ( processor->active_thread == THREAD_NULL &&
processor->next_thread == THREAD_NULL ) {
result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread);
if (result != KERN_SUCCESS) {
s = splsched();
pset_lock(pset);
processor->state = PROCESSOR_OFF_LINE;
pset_unlock(pset);
splx(s);
return (result);
}
s = splsched();
thread_lock(thread);
thread->bound_processor = processor;
processor->next_thread = thread;
thread->state = TH_RUN;
thread_unlock(thread);
splx(s);
thread_deallocate(thread);
}
if (processor->processor_self == IP_NULL)
ipc_processor_init(processor);
result = cpu_start(processor->cpu_id);
if (result != KERN_SUCCESS) {
s = splsched();
pset_lock(pset);
processor->state = PROCESSOR_OFF_LINE;
pset_unlock(pset);
splx(s);
return (result);
}
ipc_processor_enable(processor);
return (KERN_SUCCESS);
}
/*
* Running in virtual memory, on the interrupt stack.
* Does not return. Dispatches initial thread.
*
* Assumes that master_cpu is set.
*/
void
setup_main(void)
{
thread_t startup_thread;
printf_init();
panic_init();
sched_init();
vm_mem_bootstrap();
ipc_bootstrap();
vm_mem_init();
ipc_init();
/*
* As soon as the virtual memory system is up, we record
* that this CPU is using the kernel pmap.
*/
PMAP_ACTIVATE_KERNEL(master_cpu);
init_timers();
timeout_init();
#if CDLI > 0
ns_init(); /* Initialize CDLI */
#endif /* CDLI > 0 */
dev_lookup_init();
timeout_init();
machine_init();
machine_info.max_cpus = NCPUS;
machine_info.memory_size = mem_size;
machine_info.avail_cpus = 0;
machine_info.major_version = KERNEL_MAJOR_VERSION;
machine_info.minor_version = KERNEL_MINOR_VERSION;
#if XPR_DEBUG
xprbootstrap();
#endif /* XPR_DEBUG */
/*
* Initialize the IPC, task, and thread subsystems.
*/
clock_init();
utime_init();
ledger_init();
#if THREAD_SWAPPER
thread_swapper_init();
#endif /* THREAD_SWAPPER */
#if TASK_SWAPPER
task_swapper_init();
#endif /* TASK_SWAPPER */
task_init();
act_init();
thread_init();
subsystem_init();
#if TASK_SWAPPER
task_swappable(&realhost, kernel_task, FALSE);
#endif /* TASK_SWAPPER */
#if MACH_HOST
pset_sys_init();
#endif /* MACH_HOST */
/*
* Kick off the time-out driven routines by calling
* them the first time.
*/
recompute_priorities();
compute_mach_factor();
/*
* Initialize the Event Trace Analysis Package.
* Dynamic Phase: 2 of 2
*/
etap_init_phase2();
/*
* Create a kernel thread to start the other kernel
* threads. Thread_resume (from kernel_thread) calls
* thread_setrun, which may look at current thread;
* we must avoid this, since there is no current thread.
*/
/*
* Create the thread, and point it at the routine.
*/
(void) thread_create_at(kernel_task, &startup_thread,
start_kernel_threads);
#if NCPUS > 1 && PARAGON860
thread_bind(startup_thread, cpu_to_processor(master_cpu));
#endif
/*
* Pretend it is already running, and resume it.
* Since it looks as if it is running, thread_resume
* will not try to put it on the run queues.
*
* We can do all of this without locking, because nothing
* else is running yet.
*/
startup_thread->state |= TH_RUN;
(void) thread_resume(startup_thread->top_act);
/*
* Start the thread.
*/
cpu_launch_first_thread(startup_thread);
/*NOTREACHED*/
panic("cpu_launch_first_thread returns!");
}
/*
* The idea behind this state machine is taken from Linux's
* lockless stop-machine code.
*/
void *state_machine(void *args) {
struct rusage u, v;
int tid = (int)((unsigned long)args);
unsigned long assemble, stamp_counter, stamp_counter_mp;
int master = 0, slave;
__e_m(tid > (spawn - 1), "How'd this happen?");
if (tid == 0)
master = 1;
slave = !master;
/* We want to see how badly we ended up spinning,
* waiting for threads on other CPUs to catch up.
*
* We'll use these variables as very unscientific
* counters.
*/
assemble = stamp_counter = stamp_counter_mp = 0;
if (slave) {
while (atomic_read(&command) != SETUP);
}
if (master) {
atomic_set(&assembled, 0);
atomic_set(&command, SETUP);
}
/*
* We are screwed if this fails: there will be gaps
* in ids. And the _mp version will hang forever.
*/
thread_bind(tid);
/* Remember the page-faults and such. */
__w(
getrusage(RUSAGE_THREAD, &u) != 0,
"(errno=%d)", errno);
/* Done with setup. */
atomic_inc(&assembled);
/* wait for master to tell us to assemble */
if (slave) {
while (atomic_read(&command) != BEFORE_STAMP)
assemble++;
}
/* give assemble command */
if (master) {
/* We want to kick-off all the slaves at
* the same time. Wait for them to assemble. */
while (atomic_read(&assembled) != spawn);
atomic_set(&assembled, 0);
atomic_set(&command, BEFORE_STAMP);
}
/* assemble work */
atomic_inc(&assembled);
/* Everybody is at BEFORE_STAMP, raring to STAMP */
/* give stamp command */
if (master) {
/* wait for slaves */
while (atomic_read(&assembled) != spawn)
assemble++;
atomic_set(&assembled, 0);
atomic_set(&command, STAMP_COUNTER);
}
/* wait for stamp command */
if (slave) {
while (atomic_read(&command) != STAMP_COUNTER)
stamp_counter++;
}
/* stamp work */
rdtscll(counter[tid].ts);
atomic_inc(&assembled);
/* give stamp_mp command */
if (master) {
/* wait for slaves */
while (atomic_read(&assembled) != spawn)
stamp_counter++;
atomic_set(&assembled, 0);
atomic_set(&command, STAMP_COUNTER_MP);
}
/* wait for stamp command */
if (slave) {
while (atomic_read(&command) != STAMP_COUNTER_MP)
stamp_counter_mp++;
}
/*
* Do the work.
*/
while (atomic_read(&assembled) != tid);
rdtscll(counter_mp[tid].ts);
atomic_inc(&assembled);
if (master) {
while (atomic_read(&assembled) != spawn);
}
__w(
getrusage(RUSAGE_THREAD, &v) != 0,
"(errno=%d)", errno);
rundata[tid].u = u;
rundata[tid].v = v;
rundata[tid].assemble = assemble;
rundata[tid].stamp_counter = stamp_counter;
rundata[tid].stamp_counter_mp = stamp_counter_mp;
return NULL;
}
/*
* Now running in a thread. Kick off other services,
* invoke user bootstrap, enter pageout loop.
*/
static void
kernel_bootstrap_thread(void)
{
processor_t processor = current_processor();
#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
kernel_bootstrap_thread_kprintf("calling idle_thread_create\n");
/*
* Create the idle processor thread.
*/
idle_thread_create(processor);
/*
* N.B. Do not stick anything else
* before this point.
*
* Start up the scheduler services.
*/
kernel_bootstrap_thread_kprintf("calling sched_startup\n");
sched_startup();
/*
* Thread lifecycle maintenance (teardown, stack allocation)
*/
kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n");
thread_daemon_init();
/*
* Thread callout service.
*/
kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n");
thread_call_initialize();
/*
* Remain on current processor as
* additional processors come online.
*/
kernel_bootstrap_thread_kprintf("calling thread_bind\n");
thread_bind(processor);
/*
* Kick off memory mapping adjustments.
*/
kernel_bootstrap_thread_kprintf("calling mapping_adjust\n");
mapping_adjust();
/*
* Create the clock service.
*/
kernel_bootstrap_thread_kprintf("calling clock_service_create\n");
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
kth_started = 1;
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the physical copy window for processor 0
* This is required before starting kicking off IOKit.
*/
cpu_physwindow_init(0);
#endif
vm_kernel_reserved_entry_init();
#if MACH_KDP
kernel_bootstrap_kprintf("calling kdp_init\n");
kdp_init();
#endif
#if CONFIG_COUNTERS
pmc_bootstrap();
#endif
#if (defined(__i386__) || defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
start_kern_tracing(new_nkdbufs);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
#ifdef IOKIT
PE_init_iokit();
#endif
(void) spllo(); /* Allow interruptions */
#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
/*
* Create and initialize the copy window for processor 0
* This also allocates window space for all other processors.
* However, this is dependent on the number of processors - so this call
* must be after IOKit has been started because IOKit performs processor
* discovery.
*/
cpu_userwindow_init(0);
#endif
#if (!defined(__i386__) && !defined(__x86_64__))
if (turn_on_log_leaks && !new_nkdbufs)
new_nkdbufs = 200000;
start_kern_tracing(new_nkdbufs);
if (turn_on_log_leaks)
log_leaks = 1;
#endif
/*
* Initialize the shared region module.
*/
vm_shared_region_init();
vm_commpage_init();
vm_commpage_text_init();
#if CONFIG_MACF
mac_policy_initmach();
#endif
/*
* Initialize the global used for permuting kernel
* addresses that may be exported to userland as tokens
* using VM_KERNEL_ADDRPERM(). Force the random number
* to be odd to avoid mapping a non-zero
* word-aligned address to zero via addition.
*/
vm_kernel_addrperm = (vm_offset_t)early_random() | 1;
/*
* Start the user bootstrap.
*/
#ifdef MACH_BSD
bsd_init();
#endif
/*
* Get rid of segments used to bootstrap kext loading. This removes
* the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
*/
#if 0
OSKextRemoveKextBootstrap();
#endif
serial_keyboard_init(); /* Start serial keyboard if wanted */
vm_page_init_local_q();
thread_bind(PROCESSOR_NULL);
/*
* Become the pageout daemon.
*/
vm_pageout();
/*NOTREACHED*/
}
int main(int argc, char *argv[])
{
if (argc != 2)
{
fprintf(stderr, "usage: %s csr_matrix_file\n", argv[0]);
exit(0);
}
int i, j, k;
struct timespec start, end;
int num_threads = 1;
#pragma omp parallel
{
#pragma omp master
{
num_threads = omp_get_num_threads();
}
}
printf("Thread number: %d.\n", num_threads);
#pragma omp parallel for
for (i = 0; i < num_threads; i++)
{
int cpu = omp_get_thread_num();
thread_bind(cpu);
}
FILE *fp;
struct csr_mat_t csr, csr_re, csr_t, csr_t_re, csr_t_t;
struct blk_mat_t blk;
struct csr_cont_t csr_h, csr_v;
struct blk_cont_t blk_h, blk_t_h;
read_csr_mat(argv[1], &csr);
int rows = csr.rows;
int cols = csr.cols;
INT64 non_zeros = csr.non_zeros;
csr_transpose(&csr, &csr_t);
release_csr_mat(&csr);
int *reorder_map = (int*)malloc(cols * sizeof(int));
csr_reorder(&csr_t, &csr_re, reorder_map);
release_csr_mat(&csr_t);
csr_transpose(&csr_re, &csr_t_t);
release_csr_mat(&csr_re);
split_csr_lb_nz(&csr_t_t, &csr_h, num_threads, SPLIT_HORIZON);
release_csr_mat(&csr_t_t);
csr_cont_to_blk_cont(&csr_h, &blk_h);
release_csr_cont(&csr_h);
printf("Notify: finished the preprocessing.\n");
FLOAT *x = (FLOAT*)numa_alloc(cols * sizeof(FLOAT));
FLOAT *y = (FLOAT*)numa_alloc(rows * sizeof(FLOAT));
for (i = 0; i < cols; i++)
{
x[i] = 1.0;
}
// warm up
spmv_blks(&blk_h, x, y, NULL);
printf("Notify: begin csr spmv.\n");
clock_gettime(CLOCK_MONOTONIC_RAW, &start);
for (i = 0; i < LOOP_TIME; i++)
{
spmv_blks(&blk_h, x, y, NULL);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
double time = get_sec(&start, &end) / LOOP_TIME;
double gflops = 2.0 * non_zeros / time * 1e-9;
printf("Notify: blk spmv time = %lfs, perf = %lf GFLOPS.\n", time, gflops);
// result_file(y, rows);
return 0;
}
__private_extern__
kern_return_t chudxnu_unbind_current_thread(void)
{
thread_bind(current_thread(), PROCESSOR_NULL);
return KERN_SUCCESS;
}
void test_kernel()
{
int i;
struct timespec start, end;
double t, gflops;
thread_bind(0);
float *a = (float*)page_alloc(48 * 48 * sizeof(float));
float *b = (float*)page_alloc(48 * 48 * sizeof(float));
float *cn = (float*)page_alloc(48 * 48 * sizeof(float));
float *ca = (float*)page_alloc(48 * 48 * sizeof(float));
float *cf = (float*)page_alloc(48 * 48 * sizeof(float));
srand(time(NULL));
for (i = 0; i < 48 * 48; i++)
{
a[i] = (float)rand() / RAND_MAX;
b[i] = (float)rand() / RAND_MAX;
}
memset(cn, 0, 48 * 48 * sizeof(float));
memset(ca, 0, 48 * 48 * sizeof(float));
memset(cf, 0, 48 * 48 * sizeof(float));
// check error
sgemm_naive_48_48_48(a, b, cn);
sgemm_kernel_avx_48_48_48(a, b, ca);
sgemm_kernel_fma_48_48_48(a, b, cf);
printf("AVX-tuned version check result:\n");
verify(cn, ca);
printf("FMA-tuned version check result:\n");
verify(cn, cf);
// naive version
// warm up
for (i = 0; i < NAIVE_LOOP_TIME; i++)
{
sgemm_naive_48_48_48(a, b, cn);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &start);
for (i = 0; i < NAIVE_LOOP_TIME; i++)
{
sgemm_naive_48_48_48(a, b, cn);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
t = get_time(&start, &end);
gflops = 2.0 * NAIVE_LOOP_TIME * 48 * 48 * 48 / t * 1e-9;
printf("Naive version: time = %lfs, perf = %lf GFLOPS.\n", t, gflops);
// avx-tuned version
// warm up
for (i = 0; i < LOOP_TIME; i++)
{
sgemm_kernel_avx_48_48_48(a, b, ca);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &start);
for (i = 0; i < LOOP_TIME; i++)
{
sgemm_kernel_avx_48_48_48(a, b, ca);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
t = get_time(&start, &end);
gflops = 2.0 * LOOP_TIME * 48 * 48 * 48 / t * 1e-9;
printf("AVX-tuned version: time = %lfs, perf = %lf GFLOPS.\n", t, gflops);
// fma-tuned version
// warm up
for (i = 0; i < LOOP_TIME; i++)
{
sgemm_kernel_fma_48_48_48(a, b, cf);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &start);
for (i = 0; i < LOOP_TIME; i++)
{
sgemm_kernel_fma_48_48_48(a, b, cf);
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
t = get_time(&start, &end);
gflops = 2.0 * LOOP_TIME * 48 * 48 * 48 / t * 1e-9;
printf("FMA-tuned version: time = %lfs, perf = %lf GFLOPS.\n", t, gflops);
page_free(a, 48 * 48 * sizeof(float));
page_free(b, 48 * 48 * sizeof(float));
page_free(cn, 48 * 48 * sizeof(float));
page_free(ca, 48 * 48 * sizeof(float));
page_free(cf, 48 * 48 * sizeof(float));
}
/*
* Now running in a thread. Create the rest of the kernel threads
* and the bootstrap task.
*/
void start_kernel_threads()
{
register int i;
/*
* Create the idle threads and the other
* service threads.
*/
for (i = 0; i < NCPUS; i++) {
if (machine_slot[i].is_cpu) {
thread_t th;
(void) thread_create(kernel_task, &th);
thread_bind(th, cpu_to_processor(i));
thread_start(th, idle_thread);
thread_doswapin(th);
(void) thread_resume(th);
}
}
(void) kernel_thread(kernel_task, reaper_thread, (char *) 0);
(void) kernel_thread(kernel_task, swapin_thread, (char *) 0);
(void) kernel_thread(kernel_task, sched_thread, (char *) 0);
#if NCPUS > 1
/*
* Create the shutdown thread.
*/
(void) kernel_thread(kernel_task, action_thread, (char *) 0);
/*
* Allow other CPUs to run.
*/
start_other_cpus();
#endif NCPUS > 1
/*
* Create the device service.
*/
device_service_create();
/*
* Initialize NORMA ipc system.
*/
#if NORMA_IPC
norma_ipc_init();
#endif NORMA_IPC
/*
* Initialize NORMA vm system.
*/
#if NORMA_VM
norma_vm_init();
#endif NORMA_VM
/*
* Start the user bootstrap.
*/
bootstrap_create();
#if XPR_DEBUG
xprinit(); /* XXX */
#endif XPR_DEBUG
/*
* Become the pageout daemon.
*/
(void) spl0();
vm_pageout();
/*NOTREACHED*/
}
