boolean_t swtch_pri( __unused struct swtch_pri_args *args) { register processor_t myprocessor; boolean_t result; disable_preemption(); myprocessor = current_processor(); if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return (FALSE); } enable_preemption(); counter(c_swtch_pri_block++); thread_depress_abstime(thread_depress_time); thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD); thread_depress_abort_internal(current_thread()); disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); return (result); }
static void swtch_pri_continue(void) { register processor_t myprocessor; boolean_t result; thread_depress_abort_internal(current_thread()); disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; mp_enable_preemption(); thread_syscall_return(result); /*NOTREACHED*/ }
static void thread_switch_continue(void) { register thread_t self = current_thread(); int option = self->saved.swtch.option; boolean_t reenable_workq_callback = self->saved.swtch.reenable_workq_callback; if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS) thread_depress_abort_internal(self); if (reenable_workq_callback) thread_switch_enable_workqueue_sched_callback(); thread_syscall_return(KERN_SUCCESS); /*NOTREACHED*/ }
/* * thread_depress_abort: * * Prematurely abort priority depression if there is one. */ kern_return_t thread_depress_abort( register thread_t thread) { kern_return_t result; if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); thread_mtx_lock(thread); if (thread->active) result = thread_depress_abort_internal(thread); else result = KERN_TERMINATED; thread_mtx_unlock(thread); return (result); }
/* * thread_switch: * * Context switch. User may supply thread hint. */ kern_return_t thread_switch( struct thread_switch_args *args) { thread_t thread = THREAD_NULL; thread_t self = current_thread(); mach_port_name_t thread_name = args->thread_name; int option = args->option; mach_msg_timeout_t option_time = args->option_time; uint32_t scale_factor = NSEC_PER_MSEC; boolean_t reenable_workq_callback = FALSE; boolean_t depress_option = FALSE; boolean_t wait_option = FALSE; /* * Validate and process option. */ switch (option) { case SWITCH_OPTION_NONE: workqueue_thread_yielded(); break; case SWITCH_OPTION_WAIT: wait_option = TRUE; workqueue_thread_yielded(); break; case SWITCH_OPTION_DEPRESS: depress_option = TRUE; workqueue_thread_yielded(); break; case SWITCH_OPTION_DISPATCH_CONTENTION: scale_factor = NSEC_PER_USEC; wait_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; case SWITCH_OPTION_OSLOCK_DEPRESS: depress_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; case SWITCH_OPTION_OSLOCK_WAIT: wait_option = TRUE; if (thread_switch_disable_workqueue_sched_callback()) reenable_workq_callback = TRUE; break; default: return (KERN_INVALID_ARGUMENT); } /* * Translate the port name if supplied. */ if (thread_name != MACH_PORT_NULL) { ipc_port_t port; if (ipc_port_translate_send(self->task->itk_space, thread_name, &port) == KERN_SUCCESS) { ip_reference(port); ip_unlock(port); thread = convert_port_to_thread(port); ip_release(port); if (thread == self) { thread_deallocate(thread); thread = THREAD_NULL; } } } if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) { if (thread != THREAD_NULL) { if (thread->task != self->task) { /* * OSLock boosting only applies to other threads * in your same task (even if you have a port for * a thread in another task) */ thread_deallocate(thread); thread = THREAD_NULL; } else { /* * Attempt to kick the lock owner up to our same IO throttling tier. * If the thread is currently blocked in throttle_lowpri_io(), * it will immediately break out. * * TODO: SFI break out? */ int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO); set_thread_iotier_override(thread, new_policy); } } } /* * Try to handoff if supplied. */ if (thread != THREAD_NULL) { spl_t s = splsched(); /* This may return a different thread if the target is pushing on something */ thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread); KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE, thread_tid(thread), thread->state, pulled_thread ? TRUE : FALSE, 0, 0); if (pulled_thread != THREAD_NULL) { /* We can't be dropping the last ref here */ thread_deallocate_safe(thread); if (wait_option) assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor); else if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread); /* NOTREACHED */ panic("returned from thread_run!"); } splx(s); thread_deallocate(thread); } if (wait_option) assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor); else if (depress_option) thread_depress_ms(option_time); self->saved.swtch.option = option; self->saved.swtch.reenable_workq_callback = reenable_workq_callback; thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD); if (depress_option) thread_depress_abort_internal(self); if (reenable_workq_callback) thread_switch_enable_workqueue_sched_callback(); return (KERN_SUCCESS); }