kern_return_t thread_policy_get( thread_t thread, thread_policy_flavor_t flavor, thread_policy_t policy_info, mach_msg_type_number_t *count, boolean_t *get_default) { kern_return_t result = KERN_SUCCESS; spl_t s; if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); thread_mtx_lock(thread); if (!thread->active) { thread_mtx_unlock(thread); return (KERN_TERMINATED); } switch (flavor) { case THREAD_EXTENDED_POLICY: { boolean_t timeshare = TRUE; if (!(*get_default)) { s = splsched(); thread_lock(thread); if ( !(thread->sched_mode & TH_MODE_REALTIME) && !(thread->safe_mode & TH_MODE_REALTIME) ) { if (!(thread->sched_mode & TH_MODE_FAILSAFE)) timeshare = (thread->sched_mode & TH_MODE_TIMESHARE) != 0; else timeshare = (thread->safe_mode & TH_MODE_TIMESHARE) != 0; } else *get_default = TRUE; thread_unlock(thread); splx(s); } if (*count >= THREAD_EXTENDED_POLICY_COUNT) { thread_extended_policy_t info; info = (thread_extended_policy_t)policy_info; info->timeshare = timeshare; } break; } case THREAD_TIME_CONSTRAINT_POLICY: { thread_time_constraint_policy_t info; if (*count < THREAD_TIME_CONSTRAINT_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_time_constraint_policy_t)policy_info; if (!(*get_default)) { s = splsched(); thread_lock(thread); if ( (thread->sched_mode & TH_MODE_REALTIME) || (thread->safe_mode & TH_MODE_REALTIME) ) { info->period = thread->realtime.period; info->computation = thread->realtime.computation; info->constraint = thread->realtime.constraint; info->preemptible = thread->realtime.preemptible; } else *get_default = TRUE; thread_unlock(thread); splx(s); } if (*get_default) { info->period = 0; info->computation = std_quantum / 2; info->constraint = std_quantum; info->preemptible = TRUE; } break; } case THREAD_PRECEDENCE_POLICY: { thread_precedence_policy_t info; if (*count < THREAD_PRECEDENCE_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_precedence_policy_t)policy_info; if (!(*get_default)) { s = splsched(); thread_lock(thread); info->importance = thread->importance; thread_unlock(thread); splx(s); } else info->importance = 0; break; } case THREAD_AFFINITY_POLICY: { thread_affinity_policy_t info; if (!thread_affinity_is_supported()) { result = KERN_NOT_SUPPORTED; break; } if (*count < THREAD_AFFINITY_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_affinity_policy_t)policy_info; if (!(*get_default)) info->affinity_tag = thread_affinity_get(thread); else info->affinity_tag = THREAD_AFFINITY_TAG_NULL; break; } default: result = KERN_INVALID_ARGUMENT; break; } thread_mtx_unlock(thread); return (result); }
kern_return_t processor_start( processor_t processor) { processor_set_t pset; thread_t thread; kern_return_t result; spl_t s; if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL) return (KERN_INVALID_ARGUMENT); if (processor == master_processor) { processor_t prev; prev = thread_bind(processor); thread_block(THREAD_CONTINUE_NULL); result = cpu_start(processor->cpu_id); thread_bind(prev); return (result); } s = splsched(); pset = processor->processor_set; pset_lock(pset); if (processor->state != PROCESSOR_OFF_LINE) { pset_unlock(pset); splx(s); return (KERN_FAILURE); } processor->state = PROCESSOR_START; pset_unlock(pset); splx(s); /* * Create the idle processor thread. */ if (processor->idle_thread == THREAD_NULL) { result = idle_thread_create(processor); if (result != KERN_SUCCESS) { s = splsched(); pset_lock(pset); processor->state = PROCESSOR_OFF_LINE; pset_unlock(pset); splx(s); return (result); } } /* * If there is no active thread, the processor * has never been started. Create a dedicated * start up thread. */ if ( processor->active_thread == THREAD_NULL && processor->next_thread == THREAD_NULL ) { result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread); if (result != KERN_SUCCESS) { s = splsched(); pset_lock(pset); processor->state = PROCESSOR_OFF_LINE; pset_unlock(pset); splx(s); return (result); } s = splsched(); thread_lock(thread); thread->bound_processor = processor; processor->next_thread = thread; thread->state = TH_RUN; thread_unlock(thread); splx(s); thread_deallocate(thread); } if (processor->processor_self == IP_NULL) ipc_processor_init(processor); result = cpu_start(processor->cpu_id); if (result != KERN_SUCCESS) { s = splsched(); pset_lock(pset); processor->state = PROCESSOR_OFF_LINE; pset_unlock(pset); splx(s); return (result); } ipc_processor_enable(processor); return (KERN_SUCCESS); }
kern_return_t thread_info_internal( register thread_t thread, thread_flavor_t flavor, thread_info_t thread_info_out, /* ptr to OUT array */ mach_msg_type_number_t *thread_info_count) /*IN/OUT*/ { int state, flags; spl_t s; if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); if (flavor == THREAD_BASIC_INFO) { register thread_basic_info_t basic_info; if (*thread_info_count < THREAD_BASIC_INFO_COUNT) return (KERN_INVALID_ARGUMENT); basic_info = (thread_basic_info_t) thread_info_out; s = splsched(); thread_lock(thread); /* fill in info */ thread_read_times(thread, &basic_info->user_time, &basic_info->system_time); /* * Update lazy-evaluated scheduler info because someone wants it. */ if (thread->sched_stamp != sched_tick) update_priority(thread); basic_info->sleep_time = 0; /* * To calculate cpu_usage, first correct for timer rate, * then for 5/8 ageing. The correction factor [3/5] is * (1/(5/8) - 1). */ basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage * TH_USAGE_SCALE) / sched_tick_interval); basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5; if (basic_info->cpu_usage > TH_USAGE_SCALE) basic_info->cpu_usage = TH_USAGE_SCALE; basic_info->policy = ((thread->sched_mode & TH_MODE_TIMESHARE)? POLICY_TIMESHARE: POLICY_RR); flags = 0; if (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor->idle_thread == thread) flags |= TH_FLAGS_IDLE; if (!thread->kernel_stack) flags |= TH_FLAGS_SWAPPED; state = 0; if (thread->state & TH_TERMINATE) state = TH_STATE_HALTED; else if (thread->state & TH_RUN) state = TH_STATE_RUNNING; else if (thread->state & TH_UNINT) state = TH_STATE_UNINTERRUPTIBLE; else if (thread->state & TH_SUSP) state = TH_STATE_STOPPED; else if (thread->state & TH_WAIT) state = TH_STATE_WAITING; basic_info->run_state = state; basic_info->flags = flags; basic_info->suspend_count = thread->user_stop_count; thread_unlock(thread); splx(s); *thread_info_count = THREAD_BASIC_INFO_COUNT; return (KERN_SUCCESS); } else if (flavor == THREAD_IDENTIFIER_INFO) { register thread_identifier_info_t identifier_info; if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) return (KERN_INVALID_ARGUMENT); identifier_info = (thread_identifier_info_t) thread_info_out; s = splsched(); thread_lock(thread); identifier_info->thread_id = thread->thread_id; #if defined(__ppc__) || defined(__arm__) identifier_info->thread_handle = thread->machine.cthread_self; #else identifier_info->thread_handle = thread->machine.pcb->cthread_self; #endif if(thread->task->bsd_info) { identifier_info->dispatch_qaddr = identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info); } else { thread_unlock(thread); splx(s); return KERN_INVALID_ARGUMENT; } thread_unlock(thread); splx(s); return KERN_SUCCESS; } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) { policy_timeshare_info_t ts_info; if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) return (KERN_INVALID_ARGUMENT); ts_info = (policy_timeshare_info_t)thread_info_out; s = splsched(); thread_lock(thread); if (!(thread->sched_mode & TH_MODE_TIMESHARE)) { thread_unlock(thread); splx(s); return (KERN_INVALID_POLICY); } ts_info->depressed = (thread->sched_mode & TH_MODE_ISDEPRESSED) != 0; if (ts_info->depressed) { ts_info->base_priority = DEPRESSPRI; ts_info->depress_priority = thread->priority; } else { ts_info->base_priority = thread->priority; ts_info->depress_priority = -1; } ts_info->cur_priority = thread->sched_pri; ts_info->max_priority = thread->max_priority; thread_unlock(thread); splx(s); *thread_info_count = POLICY_TIMESHARE_INFO_COUNT; return (KERN_SUCCESS); } else if (flavor == THREAD_SCHED_FIFO_INFO) { if (*thread_info_count < POLICY_FIFO_INFO_COUNT) return (KERN_INVALID_ARGUMENT); return (KERN_INVALID_POLICY); } else if (flavor == THREAD_SCHED_RR_INFO) { policy_rr_info_t rr_info; if (*thread_info_count < POLICY_RR_INFO_COUNT) return (KERN_INVALID_ARGUMENT); rr_info = (policy_rr_info_t) thread_info_out; s = splsched(); thread_lock(thread); if (thread->sched_mode & TH_MODE_TIMESHARE) { thread_unlock(thread); splx(s); return (KERN_INVALID_POLICY); } rr_info->depressed = (thread->sched_mode & TH_MODE_ISDEPRESSED) != 0; if (rr_info->depressed) { rr_info->base_priority = DEPRESSPRI; rr_info->depress_priority = thread->priority; } else { rr_info->base_priority = thread->priority; rr_info->depress_priority = -1; } rr_info->max_priority = thread->max_priority; rr_info->quantum = std_quantum_us / 1000; thread_unlock(thread); splx(s); *thread_info_count = POLICY_RR_INFO_COUNT; return (KERN_SUCCESS); } return (KERN_INVALID_ARGUMENT); }
/* * Routine: ipc_mqueue_post * Purpose: * Post a message to a waiting receiver or enqueue it. If a * receiver is waiting, we can release our reserved space in * the message queue. * * Conditions: * If we need to queue, our space in the message queue is reserved. */ void ipc_mqueue_post( register ipc_mqueue_t mqueue, register ipc_kmsg_t kmsg) { spl_t s; /* * While the msg queue is locked, we have control of the * kmsg, so the ref in it for the port is still good. * * Check for a receiver for the message. */ s = splsched(); imq_lock(mqueue); for (;;) { wait_queue_t waitq = &mqueue->imq_wait_queue; thread_t receiver; mach_msg_size_t msize; receiver = wait_queue_wakeup64_identity_locked( waitq, IPC_MQUEUE_RECEIVE, THREAD_AWAKENED, FALSE); /* waitq still locked, thread locked */ if (receiver == THREAD_NULL) { /* * no receivers; queue kmsg */ assert(mqueue->imq_msgcount > 0); ipc_kmsg_enqueue_macro(&mqueue->imq_messages, kmsg); break; } /* * If the receiver waited with a facility not directly * related to Mach messaging, then it isn't prepared to get * handed the message directly. Just set it running, and * go look for another thread that can. */ if (receiver->ith_state != MACH_RCV_IN_PROGRESS) { thread_unlock(receiver); continue; } /* * We found a waiting thread. * If the message is too large or the scatter list is too small * the thread we wake up will get that as its status. */ msize = ipc_kmsg_copyout_size(kmsg, receiver->map); if (receiver->ith_msize < (msize + REQUESTED_TRAILER_SIZE(thread_is_64bit(receiver), receiver->ith_option))) { receiver->ith_msize = msize; receiver->ith_state = MACH_RCV_TOO_LARGE; } else { receiver->ith_state = MACH_MSG_SUCCESS; } /* * If there is no problem with the upcoming receive, or the * receiver thread didn't specifically ask for special too * large error condition, go ahead and select it anyway. */ if ((receiver->ith_state == MACH_MSG_SUCCESS) || !(receiver->ith_option & MACH_RCV_LARGE)) { receiver->ith_kmsg = kmsg; receiver->ith_seqno = mqueue->imq_seqno++; thread_unlock(receiver); /* we didn't need our reserved spot in the queue */ ipc_mqueue_release_msgcount(mqueue); break; } /* * Otherwise, this thread needs to be released to run * and handle its error without getting the message. We * need to go back and pick another one. */ receiver->ith_receiver_name = mqueue->imq_receiver_name; receiver->ith_kmsg = IKM_NULL; receiver->ith_seqno = 0; thread_unlock(receiver); } imq_unlock(mqueue); splx(s); current_task()->messages_sent++; return; }
/* * Routine: semaphore_signal_internal * * Signals the semaphore as direct. * Assumptions: * Semaphore is locked. */ kern_return_t semaphore_signal_internal( semaphore_t semaphore, thread_t thread, int options) { kern_return_t kr; spl_t spl_level; spl_level = splsched(); semaphore_lock(semaphore); if (!semaphore->active) { semaphore_unlock(semaphore); splx(spl_level); return KERN_TERMINATED; } if (thread != THREAD_NULL) { if (semaphore->count < 0) { kr = wait_queue_wakeup64_thread_locked( &semaphore->wait_queue, SEMAPHORE_EVENT, thread, THREAD_AWAKENED, TRUE); /* unlock? */ } else { semaphore_unlock(semaphore); kr = KERN_NOT_WAITING; } splx(spl_level); return kr; } if (options & SEMAPHORE_SIGNAL_ALL) { int old_count = semaphore->count; if (old_count < 0) { semaphore->count = 0; /* always reset */ kr = wait_queue_wakeup64_all_locked( &semaphore->wait_queue, SEMAPHORE_EVENT, THREAD_AWAKENED, TRUE); /* unlock? */ } else { if (options & SEMAPHORE_SIGNAL_PREPOST) semaphore->count++; semaphore_unlock(semaphore); kr = KERN_SUCCESS; } splx(spl_level); return kr; } if (semaphore->count < 0) { if (wait_queue_wakeup64_one_locked( &semaphore->wait_queue, SEMAPHORE_EVENT, THREAD_AWAKENED, FALSE) == KERN_SUCCESS) { semaphore_unlock(semaphore); splx(spl_level); return KERN_SUCCESS; } else semaphore->count = 0; /* all waiters gone */ } if (options & SEMAPHORE_SIGNAL_PREPOST) { semaphore->count++; } semaphore_unlock(semaphore); splx(spl_level); return KERN_NOT_WAITING; }
/* * ROUTINE: ulock_release_internal [internal] * * Releases the ulock. * If any threads are blocked waiting for the ulock, one is woken-up. * */ kern_return_t ulock_release_internal (ulock_t ulock, thread_t thread) { lock_set_t lock_set; if ((lock_set = ulock->lock_set) == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != thread) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If we have a hint that threads might be waiting, * try to transfer the lock ownership to a waiting thread * and wake it up. */ if (ulock->blocked) { wait_queue_t wq = &ulock->wait_queue; thread_t wqthread; spl_t s; s = splsched(); wait_queue_lock(wq); wqthread = wait_queue_wakeup64_identity_locked(wq, LOCK_SET_EVENT, THREAD_AWAKENED, TRUE); /* wait_queue now unlocked, thread locked */ if (wqthread != THREAD_NULL) { thread_unlock(wqthread); splx(s); /* * Transfer ulock ownership * from the current thread to the acquisition thread. */ ulock_ownership_clear(ulock); ulock_ownership_set(ulock, wqthread); ulock_unlock(ulock); return KERN_SUCCESS; } else { ulock->blocked = FALSE; splx(s); } } /* * Disown ulock */ ulock_ownership_clear(ulock); ulock_unlock(ulock); return KERN_SUCCESS; }
/* * Now running in a thread. Create the rest of the kernel threads * and the bootstrap task. */ void start_kernel_threads(void) { register int i; /* * Create the idle threads and the other * service threads. */ for (i = 0; i < NCPUS; i++) { if (machine_slot[i].is_cpu) { thread_t th; spl_t s; processor_t processor = cpu_to_processor(i); (void) thread_create_at(kernel_task, &th, idle_thread); s=splsched(); thread_lock(th); thread_bind_locked(th, processor); processor->idle_thread = th; /*(void) thread_resume(th->top_act);*/ th->state |= TH_RUN; thread_setrun( th, TRUE, TAIL_Q); thread_unlock( th ); splx(s); } } (void) kernel_thread(kernel_task, reaper_thread, (char *) 0); #if THREAD_SWAPPER (void) kernel_thread(kernel_task, swapin_thread, (char *) 0); (void) kernel_thread(kernel_task, swapout_thread, (char *) 0); #endif /* THREAD_SWAPPER */ #if TASK_SWAPPER if (task_swap_on) { (void) kernel_thread(kernel_task, task_swapper, (char *) 0); (void) kernel_thread(kernel_task, task_swap_swapout_thread, (char *) 0); } #endif /* TASK_SWAPPER */ (void) kernel_thread(kernel_task, sched_thread, (char *) 0); (void) kernel_thread(kernel_task, timeout_thread, (char *) 0); #if NORMA_VM (void) kernel_thread(kernel_task, vm_object_thread, (char *) 0); #endif /* NORMA_VM */ /* * Create the clock service. */ clock_service_create(); /* * Create the device service. */ device_service_create(); /* * Initialize distributed services, starting * with distributed IPC and progressing to any * services layered on top of that. * * This stub exists even in non-NORMA systems. */ norma_bootstrap(); /* * Initialize any testing services blocking the main kernel * thread so that the in-kernel tests run without interference * from other boot time activities. We will resume this thread * in kernel_test_thread(). */ #if KERNEL_TEST /* * Initialize the lock that will be used to guard * variables that will be used in the test synchronization * scheme. */ simple_lock_init(&kernel_test_lock, ETAP_MISC_KERNEL_TEST); #if PARAGON860 { char *s; unsigned int firstnode; /* * Only start up loopback tests on boot node. */ if ((s = (char *) getbootenv("BOOT_FIRST_NODE")) == 0) panic("startup"); firstnode = atoi(s); (void) kernel_thread(kernel_task, kernel_test_thread, (char * )(dipc_node_self() == (node_name) firstnode)); } #else /* PARAGON860 */ (void) kernel_thread(kernel_task, kernel_test_thread, (char *) 0); #endif /* PARAGON860 */ { /* * The synchronization scheme uses a simple lock, two * booleans and the wakeup event. The wakeup event will * be posted by kernel_test_thread(). */ spl_t s; s = splsched(); simple_lock(&kernel_test_lock); while(!kernel_test_thread_sync_done){ assert_wait((event_t) &start_kernel_threads, FALSE); start_kernel_threads_blocked = TRUE; simple_unlock(&kernel_test_lock); splx(s); thread_block((void (*)(void)) 0); s = splsched(); simple_lock(&kernel_test_lock); start_kernel_threads_blocked = FALSE; } kernel_test_thread_sync_done = FALSE; /* Reset for next use */ simple_unlock(&kernel_test_lock); splx(s); } #endif /* KERNEL_TEST */ /* * Start the user bootstrap. */ bootstrap_create(); #if XPR_DEBUG xprinit(); /* XXX */ #endif /* XPR_DEBUG */ #if NCPUS > 1 /* * Create the shutdown thread. */ (void) kernel_thread(kernel_task, action_thread, (char *) 0); /* * Allow other CPUs to run. * * (this must be last, to allow bootstrap_create to fiddle with * its child thread before some cpu tries to run it) */ start_other_cpus(); #endif /* NCPUS > 1 */ /* * Become the pageout daemon. */ (void) spllo(); vm_pageout(); /*NOTREACHED*/ }
/* * Routine: lck_mtx_lock_wait * * Invoked in order to wait on contention. * * Called with the interlock locked and * returns it unlocked. */ void lck_mtx_lock_wait ( lck_mtx_t *lck, thread_t holder) { thread_t self = current_thread(); lck_mtx_t *mutex; integer_t priority; spl_t s = splsched(); #if CONFIG_DTRACE uint64_t sleep_start = 0; if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) { sleep_start = mach_absolute_time(); } #endif if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) mutex = lck; else mutex = &lck->lck_mtx_ptr->lck_mtx; KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0); priority = self->sched_pri; if (priority < self->priority) priority = self->priority; if (priority < BASEPRI_DEFAULT) priority = BASEPRI_DEFAULT; thread_lock(holder); if (mutex->lck_mtx_pri == 0) holder->promotions++; holder->sched_mode |= TH_MODE_PROMOTED; if ( mutex->lck_mtx_pri < priority && holder->sched_pri < priority ) { KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, holder->sched_pri, priority, (int)holder, (int)lck, 0); set_sched_pri(holder, priority); } thread_unlock(holder); splx(s); if (mutex->lck_mtx_pri < priority) mutex->lck_mtx_pri = priority; if (self->pending_promoter[self->pending_promoter_index] == NULL) { self->pending_promoter[self->pending_promoter_index] = mutex; mutex->lck_mtx_waiters++; } else if (self->pending_promoter[self->pending_promoter_index] != mutex) { self->pending_promoter[++self->pending_promoter_index] = mutex; mutex->lck_mtx_waiters++; } assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT); lck_mtx_ilk_unlock(mutex); thread_block(THREAD_CONTINUE_NULL); KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); #if CONFIG_DTRACE /* * Record the Dtrace lockstat probe for blocking, block time * measured from when we were entered. */ if (sleep_start) { if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) { LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck, mach_absolute_time() - sleep_start); } else { LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck, mach_absolute_time() - sleep_start); } } #endif }
/* * Process an AST_SWAPOUT. */ void swapout_ast() { spl_t s; thread_act_t act; thread_t thread; act = current_act(); /* * Task is being swapped out. First mark it as suspended * and halted, then call thread_swapout_enqueue to put * the thread on the queue for task_swap_swapout_threads * to swap out the thread. */ /* * Don't swap unswappable threads */ thread = act_lock_thread(act); s = splsched(); if (thread) thread_lock(thread); if ((act->ast & AST_SWAPOUT) == 0) { /* * Race with task_swapin. Abort swapout. */ task_swap_ast_aborted++; /* not locked XXX */ if (thread) thread_unlock(thread); splx(s); act_unlock_thread(act); } else if (act->swap_state == TH_SW_IN) { /* * Mark swap_state as TH_SW_TASK_SWAPPING to avoid * race with thread swapper, which will only * swap thread if swap_state is TH_SW_IN. * This way, the thread can only be swapped by * the task swapping mechanism. */ act->swap_state |= TH_SW_TASK_SWAPPING; /* assert(act->suspend_count == 0); XXX ? */ if (thread) thread_unlock(thread); if (act->suspend_count++ == 0) /* inline thread_hold */ install_special_handler(act); /* self->state |= TH_HALTED; */ thread_ast_clear(act, AST_SWAPOUT); /* * Initialize the swap_queue fields to allow an extra * queue_remove() in task_swapin if we lose the race * (task_swapin can be called before we complete * thread_swapout_enqueue). */ queue_init((queue_t) &act->swap_queue); splx(s); act_unlock_thread(act); /* this must be called at normal interrupt level */ thread_swapout_enqueue(act); } else { /* thread isn't swappable; continue running */ assert(act->swap_state == TH_SW_UNSWAPPABLE); if (thread) thread_unlock(thread); thread_ast_clear(act, AST_SWAPOUT); splx(s); act_unlock_thread(act); } }
/* * Routine: semaphore_signal_internal * * Signals the semaphore as direct. * Assumptions: * Semaphore is locked. */ kern_return_t semaphore_signal_internal( semaphore_t semaphore, thread_t thread, int options) { kern_return_t kr; spl_t spl_level; spl_level = splsched(); semaphore_lock(semaphore); if (!semaphore->active) { semaphore_unlock(semaphore); splx(spl_level); return KERN_TERMINATED; } if (thread != THREAD_NULL) { if (semaphore->count < 0) { kr = waitq_wakeup64_thread_locked( &semaphore->waitq, SEMAPHORE_EVENT, thread, THREAD_AWAKENED, WAITQ_UNLOCK); /* waitq/semaphore is unlocked */ } else { kr = KERN_NOT_WAITING; semaphore_unlock(semaphore); } splx(spl_level); return kr; } if (options & SEMAPHORE_SIGNAL_ALL) { int old_count = semaphore->count; kr = KERN_NOT_WAITING; if (old_count < 0) { semaphore->count = 0; /* always reset */ kr = waitq_wakeup64_all_locked( &semaphore->waitq, SEMAPHORE_EVENT, THREAD_AWAKENED, NULL, WAITQ_ALL_PRIORITIES, WAITQ_UNLOCK); /* waitq / semaphore is unlocked */ } else { if (options & SEMAPHORE_SIGNAL_PREPOST) semaphore->count++; kr = KERN_SUCCESS; semaphore_unlock(semaphore); } splx(spl_level); return kr; } if (semaphore->count < 0) { kr = waitq_wakeup64_one_locked( &semaphore->waitq, SEMAPHORE_EVENT, THREAD_AWAKENED, NULL, WAITQ_ALL_PRIORITIES, WAITQ_KEEP_LOCKED); if (kr == KERN_SUCCESS) { semaphore_unlock(semaphore); splx(spl_level); return KERN_SUCCESS; } else { semaphore->count = 0; /* all waiters gone */ } } if (options & SEMAPHORE_SIGNAL_PREPOST) { semaphore->count++; } semaphore_unlock(semaphore); splx(spl_level); return KERN_NOT_WAITING; }
kern_return_t processor_shutdown( processor_t processor) { processor_set_t pset; spl_t s; s = splsched(); pset = processor->processor_set; pset_lock(pset); if (processor->state == PROCESSOR_OFF_LINE) { /* * Success if already shutdown. */ pset_unlock(pset); splx(s); return (KERN_SUCCESS); } if (processor->state == PROCESSOR_START) { /* * Failure if currently being started. */ pset_unlock(pset); splx(s); return (KERN_FAILURE); } /* * If the processor is dispatching, let it finish. */ while (processor->state == PROCESSOR_DISPATCHING) { pset_unlock(pset); splx(s); delay(1); s = splsched(); pset_lock(pset); } /* * Success if already being shutdown. */ if (processor->state == PROCESSOR_SHUTDOWN) { pset_unlock(pset); splx(s); return (KERN_SUCCESS); } if (processor->state == PROCESSOR_IDLE) remqueue((queue_entry_t)processor); else if (processor->state == PROCESSOR_RUNNING) remqueue((queue_entry_t)processor); processor->state = PROCESSOR_SHUTDOWN; pset_unlock(pset); processor_doshutdown(processor); splx(s); cpu_exit_wait(processor->cpu_id); return (KERN_SUCCESS); }
kern_return_t mach_port_get_attributes( ipc_space_t space, mach_port_name_t name, int flavor, mach_port_info_t info, mach_msg_type_number_t *count) { ipc_port_t port; kern_return_t kr; if (space == IS_NULL) return KERN_INVALID_TASK; switch (flavor) { case MACH_PORT_LIMITS_INFO: { mach_port_limits_t *lp = (mach_port_limits_t *)info; if (*count < MACH_PORT_LIMITS_INFO_COUNT) return KERN_FAILURE; if (!MACH_PORT_VALID(name)) { *count = 0; break; } kr = ipc_port_translate_receive(space, name, &port); if (kr != KERN_SUCCESS) return kr; /* port is locked and active */ lp->mpl_qlimit = port->ip_messages.imq_qlimit; *count = MACH_PORT_LIMITS_INFO_COUNT; ip_unlock(port); break; } case MACH_PORT_RECEIVE_STATUS: { mach_port_status_t *statusp = (mach_port_status_t *)info; spl_t s; if (*count < MACH_PORT_RECEIVE_STATUS_COUNT) return KERN_FAILURE; if (!MACH_PORT_VALID(name)) return KERN_INVALID_RIGHT; kr = ipc_port_translate_receive(space, name, &port); if (kr != KERN_SUCCESS) return kr; /* port is locked and active */ statusp->mps_pset = port->ip_pset_count; s = splsched(); imq_lock(&port->ip_messages); statusp->mps_seqno = port->ip_messages.imq_seqno; statusp->mps_qlimit = port->ip_messages.imq_qlimit; statusp->mps_msgcount = port->ip_messages.imq_msgcount; imq_unlock(&port->ip_messages); splx(s); statusp->mps_mscount = port->ip_mscount; statusp->mps_sorights = port->ip_sorights; statusp->mps_srights = port->ip_srights > 0; statusp->mps_pdrequest = port->ip_pdrequest != IP_NULL; statusp->mps_nsrequest = port->ip_nsrequest != IP_NULL; statusp->mps_flags = 0; *count = MACH_PORT_RECEIVE_STATUS_COUNT; ip_unlock(port); break; } case MACH_PORT_DNREQUESTS_SIZE: { ipc_port_request_t table; if (*count < MACH_PORT_DNREQUESTS_SIZE_COUNT) return KERN_FAILURE; if (!MACH_PORT_VALID(name)) { *(int *)info = 0; break; } kr = ipc_port_translate_receive(space, name, &port); if (kr != KERN_SUCCESS) return kr; /* port is locked and active */ table = port->ip_dnrequests; if (table == IPR_NULL) *(int *)info = 0; else *(int *)info = table->ipr_size->its_size; *count = MACH_PORT_DNREQUESTS_SIZE_COUNT; ip_unlock(port); break; } default: return KERN_INVALID_ARGUMENT; /*NOTREACHED*/ } return KERN_SUCCESS; }
/* * thread_switch: * * Context switch. User may supply thread hint. * * Fixed priority threads that call this get what they asked for * even if that violates priority order. */ kern_return_t thread_switch( mach_port_t thread_name, int option, mach_msg_timeout_t option_time) { thread_t cur_thread = current_thread(); processor_t myprocessor; ipc_port_t port; /* * Process option. */ switch (option) { case SWITCH_OPTION_NONE: /* * Nothing to do. */ break; case SWITCH_OPTION_DEPRESS: /* * Depress priority for given time. */ thread_depress_priority(cur_thread, option_time); break; case SWITCH_OPTION_WAIT: thread_will_wait_with_timeout(cur_thread, option_time); break; default: return(KERN_INVALID_ARGUMENT); } #ifndef MIGRATING_THREADS /* XXX thread_run defunct */ /* * Check and act on thread hint if appropriate. */ if ((thread_name != 0) && (ipc_port_translate_send(cur_thread->task->itk_space, thread_name, &port) == KERN_SUCCESS)) { /* port is locked, but it might not be active */ /* * Get corresponding thread. */ if (ip_active(port) && (ip_kotype(port) == IKOT_THREAD)) { thread_t thread; spl_t s; thread = (thread_t) port->ip_kobject; /* * Check if the thread is in the right pset. Then * pull it off its run queue. If it * doesn't come, then it's not eligible. */ s = splsched(); thread_lock(thread); if ((thread->processor_set == cur_thread->processor_set) && (rem_runq(thread) != RUN_QUEUE_NULL)) { /* * Hah, got it!! */ thread_unlock(thread); (void) splx(s); ip_unlock(port); /* XXX thread might disappear on us now? */ #if MACH_FIXPRI if (thread->policy == POLICY_FIXEDPRI) { myprocessor = current_processor(); myprocessor->quantum = thread->sched_data; myprocessor->first_quantum = TRUE; } #endif /* MACH_FIXPRI */ counter(c_thread_switch_handoff++); thread_run(thread_switch_continue, thread); /* * Restore depressed priority */ if (cur_thread->depress_priority >= 0) (void) thread_depress_abort(cur_thread); return(KERN_SUCCESS); } thread_unlock(thread); (void) splx(s); } ip_unlock(port); } #endif /* not MIGRATING_THREADS */ /* * No handoff hint supplied, or hint was wrong. Call thread_block() in * hopes of running something else. If nothing else is runnable, * thread_block will detect this. WARNING: thread_switch with no * option will not do anything useful if the thread calling it is the * highest priority thread (can easily happen with a collection * of timesharing threads). */ #if NCPUS > 1 myprocessor = current_processor(); if (myprocessor->processor_set->runq.count > 0 || myprocessor->runq.count > 0) #endif /* NCPUS > 1 */ { counter(c_thread_switch_block++); thread_block(thread_switch_continue); } /* * Restore depressed priority */ if (cur_thread->depress_priority >= 0) (void) thread_depress_abort(cur_thread); return(KERN_SUCCESS); }
kern_return_t thread_policy_set( thread_t thread, thread_policy_flavor_t flavor, thread_policy_t policy_info, mach_msg_type_number_t count) { kern_return_t result = KERN_SUCCESS; spl_t s; if (thread == THREAD_NULL) return (KERN_INVALID_ARGUMENT); thread_mtx_lock(thread); if (!thread->active) { thread_mtx_unlock(thread); return (KERN_TERMINATED); } if (thread->static_param) { thread_mtx_unlock(thread); return (KERN_SUCCESS); } switch (flavor) { case THREAD_EXTENDED_POLICY: { boolean_t timeshare = TRUE; if (count >= THREAD_EXTENDED_POLICY_COUNT) { thread_extended_policy_t info; info = (thread_extended_policy_t)policy_info; timeshare = info->timeshare; } s = splsched(); thread_lock(thread); if (!(thread->sched_mode & TH_MODE_FAILSAFE)) { integer_t oldmode = (thread->sched_mode & TH_MODE_TIMESHARE); thread->sched_mode &= ~TH_MODE_REALTIME; if (timeshare && !oldmode) { thread->sched_mode |= TH_MODE_TIMESHARE; if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN) sched_share_incr(); } else if (!timeshare && oldmode) { thread->sched_mode &= ~TH_MODE_TIMESHARE; if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN) sched_share_decr(); } thread_recompute_priority(thread); } else { thread->safe_mode &= ~TH_MODE_REALTIME; if (timeshare) thread->safe_mode |= TH_MODE_TIMESHARE; else thread->safe_mode &= ~TH_MODE_TIMESHARE; } thread_unlock(thread); splx(s); break; } case THREAD_TIME_CONSTRAINT_POLICY: { thread_time_constraint_policy_t info; if (count < THREAD_TIME_CONSTRAINT_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_time_constraint_policy_t)policy_info; if ( info->constraint < info->computation || info->computation > max_rt_quantum || info->computation < min_rt_quantum ) { result = KERN_INVALID_ARGUMENT; break; } s = splsched(); thread_lock(thread); thread->realtime.period = info->period; thread->realtime.computation = info->computation; thread->realtime.constraint = info->constraint; thread->realtime.preemptible = info->preemptible; if (!(thread->sched_mode & TH_MODE_FAILSAFE)) { if (thread->sched_mode & TH_MODE_TIMESHARE) { thread->sched_mode &= ~TH_MODE_TIMESHARE; if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN) sched_share_decr(); } thread->sched_mode |= TH_MODE_REALTIME; thread_recompute_priority(thread); } else { thread->safe_mode &= ~TH_MODE_TIMESHARE; thread->safe_mode |= TH_MODE_REALTIME; } thread_unlock(thread); splx(s); break; } case THREAD_PRECEDENCE_POLICY: { thread_precedence_policy_t info; if (count < THREAD_PRECEDENCE_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_precedence_policy_t)policy_info; s = splsched(); thread_lock(thread); thread->importance = info->importance; thread_recompute_priority(thread); thread_unlock(thread); splx(s); break; } case THREAD_AFFINITY_POLICY: { thread_affinity_policy_t info; if (!thread_affinity_is_supported()) { result = KERN_NOT_SUPPORTED; break; } if (count < THREAD_AFFINITY_POLICY_COUNT) { result = KERN_INVALID_ARGUMENT; break; } info = (thread_affinity_policy_t) policy_info; /* * Unlock the thread mutex here and * return directly after calling thread_affinity_set(). * This is necessary for correct lock ordering because * thread_affinity_set() takes the task lock. */ thread_mtx_unlock(thread); return thread_affinity_set(thread, info->affinity_tag); } default: result = KERN_INVALID_ARGUMENT; break; } thread_mtx_unlock(thread); return (result); }
void cpu_exit(struct lwp *l) { (void) splsched(); switch_exit(l, lwp_exit2); }
/* * task_swapout: * A reference to the task must be held. * * Start swapping out a task by sending an AST_SWAPOUT to each thread. * When the threads reach a clean point, they queue themselves up on the * swapout_thread_q to be swapped out by the task_swap_swapout_thread. * The task can be swapped in at any point in this process. * * A task will not be fully swapped out (i.e. its map residence count * at zero) until all currently-swapped threads run and reach * a clean point, at which time they will be swapped again, * decrementing the swap_ast_waiting count on the task. * * Locking: no locks held upon entry and exit. * Task_lock is held throughout this function. */ kern_return_t task_swapout(task_t task) { thread_act_t thr_act; thread_t thread; queue_head_t *list; int s; task_swapout_lock(); task_lock(task); /* * NOTE: look into turning these into assertions if they * are invariants. */ if ((task->swap_state != TASK_SW_IN) || (!task->active)) { task_unlock(task); task_swapout_unlock(); return(KERN_FAILURE); } if (task->swap_flags & TASK_SW_ELIGIBLE) { queue_remove(&eligible_tasks, task, task_t, swapped_tasks); task->swap_flags &= ~TASK_SW_ELIGIBLE; } task_swapout_unlock(); /* set state to avoid races with task_swappable(FALSE) */ task->swap_state = TASK_SW_GOING_OUT; task->swap_rss = pmap_resident_count(task->map->pmap); task_swaprss_out += task->swap_rss; task->swap_ast_waiting = task->thr_act_count; /* * halt all threads in this task: * We don't need the thread list lock for traversal. */ list = &task->thr_acts; thr_act = (thread_act_t) queue_first(list); while (!queue_end(list, (queue_entry_t) thr_act)) { boolean_t swappable; thread_act_t ract; thread = act_lock_thread(thr_act); s = splsched(); if (!thread) swappable = (thr_act->swap_state != TH_SW_UNSWAPPABLE); else { thread_lock(thread); swappable = TRUE; for (ract = thread->top_act; ract; ract = ract->lower) if (ract->swap_state == TH_SW_UNSWAPPABLE) { swappable = FALSE; break; } } if (swappable) thread_ast_set(thr_act, AST_SWAPOUT); if (thread) thread_unlock(thread); splx(s); assert((thr_act->ast & AST_TERMINATE) == 0); act_unlock_thread(thr_act); thr_act = (thread_act_t) queue_next(&thr_act->thr_acts); } task->swap_stamp = sched_tick; task->swap_nswap++; assert((task->swap_flags&TASK_SW_WANT_IN) == 0); /* put task on the queue of swapped out tasks */ task_swapper_lock(); #if TASK_SW_DEBUG if (task_swap_debug && on_swapped_list(task)) { printf("task 0x%X already on list\n", task); Debugger(""); } #endif /* TASK_SW_DEBUG */ queue_enter(&swapped_tasks, task, task_t, swapped_tasks); tasks_swapped_out++; task_swapouts++; task_swapper_unlock(); task_unlock(task); return(KERN_SUCCESS); }
/* * Manage next request event */ void datadev_request( datadev_t ddp) { kern_return_t rc; io_req_t ior; spl_t s; s = splsched(); mutex_lock(&datadev_lock); if (ddp != (datadev_t)0) { /* * Queue current request */ queue_enter(&datadev_wait, ddp, datadev_t, dd_chain); } /* * Try to start next request */ if (queue_empty(&datadev_wait) || datadev_ior == (io_req_t)0) { /* * No request or no pending read */ mutex_unlock(&datadev_lock); splx(s); return; } /* * Extract first waiting request */ ddp = (datadev_t)queue_first(&datadev_wait); /* * Extract pending I/O request */ ior = datadev_ior; datadev_ior = (io_req_t)0; /* * Allocate read memory */ if (ior->io_count < ddp->dd_size) { /* * Return size error for this request */ mutex_unlock(&datadev_lock); splx(s); ior->io_error = D_INVALID_SIZE; } else { /* * Move waiting request from the waiting queue to the active one. */ queue_remove(&datadev_wait, ddp, datadev_t, dd_chain); queue_enter(&datadev_curr, ddp, datadev_t, dd_chain); mutex_unlock(&datadev_lock); splx(s); /* * Activate the request */ bcopy(ddp->dd_name, ior->io_data, ddp->dd_size); ddp->dd_dev = ior->io_unit; ior->io_residual = ior->io_count - ddp->dd_size; ior->io_error = D_SUCCESS; } io_completed(ior, FALSE); }
/* * task_swap_swapout_thread: [exported] * * Executes as a separate kernel thread. * Its job is to swap out threads that have been halted by AST_SWAPOUT. */ void task_swap_swapout_thread(void) { thread_act_t thr_act; thread_t thread, nthread; task_t task; int s; thread_swappable(current_act(), FALSE); stack_privilege(current_thread()); spllo(); while (TRUE) { task_swapper_lock(); while (! queue_empty(&swapout_thread_q)) { queue_remove_first(&swapout_thread_q, thr_act, thread_act_t, swap_queue); /* * If we're racing with task_swapin, we need * to make it safe for it to do remque on the * thread, so make its links point to itself. * Allowing this ugliness is cheaper than * making task_swapin search the entire queue. */ act_lock(thr_act); queue_init((queue_t) &thr_act->swap_queue); act_unlock(thr_act); task_swapper_unlock(); /* * Wait for thread's RUN bit to be deasserted. */ thread = act_lock_thread(thr_act); if (thread == THREAD_NULL) act_unlock_thread(thr_act); else { boolean_t r; thread_reference(thread); thread_hold(thr_act); act_unlock_thread(thr_act); r = thread_stop_wait(thread); nthread = act_lock_thread(thr_act); thread_release(thr_act); thread_deallocate(thread); act_unlock_thread(thr_act); if (!r || nthread != thread) { task_swapper_lock(); continue; } } task = thr_act->task; task_lock(task); /* * we can race with swapin, which would set the * state to TASK_SW_IN. */ if ((task->swap_state != TASK_SW_OUT) && (task->swap_state != TASK_SW_GOING_OUT)) { task_unlock(task); task_swapper_lock(); TASK_STATS_INCR(task_sw_race_in_won); if (thread != THREAD_NULL) thread_unstop(thread); continue; } nthread = act_lock_thread(thr_act); if (nthread != thread || thr_act->active == FALSE) { act_unlock_thread(thr_act); task_unlock(task); task_swapper_lock(); TASK_STATS_INCR(task_sw_act_inactive); if (thread != THREAD_NULL) thread_unstop(thread); continue; } s = splsched(); if (thread != THREAD_NULL) thread_lock(thread); /* * Thread cannot have been swapped out yet because * TH_SW_TASK_SWAPPING was set in AST. If task_swapin * beat us here, we either wouldn't have found it on * the queue, or the task->swap_state would have * changed. The synchronization is on the * task's swap_state and the task_lock. * The thread can't be swapped in any other way * because its task has been swapped. */ assert(thr_act->swap_state & TH_SW_TASK_SWAPPING); assert(thread == THREAD_NULL || !(thread->state & (TH_SWAPPED_OUT|TH_RUN))); assert((thr_act->swap_state & TH_SW_STATE) == TH_SW_IN); /* assert(thread->state & TH_HALTED); */ /* this also clears TH_SW_TASK_SWAPPING flag */ thr_act->swap_state = TH_SW_GOING_OUT; if (thread != THREAD_NULL) { if (thread->top_act == thr_act) { thread->state |= TH_SWAPPED_OUT; /* * Once we unlock the task, things can happen * to the thread, so make sure it's consistent * for thread_swapout. */ } thread->ref_count++; thread_unlock(thread); thread_unstop(thread); } splx(s); act_locked_act_reference(thr_act); act_unlock_thread(thr_act); task_unlock(task); thread_swapout(thr_act); /* do the work */ if (thread != THREAD_NULL) thread_deallocate(thread); act_deallocate(thr_act); task_swapper_lock(); } assert_wait((event_t)&swapout_thread_q, FALSE); task_swapper_unlock(); thread_block((void (*)(void)) 0); } }
kern_return_t lock_handoff (lock_set_t lock_set, int lock_id) { ulock_t ulock; int wait_result; if (lock_set == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; if (lock_id < 0 || lock_id >= lock_set->n_ulocks) return KERN_INVALID_ARGUMENT; retry: lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock = (ulock_t) &lock_set->ulock_list[lock_id]; ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != current_thread()) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If the accepting thread (the receiver) is already waiting * to accept the lock from the handoff thread (the sender), * then perform the hand-off now. */ if (ulock->accept_wait) { wait_queue_t wq = &ulock->wait_queue; thread_t thread; spl_t s; /* * See who the lucky devil is, if he is still there waiting. */ s = splsched(); wait_queue_lock(wq); thread = wait_queue_wakeup64_identity_locked( wq, LOCK_SET_HANDOFF, THREAD_AWAKENED, TRUE); /* wait queue unlocked, thread locked */ /* * Transfer lock ownership */ if (thread != THREAD_NULL) { /* * The thread we are transferring to will try * to take the lock on the ulock, and therefore * will wait for us complete the handoff even * through we set the thread running. */ thread_unlock(thread); splx(s); ulock_ownership_clear(ulock); ulock_ownership_set(ulock, thread); ulock->accept_wait = FALSE; ulock_unlock(ulock); return KERN_SUCCESS; } else { /* * OOPS. The accepting thread must have been aborted. * and is racing back to clear the flag that says is * waiting for an accept. He will clear it when we * release the lock, so just fall thru and wait for * the next accept thread (that's the way it is * specified). */ splx(s); } } /* * Indicate that there is a hand-off thread waiting, and then wait * for an accepting thread. */ ulock->ho_wait = TRUE; wait_result = wait_queue_assert_wait64(&ulock->wait_queue, LOCK_SET_HANDOFF, THREAD_ABORTSAFE, 0); ulock_unlock(ulock); if (wait_result == THREAD_WAITING) wait_result = thread_block(THREAD_CONTINUE_NULL); /* * If the thread was woken-up via some action other than * lock_handoff_accept or lock_set_destroy (i.e. thread_terminate), * then we need to clear the ulock's handoff state. */ switch (wait_result) { case THREAD_AWAKENED: /* * we take the ulock lock to syncronize with the * thread that is accepting ownership. */ ulock_lock(ulock); assert(ulock->holder != current_thread()); ulock_unlock(ulock); return KERN_SUCCESS; case THREAD_INTERRUPTED: ulock_lock(ulock); assert(ulock->holder == current_thread()); ulock->ho_wait = FALSE; ulock_unlock(ulock); return KERN_ABORTED; case THREAD_RESTART: goto retry; } panic("lock_handoff"); return KERN_FAILURE; }
kern_return_t task_swapin(task_t task, boolean_t make_unswappable) { register queue_head_t *list; register thread_act_t thr_act, next; thread_t thread; int s; boolean_t swappable = TRUE; task_lock(task); switch (task->swap_state) { case TASK_SW_OUT: { vm_map_t map = task->map; /* * Task has made it all the way out, which means * that vm_map_res_deallocate has been done; set * state to TASK_SW_COMING_IN, then bring map * back in. We could actually be racing with * the thread_swapout_enqueue, which does the * vm_map_res_deallocate, but that race is covered. */ task->swap_state = TASK_SW_COMING_IN; assert(task->swap_ast_waiting == 0); assert(map->res_count >= 0); task_unlock(task); mutex_lock(&map->s_lock); vm_map_res_reference(map); mutex_unlock(&map->s_lock); task_lock(task); assert(task->swap_state == TASK_SW_COMING_IN); } break; case TASK_SW_GOING_OUT: /* * Task isn't all the way out yet. There is * still at least one thread not swapped, and * vm_map_res_deallocate has not been done. */ task->swap_state = TASK_SW_COMING_IN; assert(task->swap_ast_waiting > 0 || (task->swap_ast_waiting == 0 && task->thr_act_count == 0)); assert(task->map->res_count > 0); TASK_STATS_INCR(task_sw_race_going_out); break; case TASK_SW_IN: assert(task->map->res_count > 0); #if TASK_SW_DEBUG task_swapper_lock(); if (task_swap_debug && on_swapped_list(task)) { printf("task 0x%X on list, state is SW_IN\n", task); Debugger(""); } task_swapper_unlock(); #endif /* TASK_SW_DEBUG */ TASK_STATS_INCR(task_sw_race_in); if (make_unswappable) { task->swap_state = TASK_SW_UNSWAPPABLE; task_unlock(task); task_swapout_ineligible(task); } else task_unlock(task); return(KERN_SUCCESS); case TASK_SW_COMING_IN: /* * Raced with another task_swapin and lost; * wait for other one to complete first */ assert(task->map->res_count >= 0); /* * set MAKE_UNSWAPPABLE so that whoever is swapping * the task in will make it unswappable, and return */ if (make_unswappable) task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE; task->swap_flags |= TASK_SW_WANT_IN; assert_wait((event_t)&task->swap_state, FALSE); task_unlock(task); thread_block((void (*)(void)) 0); TASK_STATS_INCR(task_sw_race_coming_in); return(KERN_SUCCESS); case TASK_SW_UNSWAPPABLE: /* * This can happen, since task_terminate * unconditionally calls task_swapin. */ task_unlock(task); return(KERN_SUCCESS); default: panic("task_swapin bad state"); break; } if (make_unswappable) task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE; assert(task->swap_state == TASK_SW_COMING_IN); task_swapper_lock(); #if TASK_SW_DEBUG if (task_swap_debug && !on_swapped_list(task)) { printf("task 0x%X not on list\n", task); Debugger(""); } #endif /* TASK_SW_DEBUG */ queue_remove(&swapped_tasks, task, task_t, swapped_tasks); tasks_swapped_out--; task_swapins++; task_swapper_unlock(); /* * Iterate through all threads for this task and * release them, as required. They may not have been swapped * out yet. The task remains locked throughout. */ list = &task->thr_acts; thr_act = (thread_act_t) queue_first(list); while (!queue_end(list, (queue_entry_t) thr_act)) { boolean_t need_to_release; next = (thread_act_t) queue_next(&thr_act->thr_acts); /* * Keep task_swapper_lock across thread handling * to synchronize with task_swap_swapout_thread */ task_swapper_lock(); thread = act_lock_thread(thr_act); s = splsched(); if (thr_act->ast & AST_SWAPOUT) { /* thread hasn't gotten the AST yet, just clear it */ thread_ast_clear(thr_act, AST_SWAPOUT); need_to_release = FALSE; TASK_STATS_INCR(task_sw_before_ast); splx(s); act_unlock_thread(thr_act); } else { /* * If AST_SWAPOUT was cleared, then thread_hold, * or equivalent was done. */ need_to_release = TRUE; /* * Thread has hit AST, but it may not have * been dequeued yet, so we need to check. * NOTE: the thread may have been dequeued, but * has not yet been swapped (the task_swapper_lock * has been dropped, but the thread is not yet * locked), and the TH_SW_TASK_SWAPPING flag may * not have been cleared. In this case, we will do * an extra remque, which the task_swap_swapout_thread * has made safe, and clear the flag, which is also * checked by the t_s_s_t before doing the swapout. */ if (thread) thread_lock(thread); if (thr_act->swap_state & TH_SW_TASK_SWAPPING) { /* * hasn't yet been dequeued for swapout, * so clear flags and dequeue it first. */ thr_act->swap_state &= ~TH_SW_TASK_SWAPPING; assert(thr_act->thread == THREAD_NULL || !(thr_act->thread->state & TH_SWAPPED_OUT)); queue_remove(&swapout_thread_q, thr_act, thread_act_t, swap_queue); TASK_STATS_INCR(task_sw_before_swap); } else { TASK_STATS_INCR(task_sw_after_swap); /* * It's possible that the thread was * made unswappable before hitting the * AST, in which case it's still running. */ if (thr_act->swap_state == TH_SW_UNSWAPPABLE) { need_to_release = FALSE; TASK_STATS_INCR(task_sw_unswappable); } } if (thread) thread_unlock(thread); splx(s); act_unlock_thread(thr_act); } task_swapper_unlock(); /* * thread_release will swap in the thread if it's been * swapped out. */ if (need_to_release) { act_lock_thread(thr_act); thread_release(thr_act); act_unlock_thread(thr_act); } thr_act = next; } if (task->swap_flags & TASK_SW_MAKE_UNSWAPPABLE) { task->swap_flags &= ~TASK_SW_MAKE_UNSWAPPABLE; task->swap_state = TASK_SW_UNSWAPPABLE; swappable = FALSE; } else { task->swap_state = TASK_SW_IN; } task_swaprss_in += pmap_resident_count(task->map->pmap); task_swap_total_time += sched_tick - task->swap_stamp; /* note when task came back in */ task->swap_stamp = sched_tick; if (task->swap_flags & TASK_SW_WANT_IN) { task->swap_flags &= ~TASK_SW_WANT_IN; thread_wakeup((event_t)&task->swap_state); } assert((task->swap_flags & TASK_SW_ELIGIBLE) == 0); task_unlock(task); #if TASK_SW_DEBUG task_swapper_lock(); if (task_swap_debug && on_swapped_list(task)) { printf("task 0x%X on list at end of swap in\n", task); Debugger(""); } task_swapper_unlock(); #endif /* TASK_SW_DEBUG */ /* * Make the task eligible to be swapped again */ if (swappable) task_swapout_eligible(task); return(KERN_SUCCESS); }
/* * Routine: ipc_mqueue_add * Purpose: * Associate the portset's mqueue with the port's mqueue. * This has to be done so that posting the port will wakeup * a portset waiter. If there are waiters on the portset * mqueue and messages on the port mqueue, try to match them * up now. * Conditions: * May block. */ kern_return_t ipc_mqueue_add( ipc_mqueue_t port_mqueue, ipc_mqueue_t set_mqueue, wait_queue_link_t wql) { wait_queue_t port_waitq = &port_mqueue->imq_wait_queue; wait_queue_set_t set_waitq = &set_mqueue->imq_set_queue; ipc_kmsg_queue_t kmsgq; ipc_kmsg_t kmsg, next; kern_return_t kr; spl_t s; kr = wait_queue_link_noalloc(port_waitq, set_waitq, wql); if (kr != KERN_SUCCESS) return kr; /* * Now that the set has been added to the port, there may be * messages queued on the port and threads waiting on the set * waitq. Lets get them together. */ s = splsched(); imq_lock(port_mqueue); kmsgq = &port_mqueue->imq_messages; for (kmsg = ipc_kmsg_queue_first(kmsgq); kmsg != IKM_NULL; kmsg = next) { next = ipc_kmsg_queue_next(kmsgq, kmsg); for (;;) { thread_t th; mach_msg_size_t msize; th = wait_queue_wakeup64_identity_locked( port_waitq, IPC_MQUEUE_RECEIVE, THREAD_AWAKENED, FALSE); /* waitq/mqueue still locked, thread locked */ if (th == THREAD_NULL) goto leave; /* * If the receiver waited with a facility not directly * related to Mach messaging, then it isn't prepared to get * handed the message directly. Just set it running, and * go look for another thread that can. */ if (th->ith_state != MACH_RCV_IN_PROGRESS) { thread_unlock(th); continue; } /* * Found a receiver. see if they can handle the message * correctly (the message is not too large for them, or * they didn't care to be informed that the message was * too large). If they can't handle it, take them off * the list and let them go back and figure it out and * just move onto the next. */ msize = ipc_kmsg_copyout_size(kmsg, th->map); if (th->ith_msize < (msize + REQUESTED_TRAILER_SIZE(thread_is_64bit(th), th->ith_option))) { th->ith_state = MACH_RCV_TOO_LARGE; th->ith_msize = msize; if (th->ith_option & MACH_RCV_LARGE) { /* * let him go without message */ th->ith_receiver_name = port_mqueue->imq_receiver_name; th->ith_kmsg = IKM_NULL; th->ith_seqno = 0; thread_unlock(th); continue; /* find another thread */ } } else { th->ith_state = MACH_MSG_SUCCESS; } /* * This thread is going to take this message, * so give it to him. */ ipc_kmsg_rmqueue(kmsgq, kmsg); ipc_mqueue_release_msgcount(port_mqueue); th->ith_kmsg = kmsg; th->ith_seqno = port_mqueue->imq_seqno++; thread_unlock(th); break; /* go to next message */ } } leave: imq_unlock(port_mqueue); splx(s); return KERN_SUCCESS; }
task_t pick_outtask(void) { register task_t task; register task_t target_task = TASK_NULL; unsigned long task_rss; unsigned long target_rss = 0; boolean_t wired; boolean_t active; int nactive = 0; task_swapout_lock(); if (queue_empty(&eligible_tasks)) { /* not likely to happen */ task_swapout_unlock(); return(TASK_NULL); } task = (task_t)queue_first(&eligible_tasks); while (!queue_end(&eligible_tasks, (queue_entry_t)task)) { int s; register thread_act_t thr_act; thread_t th; task_lock(task); #if MACH_RT /* * Don't swap real-time tasks. * XXX Should we enforce that or can we let really critical * tasks use task_swappable() to make sure they never end up * n the eligible list ? */ if (task->policy & POLICYCLASS_FIXEDPRI) { goto tryagain; } #endif /* MACH_RT */ if (!task->active) { TASK_STATS_INCR(inactive_task_count); goto tryagain; } if (task->res_act_count == 0) { TASK_STATS_INCR(empty_task_count); goto tryagain; } assert(!queue_empty(&task->thr_acts)); thr_act = (thread_act_t)queue_first(&task->thr_acts); active = FALSE; th = act_lock_thread(thr_act); s = splsched(); if (th != THREAD_NULL) thread_lock(th); if ((th == THREAD_NULL) || (th->state == TH_RUN) || (th->state & TH_WAIT)) { /* * thread is "active": either runnable * or sleeping. Count it and examine * it further below. */ nactive++; active = TRUE; } if (th != THREAD_NULL) thread_unlock(th); splx(s); act_unlock_thread(thr_act); if (active && (task->swap_state == TASK_SW_IN) && ((sched_tick - task->swap_stamp) > min_res_time)) { long rescount = pmap_resident_count(task->map->pmap); /* * thread must be "active", task must be swapped * in and resident for at least min_res_time */ #if 0 /* DEBUG Test round-robin strategy. Picking biggest task could cause extreme * unfairness to such large interactive programs as xterm. Instead, pick the * first task that has any pages resident: */ if (rescount > 1) { task->ref_count++; target_task = task; task_unlock(task); task_swapout_unlock(); return(target_task); } #else if (rescount > target_rss) { /* * task is not swapped, and it has the * largest rss seen so far. */ task->ref_count++; target_rss = rescount; assert(target_task != task); if (target_task != TASK_NULL) task_deallocate(target_task); target_task = task; } #endif } tryagain: task_unlock(task); task = (task_t)queue_next(&task->swapped_tasks); } task_swapout_unlock(); /* only swap out if there are at least min_active_tasks */ if (nactive < min_active_tasks) { if (target_task != TASK_NULL) { task_deallocate(target_task); target_task = TASK_NULL; } } return(target_task); }
wait_result_t ipc_mqueue_receive_on_thread( ipc_mqueue_t mqueue, mach_msg_option_t option, mach_msg_size_t max_size, mach_msg_timeout_t rcv_timeout, int interruptible, thread_t thread) { ipc_kmsg_queue_t kmsgs; wait_result_t wresult; uint64_t deadline; spl_t s; s = splsched(); imq_lock(mqueue); if (imq_is_set(mqueue)) { queue_t q; q = &mqueue->imq_preposts; /* * If we are waiting on a portset mqueue, we need to see if * any of the member ports have work for us. Ports that * have (or recently had) messages will be linked in the * prepost queue for the portset. By holding the portset's * mqueue lock during the search, we tie up any attempts by * mqueue_deliver or portset membership changes that may * cross our path. */ search_set: while(!queue_empty(q)) { wait_queue_link_t wql; ipc_mqueue_t port_mq; queue_remove_first(q, wql, wait_queue_link_t, wql_preposts); assert(!wql_is_preposted(wql)); /* * This is a lock order violation, so we have to do it * "softly," putting the link back on the prepost list * if it fails (at the tail is fine since the order of * handling messages from different sources in a set is * not guaranteed and we'd like to skip to the next source * if one is available). */ port_mq = (ipc_mqueue_t)wql->wql_queue; if (!imq_lock_try(port_mq)) { queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); splx(s); mutex_pause(0); s = splsched(); imq_lock(mqueue); goto search_set; /* start again at beginning - SMP */ } /* * If there are no messages on this queue, just skip it * (we already removed the link from the set's prepost queue). */ kmsgs = &port_mq->imq_messages; if (ipc_kmsg_queue_first(kmsgs) == IKM_NULL) { imq_unlock(port_mq); continue; } /* * There are messages, so reinsert the link back * at the tail of the preposted queue (for fairness) * while we still have the portset mqueue locked. */ queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); /* * Continue on to handling the message with just * the port mqueue locked. */ ipc_mqueue_select_on_thread(port_mq, option, max_size, thread); imq_unlock(port_mq); splx(s); return THREAD_NOT_WAITING; } } else { /* * Receive on a single port. Just try to get the messages. */ kmsgs = &mqueue->imq_messages; if (ipc_kmsg_queue_first(kmsgs) != IKM_NULL) { ipc_mqueue_select_on_thread(mqueue, option, max_size, thread); imq_unlock(mqueue); splx(s); return THREAD_NOT_WAITING; } } /* * Looks like we'll have to block. The mqueue we will * block on (whether the set's or the local port's) is * still locked. */ if (option & MACH_RCV_TIMEOUT) { if (rcv_timeout == 0) { imq_unlock(mqueue); splx(s); thread->ith_state = MACH_RCV_TIMED_OUT; return THREAD_NOT_WAITING; } } thread_lock(thread); thread->ith_state = MACH_RCV_IN_PROGRESS; thread->ith_option = option; thread->ith_msize = max_size; if (option & MACH_RCV_TIMEOUT) clock_interval_to_deadline(rcv_timeout, 1000*NSEC_PER_USEC, &deadline); else deadline = 0; wresult = wait_queue_assert_wait64_locked(&mqueue->imq_wait_queue, IPC_MQUEUE_RECEIVE, interruptible, TIMEOUT_URGENCY_USER_NORMAL, deadline, 0, thread); /* preposts should be detected above, not here */ if (wresult == THREAD_AWAKENED) panic("ipc_mqueue_receive_on_thread: sleep walking"); thread_unlock(thread); imq_unlock(mqueue); splx(s); return wresult; }
/* * Routine: lck_mtx_lock_wait * * Invoked in order to wait on contention. * * Called with the interlock locked and * returns it unlocked. */ void lck_mtx_lock_wait ( lck_mtx_t *lck, thread_t holder) { thread_t self = current_thread(); lck_mtx_t *mutex; __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); __kdebug_only uintptr_t trace_holder = VM_KERNEL_UNSLIDE_OR_PERM(holder); integer_t priority; spl_t s = splsched(); #if CONFIG_DTRACE uint64_t sleep_start = 0; if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) { sleep_start = mach_absolute_time(); } #endif if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) mutex = lck; else mutex = &lck->lck_mtx_ptr->lck_mtx; KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, trace_lck, trace_holder, 0, 0, 0); priority = self->sched_pri; if (priority < self->base_pri) priority = self->base_pri; if (priority < BASEPRI_DEFAULT) priority = BASEPRI_DEFAULT; /* Do not promote past promotion ceiling */ priority = MIN(priority, MAXPRI_PROMOTE); thread_lock(holder); if (mutex->lck_mtx_pri == 0) holder->promotions++; holder->sched_flags |= TH_SFLAG_PROMOTED; if (mutex->lck_mtx_pri < priority && holder->sched_pri < priority) { KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, holder->sched_pri, priority, trace_holder, trace_lck, 0); set_sched_pri(holder, priority); } thread_unlock(holder); splx(s); if (mutex->lck_mtx_pri < priority) mutex->lck_mtx_pri = priority; if (self->pending_promoter[self->pending_promoter_index] == NULL) { self->pending_promoter[self->pending_promoter_index] = mutex; mutex->lck_mtx_waiters++; } else if (self->pending_promoter[self->pending_promoter_index] != mutex) { self->pending_promoter[++self->pending_promoter_index] = mutex; mutex->lck_mtx_waiters++; } assert_wait(LCK_MTX_EVENT(mutex), THREAD_UNINT); lck_mtx_ilk_unlock(mutex); thread_block(THREAD_CONTINUE_NULL); KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); #if CONFIG_DTRACE /* * Record the Dtrace lockstat probe for blocking, block time * measured from when we were entered. */ if (sleep_start) { if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) { LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck, mach_absolute_time() - sleep_start); } else { LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck, mach_absolute_time() - sleep_start); } } #endif }
/* * Routine: semaphore_wait_internal * * Decrements the semaphore count by one. If the count is * negative after the decrement, the calling thread blocks * (possibly at a continuation and/or with a timeout). * * Assumptions: * The reference * A reference is held on the signal semaphore. */ kern_return_t semaphore_wait_internal( semaphore_t wait_semaphore, semaphore_t signal_semaphore, mach_timespec_t *wait_timep, void (*caller_cont)(kern_return_t)) { boolean_t nonblocking; int wait_result; spl_t spl_level; kern_return_t kr = KERN_ALREADY_WAITING; spl_level = splsched(); semaphore_lock(wait_semaphore); /* * Decide if we really have to wait. */ nonblocking = (wait_timep != (mach_timespec_t *)0) ? (wait_timep->tv_sec == 0 && wait_timep->tv_nsec == 0) : FALSE; if (!wait_semaphore->active) { kr = KERN_TERMINATED; } else if (wait_semaphore->count > 0) { wait_semaphore->count--; kr = KERN_SUCCESS; } else if (nonblocking) { kr = KERN_OPERATION_TIMED_OUT; } else { uint64_t abstime; thread_t self = current_thread(); wait_semaphore->count = -1; /* we don't keep an actual count */ thread_lock(self); /* * If it is a timed wait, calculate the wake up deadline. */ if (wait_timep != (mach_timespec_t *)0) { nanoseconds_to_absolutetime((uint64_t)wait_timep->tv_sec * NSEC_PER_SEC + wait_timep->tv_nsec, &abstime); clock_absolutetime_interval_to_deadline(abstime, &abstime); } else abstime = 0; (void)wait_queue_assert_wait64_locked( &wait_semaphore->wait_queue, SEMAPHORE_EVENT, THREAD_ABORTSAFE, abstime, self); thread_unlock(self); } semaphore_unlock(wait_semaphore); splx(spl_level); /* * wait_semaphore is unlocked so we are free to go ahead and * signal the signal_semaphore (if one was provided). */ if (signal_semaphore != SEMAPHORE_NULL) { kern_return_t signal_kr; /* * lock the signal semaphore reference we got and signal it. * This will NOT block (we cannot block after having asserted * our intention to wait above). */ signal_kr = semaphore_signal_internal(signal_semaphore, THREAD_NULL, SEMAPHORE_SIGNAL_PREPOST); if (signal_kr == KERN_NOT_WAITING) signal_kr = KERN_SUCCESS; else if (signal_kr == KERN_TERMINATED) { /* * Uh!Oh! The semaphore we were to signal died. * We have to get ourselves out of the wait in * case we get stuck here forever (it is assumed * that the semaphore we were posting is gating * the decision by someone else to post the * semaphore we are waiting on). People will * discover the other dead semaphore soon enough. * If we got out of the wait cleanly (someone * already posted a wakeup to us) then return that * (most important) result. Otherwise, * return the KERN_TERMINATED status. */ thread_t self = current_thread(); clear_wait(self, THREAD_INTERRUPTED); kr = semaphore_convert_wait_result(self->wait_result); if (kr == KERN_ABORTED) kr = KERN_TERMINATED; } } /* * If we had an error, or we didn't really need to wait we can * return now that we have signalled the signal semaphore. */ if (kr != KERN_ALREADY_WAITING) return kr; /* * Now, we can block. If the caller supplied a continuation * pointer of his own for after the block, block with the * appropriate semaphore continuation. Thiswill gather the * semaphore results, release references on the semaphore(s), * and then call the caller's continuation. */ if (caller_cont) { thread_t self = current_thread(); self->sth_continuation = caller_cont; self->sth_waitsemaphore = wait_semaphore; self->sth_signalsemaphore = signal_semaphore; wait_result = thread_block((thread_continue_t)semaphore_wait_continue); } else { wait_result = thread_block(THREAD_CONTINUE_NULL); } return (semaphore_convert_wait_result(wait_result)); }
/* * Routine: lck_mtx_lock_acquire * * Invoked on acquiring the mutex when there is * contention. * * Returns the current number of waiters. * * Called with the interlock locked. */ int lck_mtx_lock_acquire( lck_mtx_t *lck) { thread_t thread = current_thread(); lck_mtx_t *mutex; integer_t priority; spl_t s; __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) mutex = lck; else mutex = &lck->lck_mtx_ptr->lck_mtx; if (thread->pending_promoter[thread->pending_promoter_index] == mutex) { thread->pending_promoter[thread->pending_promoter_index] = NULL; if (thread->pending_promoter_index > 0) thread->pending_promoter_index--; mutex->lck_mtx_waiters--; } if (mutex->lck_mtx_waiters) priority = mutex->lck_mtx_pri; else { mutex->lck_mtx_pri = 0; priority = 0; } if (priority || thread->was_promoted_on_wakeup) { s = splsched(); thread_lock(thread); if (priority) { thread->promotions++; thread->sched_flags |= TH_SFLAG_PROMOTED; if (thread->sched_pri < priority) { KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, thread->sched_pri, priority, 0, trace_lck, 0); /* Do not promote past promotion ceiling */ assert(priority <= MAXPRI_PROMOTE); set_sched_pri(thread, priority); } } if (thread->was_promoted_on_wakeup) { thread->was_promoted_on_wakeup = 0; if (thread->promotions == 0) lck_mtx_clear_promoted(thread, trace_lck); } thread_unlock(thread); splx(s); } #if CONFIG_DTRACE if (lockstat_probemap[LS_LCK_MTX_LOCK_ACQUIRE] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_ACQUIRE]) { if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) { LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, lck, 0); } else { LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, lck, 0); } } #endif return (mutex->lck_mtx_waiters); }
/* * thread_terminate_self: */ void thread_terminate_self(void) { thread_t thread = current_thread(); task_t task; spl_t s; int threadcnt; DTRACE_PROC(lwp__exit); thread_mtx_lock(thread); ulock_release_all(thread); ipc_thread_disable(thread); thread_mtx_unlock(thread); s = splsched(); thread_lock(thread); /* * Cancel priority depression, wait for concurrent expirations * on other processors. */ if (thread->sched_mode & TH_MODE_ISDEPRESSED) { thread->sched_mode &= ~TH_MODE_ISDEPRESSED; if (timer_call_cancel(&thread->depress_timer)) thread->depress_timer_active--; } while (thread->depress_timer_active > 0) { thread_unlock(thread); splx(s); delay(1); s = splsched(); thread_lock(thread); } thread_sched_call(thread, NULL); thread_unlock(thread); splx(s); thread_policy_reset(thread); task = thread->task; uthread_cleanup(task, thread->uthread, task->bsd_info); threadcnt = hw_atomic_sub(&task->active_thread_count, 1); /* * If we are the last thread to terminate and the task is * associated with a BSD process, perform BSD process exit. */ if (threadcnt == 0 && task->bsd_info != NULL) proc_exit(task->bsd_info); uthread_cred_free(thread->uthread); s = splsched(); thread_lock(thread); /* * Cancel wait timer, and wait for * concurrent expirations. */ if (thread->wait_timer_is_set) { thread->wait_timer_is_set = FALSE; if (timer_call_cancel(&thread->wait_timer)) thread->wait_timer_active--; } while (thread->wait_timer_active > 0) { thread_unlock(thread); splx(s); delay(1); s = splsched(); thread_lock(thread); } /* * If there is a reserved stack, release it. */ if (thread->reserved_stack != 0) { if (thread->reserved_stack != thread->kernel_stack) stack_free_stack(thread->reserved_stack); thread->reserved_stack = 0; } /* * Mark thread as terminating, and block. */ thread->state |= TH_TERMINATE; thread_mark_wait_locked(thread, THREAD_UNINT); assert(thread->promotions == 0); thread_unlock(thread); /* splsched */ thread_block((thread_continue_t)thread_terminate_continue); /*NOTREACHED*/ }
/* * thread_call_thread: */ static void thread_call_thread( thread_call_group_t group) { thread_t self = current_thread(); (void) splsched(); thread_call_lock_spin(); thread_sched_call(self, sched_call_thread); while (group->pending_count > 0) { thread_call_t call; thread_call_func_t func; thread_call_param_t param0, param1; call = TC(dequeue_head(&group->pending_queue)); group->pending_count--; func = call->func; param0 = call->param0; param1 = call->param1; call->queue = NULL; _internal_call_release(call); thread_call_unlock(); (void) spllo(); KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE, func, param0, param1, 0, 0); (*func)(param0, param1); if (get_preemption_level() != 0) { int pl = get_preemption_level(); panic("thread_call_thread: preemption_level %d, last callout %p(%p, %p)", pl, func, param0, param1); } (void)thread_funnel_set(self->funnel_lock, FALSE); /* XXX */ (void) splsched(); thread_call_lock_spin(); } thread_sched_call(self, NULL); group->active_count--; if (group->idle_count < thread_call_thread_min) { group->idle_count++; wait_queue_assert_wait(&group->idle_wqueue, NO_EVENT, THREAD_UNINT, 0); thread_call_unlock(); (void) spllo(); thread_block_parameter((thread_continue_t)thread_call_thread, group); /* NOTREACHED */ } thread_call_unlock(); (void) spllo(); thread_terminate(self); /* NOTREACHED */ }
thread_t old, continuation_t continuation, thread_t new) { spl_t s; assert(current_thread() == old); /* * XXX Dubious things here: * I don't check the idle_count on the processor set. * No scheduling priority or policy checks. * I assume the new thread is interruptible. */ s = splsched(); thread_lock(new); /* * The first thing we must do is check the state * of the threads, to ensure we can handoff. * This check uses current_processor()->processor_set, * which we can read without locking. */ if ((old->stack_privilege == current_stack()) || (new->state != (TH_WAIT|TH_SWAPPED)) || !check_processor_set(new) || !check_bound_processor(new)) { thread_unlock(new); (void) splx(s);
/* * ROUTINE: ulock_release_internal [internal] * * Releases the ulock. * If any threads are blocked waiting for the ulock, one is woken-up. * */ kern_return_t ulock_release_internal (ulock_t ulock, thread_t thread) { lock_set_t lock_set; if ((lock_set = ulock->lock_set) == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != thread) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If we have a hint that threads might be waiting, * try to transfer the lock ownership to a waiting thread * and wake it up. */ if (ulock->blocked) { wait_queue_t wq = &ulock->wait_queue; thread_t wqthread; spl_t s; s = splsched(); wait_queue_lock(wq); wqthread = wait_queue_wakeup64_identity_locked(wq, LOCK_SET_EVENT, THREAD_AWAKENED, TRUE); /* wait_queue now unlocked, thread locked */ if (wqthread != THREAD_NULL) { /* * JMM - These ownership transfer macros have a * locking/race problem. To keep the thread from * changing states on us (nullifying the ownership * assignment) we need to keep the thread locked * during the assignment. But we can't because the * macros take an activation lock, which is a mutex. * Since this code was already broken before I got * here, I will leave it for now. */ thread_unlock(wqthread); splx(s); /* * Transfer ulock ownership * from the current thread to the acquisition thread. */ ulock_ownership_clear(ulock); ulock_ownership_set(ulock, wqthread); ulock_unlock(ulock); return KERN_SUCCESS; } else { ulock->blocked = FALSE; splx(s); } } /* * Disown ulock */ ulock_ownership_clear(ulock); ulock_unlock(ulock); return KERN_SUCCESS; }