void
timer_queue_shutdown(
mpqueue_head_t *queue)
{
timer_call_t call;
mpqueue_head_t *new_queue;
spl_t s;
DBG("timer_queue_shutdown(%p)\n", queue);
s = splclock();
/* Note comma operator in while expression re-locking each iteration */
while (timer_queue_lock_spin(queue), !queue_empty(&queue->head)) {
call = TIMER_CALL(queue_first(&queue->head));
if (!simple_lock_try(&call->lock)) {
/*
* case (2b) lock order inversion, dequeue and skip
* Don't change the call_entry queue back-pointer
* but set the async_dequeue field.
*/
timer_queue_shutdown_lock_skips++;
timer_call_entry_dequeue_async(call);
#if TIMER_ASSERT
TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
call,
call->async_dequeue,
CE(call)->queue,
0x2b, 0);
#endif
timer_queue_unlock(queue);
continue;
}
/* remove entry from old queue */
timer_call_entry_dequeue(call);
timer_queue_unlock(queue);
/* and queue it on new */
new_queue = timer_queue_assign(CE(call)->deadline);
timer_queue_lock_spin(new_queue);
timer_call_entry_enqueue_deadline(
call, new_queue, CE(call)->deadline);
timer_queue_unlock(new_queue);
simple_unlock(&call->lock);
}
timer_queue_unlock(queue);
splx(s);
}
/*
* Assumes call_entry and queues unlocked, interrupts disabled.
*/
__inline__ mpqueue_head_t *
timer_call_enqueue_deadline_unlocked(
timer_call_t call,
mpqueue_head_t *queue,
uint64_t deadline)
{
call_entry_t entry = CE(call);
mpqueue_head_t *old_queue;
DBG("timer_call_enqueue_deadline_unlocked(%p,%p,)\n", call, queue);
simple_lock(&call->lock);
old_queue = MPQUEUE(entry->queue);
if (old_queue != NULL) {
timer_queue_lock_spin(old_queue);
if (call->async_dequeue) {
/* collision (1c): timer already dequeued, clear flag */
#if TIMER_ASSERT
TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
call,
call->async_dequeue,
CE(call)->queue,
0x1c, 0);
timer_call_enqueue_deadline_unlocked_async1++;
#endif
call->async_dequeue = FALSE;
entry->queue = NULL;
} else if (old_queue != queue) {
timer_call_entry_dequeue(call);
#if TIMER_ASSERT
timer_call_enqueue_deadline_unlocked_async2++;
#endif
}
if (old_queue == timer_longterm_queue)
timer_longterm_dequeued_locked(call);
if (old_queue != queue) {
timer_queue_unlock(old_queue);
timer_queue_lock_spin(queue);
}
} else {
timer_queue_lock_spin(queue);
}
timer_call_entry_enqueue_deadline(call, queue, deadline);
timer_queue_unlock(queue);
simple_unlock(&call->lock);
return (old_queue);
}
/*
* Assumes call_entry and queues unlocked, interrupts disabled.
*/
__inline__ mpqueue_head_t *
timer_call_enqueue_deadline_unlocked(
timer_call_t call,
mpqueue_head_t *queue,
uint64_t deadline)
{
call_entry_t entry = CE(call);
mpqueue_head_t *old_queue;
DBG("timer_call_enqueue_deadline_unlocked(%p,%p,)\n", call, queue);
simple_lock(&call->lock);
old_queue = MPQUEUE(entry->queue);
if (old_queue != NULL) {
timer_call_lock_spin(old_queue);
if (call->async_dequeue) {
/* collision (1c): null queue pointer and reset flag */
call->async_dequeue = FALSE;
entry->queue = NULL;
#if TIMER_ASSERT
timer_call_enqueue_deadline_unlocked_async1++;
#endif
} else if (old_queue != queue) {
(void)remque(qe(entry));
entry->queue = NULL;
#if TIMER_ASSERT
timer_call_enqueue_deadline_unlocked_async2++;
#endif
}
if (old_queue != queue) {
timer_call_unlock(old_queue);
timer_call_lock_spin(queue);
}
} else {
timer_call_lock_spin(queue);
}
timer_call_entry_enqueue_deadline(call, queue, deadline);
timer_call_unlock(queue);
simple_unlock(&call->lock);
return (old_queue);
}
void
timer_queue_shutdown(
mpqueue_head_t *queue)
{
timer_call_t call;
mpqueue_head_t *new_queue;
spl_t s;
DBG("timer_queue_shutdown(%p)\n", queue);
s = splclock();
/* Note comma operator in while expression re-locking each iteration */
while (timer_queue_lock_spin(queue), !queue_empty(&queue->head)) {
call = TIMER_CALL(queue_first(&queue->head));
if (!simple_lock_try(&call->lock)) {
/*
* case (2b) lock order inversion, dequeue and skip
* Don't change the call_entry queue back-pointer
* but set the async_dequeue field.
*/
timer_queue_shutdown_lock_skips++;
timer_call_entry_dequeue_async(call);
#if TIMER_ASSERT
TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
VM_KERNEL_UNSLIDE_OR_PERM(call),
call->async_dequeue,
VM_KERNEL_UNSLIDE_OR_PERM(TCE(call)->queue),
0x2b, 0);
#endif
timer_queue_unlock(queue);
continue;
}
boolean_t call_local = ((call->flags & TIMER_CALL_LOCAL) != 0);
/* remove entry from old queue */
timer_call_entry_dequeue(call);
timer_queue_unlock(queue);
if (call_local == FALSE) {
/* and queue it on new, discarding LOCAL timers */
new_queue = timer_queue_assign(TCE(call)->deadline);
timer_queue_lock_spin(new_queue);
timer_call_entry_enqueue_deadline(
call, new_queue, TCE(call)->deadline);
timer_queue_unlock(new_queue);
} else {
timer_queue_shutdown_discarded++;
}
/* The only lingering LOCAL timer should be this thread's
* quantum expiration timer.
*/
assert((call_local == FALSE) ||
(TCE(call)->func == thread_quantum_expire));
simple_unlock(&call->lock);
}
timer_queue_unlock(queue);
splx(s);
}
/*
* Assumes call_entry and queues unlocked, interrupts disabled.
*/
__inline__ mpqueue_head_t *
timer_call_enqueue_deadline_unlocked(
timer_call_t call,
mpqueue_head_t *queue,
uint64_t deadline,
uint64_t soft_deadline,
uint64_t ttd,
timer_call_param_t param1,
uint32_t callout_flags)
{
call_entry_t entry = TCE(call);
mpqueue_head_t *old_queue;
DBG("timer_call_enqueue_deadline_unlocked(%p,%p,)\n", call, queue);
simple_lock(&call->lock);
old_queue = MPQUEUE(entry->queue);
if (old_queue != NULL) {
timer_queue_lock_spin(old_queue);
if (call->async_dequeue) {
/* collision (1c): timer already dequeued, clear flag */
#if TIMER_ASSERT
TIMER_KDEBUG_TRACE(KDEBUG_TRACE,
DECR_TIMER_ASYNC_DEQ | DBG_FUNC_NONE,
VM_KERNEL_UNSLIDE_OR_PERM(call),
call->async_dequeue,
VM_KERNEL_UNSLIDE_OR_PERM(TCE(call)->queue),
0x1c, 0);
timer_call_enqueue_deadline_unlocked_async1++;
#endif
call->async_dequeue = FALSE;
entry->queue = NULL;
} else if (old_queue != queue) {
timer_call_entry_dequeue(call);
#if TIMER_ASSERT
timer_call_enqueue_deadline_unlocked_async2++;
#endif
}
if (old_queue == timer_longterm_queue)
timer_longterm_dequeued_locked(call);
if (old_queue != queue) {
timer_queue_unlock(old_queue);
timer_queue_lock_spin(queue);
}
} else {
timer_queue_lock_spin(queue);
}
call->soft_deadline = soft_deadline;
call->flags = callout_flags;
TCE(call)->param1 = param1;
call->ttd = ttd;
timer_call_entry_enqueue_deadline(call, queue, deadline);
timer_queue_unlock(queue);
simple_unlock(&call->lock);
return (old_queue);
}
/*
* timer_queue_migrate() is called by etimer_queue_migrate()
* to move timer requests from the local processor (queue_from)
* to a target processor's (queue_to).
*/
int
timer_queue_migrate(mpqueue_head_t *queue_from, mpqueue_head_t *queue_to)
{
timer_call_t call;
timer_call_t head_to;
int timers_migrated = 0;
DBG("timer_queue_migrate(%p,%p)\n", queue_from, queue_to);
assert(!ml_get_interrupts_enabled());
assert(queue_from != queue_to);
if (serverperfmode) {
/*
* if we're running a high end server
* avoid migrations... they add latency
* and don't save us power under typical
* server workloads
*/
return -4;
}
/*
* Take both local (from) and target (to) timer queue locks while
* moving the timers from the local queue to the target processor.
* We assume that the target is always the boot processor.
* But only move if all of the following is true:
* - the target queue is non-empty
* - the local queue is non-empty
* - the local queue's first deadline is later than the target's
* - the local queue contains no non-migrateable "local" call
* so that we need not have the target resync.
*/
timer_call_lock_spin(queue_to);
head_to = TIMER_CALL(queue_first(&queue_to->head));
if (queue_empty(&queue_to->head)) {
timers_migrated = -1;
goto abort1;
}
timer_call_lock_spin(queue_from);
if (queue_empty(&queue_from->head)) {
timers_migrated = -2;
goto abort2;
}
call = TIMER_CALL(queue_first(&queue_from->head));
if (CE(call)->deadline < CE(head_to)->deadline) {
timers_migrated = 0;
goto abort2;
}
/* perform scan for non-migratable timers */
do {
if (call->flags & TIMER_CALL_LOCAL) {
timers_migrated = -3;
goto abort2;
}
call = TIMER_CALL(queue_next(qe(call)));
} while (!queue_end(&queue_from->head, qe(call)));
/* migration loop itself -- both queues are locked */
while (!queue_empty(&queue_from->head)) {
call = TIMER_CALL(queue_first(&queue_from->head));
if (!simple_lock_try(&call->lock)) {
/* case (2b) lock order inversion, dequeue only */
timer_queue_migrate_lock_skips++;
(void) remque(qe(call));
call->async_dequeue = TRUE;
continue;
}
timer_call_entry_dequeue(call);
timer_call_entry_enqueue_deadline(
call, queue_to, CE(call)->deadline);
timers_migrated++;
simple_unlock(&call->lock);
}
abort2:
timer_call_unlock(queue_from);
abort1:
timer_call_unlock(queue_to);
return timers_migrated;
}
