static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
{
struct completion_queue *cq;
struct llist_node *entry;
struct nullb_cmd *cmd;
cq = &per_cpu(completion_queues, smp_processor_id());
while ((entry = llist_del_all(&cq->list)) != NULL) {
entry = llist_reverse_order(entry);
do {
struct request_queue *q = NULL;
cmd = container_of(entry, struct nullb_cmd, ll_list);
entry = entry->next;
if (cmd->rq)
q = cmd->rq->q;
end_cmd(cmd);
if (q && !q->mq_ops && blk_queue_stopped(q)) {
spin_lock(q->queue_lock);
if (blk_queue_stopped(q))
blk_start_queue(q);
spin_unlock(q->queue_lock);
}
} while (entry);
}
return HRTIMER_NORESTART;
}
/*---------------------------------------------------------------------------*/
static void priv_ev_loop_run_tasklet(unsigned long data)
{
struct xio_ev_loop *loop = (struct xio_ev_loop *) data;
struct xio_ev_data *tev;
struct llist_node *node;
while ((node = llist_del_all(&loop->ev_llist)) != NULL) {
node = llist_reverse_order(node);
while (node) {
tev = llist_entry(node, struct xio_ev_data, ev_llist);
node = llist_next(node);
tev->handler(tev->data);
}
}
}
static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
{
struct completion_queue *cq;
struct llist_node *entry;
struct nullb_cmd *cmd;
cq = &per_cpu(completion_queues, smp_processor_id());
while ((entry = llist_del_all(&cq->list)) != NULL) {
entry = llist_reverse_order(entry);
do {
cmd = container_of(entry, struct nullb_cmd, ll_list);
entry = entry->next;
end_cmd(cmd);
} while (entry);
}
return HRTIMER_NORESTART;
}
/**
* flush_smp_call_function_queue - Flush pending smp-call-function callbacks
*
* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
* offline CPU. Skip this check if set to 'false'.
*
* Flush any pending smp-call-function callbacks queued on this CPU. This is
* invoked by the generic IPI handler, as well as by a CPU about to go offline,
* to ensure that all pending IPI callbacks are run before it goes completely
* offline.
*
* Loop through the call_single_queue and run all the queued callbacks.
* Must be called with interrupts disabled.
*/
static void flush_smp_call_function_queue(bool warn_cpu_offline)
{
struct llist_head *head;
struct llist_node *entry;
call_single_data_t *csd, *csd_next;
static bool warned;
lockdep_assert_irqs_disabled();
head = this_cpu_ptr(&call_single_queue);
entry = llist_del_all(head);
entry = llist_reverse_order(entry);
/* There shouldn't be any pending callbacks on an offline CPU. */
if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
!warned && !llist_empty(head))) {
warned = true;
WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
/*
* We don't have to use the _safe() variant here
* because we are not invoking the IPI handlers yet.
*/
llist_for_each_entry(csd, entry, llist)
pr_warn("IPI callback %pS sent to offline CPU\n",
csd->func);
}
llist_for_each_entry_safe(csd, csd_next, entry, llist) {
smp_call_func_t func = csd->func;
void *info = csd->info;
/* Do we wait until *after* callback? */
if (csd->flags & CSD_FLAG_SYNCHRONOUS) {
func(info);
csd_unlock(csd);
} else {
csd_unlock(csd);
func(info);
}
}
/*---------------------------------------------------------------------------*/
int priv_ev_loop_run(void *loop_hndl)
{
struct xio_ev_loop *loop = loop_hndl;
struct xio_ev_data *tev;
struct llist_node *node;
int cpu;
clear_bit(XIO_EV_LOOP_STOP, &loop->states);
switch (loop->flags) {
case XIO_LOOP_GIVEN_THREAD:
if (loop->ctx->worker != (uint64_t) get_current()) {
ERROR_LOG("worker kthread(%p) is not current(%p).\n",
(void *) loop->ctx->worker, get_current());
goto cleanup0;
}
/* no need to disable preemption */
cpu = raw_smp_processor_id();
if (loop->ctx->cpuid != cpu) {
TRACE_LOG("worker on core(%d) scheduled to(%d).\n",
cpu, loop->ctx->cpuid);
set_cpus_allowed_ptr(get_current(),
cpumask_of(loop->ctx->cpuid));
}
break;
case XIO_LOOP_TASKLET:
/* were events added to list while in STOP state ? */
if (!llist_empty(&loop->ev_llist))
priv_kick_tasklet(loop_hndl);
return 0;
case XIO_LOOP_WORKQUEUE:
/* were events added to list while in STOP state ? */
while ((node = llist_del_all(&loop->ev_llist)) != NULL) {
node = llist_reverse_order(node);
while (node) {
tev = llist_entry(node, struct xio_ev_data,
ev_llist);
node = llist_next(node);
tev->work.func = priv_ev_loop_run_work;
queue_work_on(loop->ctx->cpuid, loop->workqueue,
&tev->work);
}
}
return 0;
default:
/* undo */
set_bit(XIO_EV_LOOP_STOP, &loop->states);
return -1;
}
retry_wait:
wait_event_interruptible(loop->wait,
test_bit(XIO_EV_LOOP_WAKE, &loop->states));
retry_dont_wait:
while ((node = llist_del_all(&loop->ev_llist)) != NULL) {
node = llist_reverse_order(node);
while (node) {
tev = llist_entry(node, struct xio_ev_data, ev_llist);
node = llist_next(node);
tev->handler(tev->data);
}
}
/* "race point" */
clear_bit(XIO_EV_LOOP_WAKE, &loop->states);
if (unlikely(test_bit(XIO_EV_LOOP_STOP, &loop->states)))
return 0;
/* if a new entry was added while we were at "race point"
* than wait event might block forever as condition is false */
if (llist_empty(&loop->ev_llist))
goto retry_wait;
/* race detected */
if (!test_and_set_bit(XIO_EV_LOOP_WAKE, &loop->states))
goto retry_dont_wait;
/* was one wakeup was called */
goto retry_wait;
cleanup0:
set_bit(XIO_EV_LOOP_STOP, &loop->states);
return -1;
}
