static void padata_parallel_worker(struct work_struct *parallel_work)
{
struct padata_parallel_queue *pqueue;
struct parallel_data *pd;
struct padata_instance *pinst;
LIST_HEAD(local_list);
local_bh_disable();
pqueue = container_of(parallel_work,
struct padata_parallel_queue, work);
pd = pqueue->pd;
pinst = pd->pinst;
spin_lock(&pqueue->parallel.lock);
list_replace_init(&pqueue->parallel.list, &local_list);
spin_unlock(&pqueue->parallel.lock);
while (!list_empty(&local_list)) {
struct padata_priv *padata;
padata = list_entry(local_list.next,
struct padata_priv, list);
list_del_init(&padata->list);
padata->parallel(padata);
}
local_bh_enable();
}
/**
* __run_timers - run all expired timers (if any) on this CPU.
* @base: the timer vector to be processed.
*
* This function executes all expired timer vectors.
*/
static inline void __run_timers(struct tti_tvec_base *base, unsigned long internal_tti)
{
struct tti_timer_list *timer;
while (time_after_eq(internal_tti, base->timer_jiffies)) {
struct list_head work_list;
struct list_head *head = &work_list;
int index = base->timer_jiffies & TVR_MASK;
++base->timer_jiffies;
list_replace_init(base->tv.vec + index, &work_list);
while (!list_empty(head)) {
void (*fn)(unsigned long);
unsigned long data;
timer = list_first_entry(head, struct tti_timer_list, entry);
fn = timer->function;
data = timer->data;
base->running_timer = timer;
detach_timer(timer, 1);
call_timer_fn(timer, fn, data);
}
}
base->running_timer = NULL;
}
/*
* Softirq action handler - move entries to local list and loop over them
* while passing them to the queue registered handler.
*/
static void blk_done_softirq(struct softirq_action *h)
{
struct list_head *cpu_list, local_list;
local_irq_disable();
cpu_list = this_cpu_ptr(&blk_cpu_done);
list_replace_init(cpu_list, &local_list);
local_irq_enable();
while (!list_empty(&local_list)) {
struct request *rq;
rq = list_entry(local_list.next, struct request, ipi_list);
list_del_init(&rq->ipi_list);
rq->q->softirq_done_fn(rq);
}
}
/*
* Softirq action handler - move entries to local list and loop over them
* while passing them to the queue registered handler.
*/
static void blk_done_softirq(struct softirq_action *h)
{
struct list_head *cpu_list, local_list;
local_irq_disable();
cpu_list = &__get_cpu_var(blk_cpu_done);
list_replace_init(cpu_list, &local_list); /* 将CPU已完成请求链表中所有项转移到局部链表 */
local_irq_enable();
while (!list_empty(&local_list)) { /* 循环处理每个项，将其充链表中删除 */
struct request *rq;
rq = list_entry(local_list.next, struct request, csd.list);
list_del_init(&rq->csd.list);
rq->q->softirq_done_fn(rq); /* 调用请求队列的软中断完成回调函数来处理，scsi_softirq_done */
}
}
/* Workaround for non-implemented aio_fsync()
*/
static
int aio_sync_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct aio_output *output = tinfo->output;
MARS_DBG("sync thread has started on '%s'.\n", output->brick->brick_path);
//set_user_nice(current, -20);
while (!brick_thread_should_stop() || atomic_read(&tinfo->queued_sum) > 0) {
LIST_HEAD(tmp_list);
unsigned long flags;
int i;
output->fdsync_active = false;
wake_up_interruptible_all(&output->fdsync_event);
wait_event_interruptible_timeout(
tinfo->event,
atomic_read(&tinfo->queued_sum) > 0,
HZ / 4);
traced_lock(&tinfo->lock, flags);
for (i = 0; i < MARS_PRIO_NR; i++) {
struct list_head *start = &tinfo->mref_list[i];
if (!list_empty(start)) {
// move over the whole list
list_replace_init(start, &tmp_list);
atomic_sub(tinfo->queued[i], &tinfo->queued_sum);
tinfo->queued[i] = 0;
break;
}
}
traced_unlock(&tinfo->lock, flags);
if (!list_empty(&tmp_list)) {
aio_sync_all(output, &tmp_list);
}
}
MARS_DBG("sync thread has stopped.\n");
tinfo->terminated = true;
wake_up_interruptible_all(&tinfo->terminate_event);
return 0;
}
/*
* iwl_pcie_rx_allocator - Allocates pages in the background for RX queues
*
* Allocates for each received request 8 pages
* Called as a scheduled work item.
*/
static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
struct iwl_rb_allocator *rba = &trans_pcie->rba;
struct list_head local_empty;
int pending = atomic_xchg(&rba->req_pending, 0);
IWL_DEBUG_RX(trans, "Pending allocation requests = %d\n", pending);
/* If we were scheduled - there is at least one request */
spin_lock(&rba->lock);
/* swap out the rba->rbd_empty to a local list */
list_replace_init(&rba->rbd_empty, &local_empty);
spin_unlock(&rba->lock);
while (pending) {
int i;
struct list_head local_allocated;
INIT_LIST_HEAD(&local_allocated);
for (i = 0; i < RX_CLAIM_REQ_ALLOC;) {
struct iwl_rx_mem_buffer *rxb;
struct page *page;
/* List should never be empty - each reused RBD is
* returned to the list, and initial pool covers any
* possible gap between the time the page is allocated
* to the time the RBD is added.
*/
BUG_ON(list_empty(&local_empty));
/* Get the first rxb from the rbd list */
rxb = list_first_entry(&local_empty,
struct iwl_rx_mem_buffer, list);
BUG_ON(rxb->page);
/* Alloc a new receive buffer */
page = iwl_pcie_rx_alloc_page(trans, GFP_KERNEL);
if (!page)
continue;
rxb->page = page;
/* Get physical address of the RB */
rxb->page_dma = dma_map_page(trans->dev, page, 0,
PAGE_SIZE << trans_pcie->rx_page_order,
DMA_FROM_DEVICE);
if (dma_mapping_error(trans->dev, rxb->page_dma)) {
rxb->page = NULL;
__free_pages(page, trans_pcie->rx_page_order);
continue;
}
/* dma address must be no more than 36 bits */
BUG_ON(rxb->page_dma & ~DMA_BIT_MASK(36));
/* and also 256 byte aligned! */
BUG_ON(rxb->page_dma & DMA_BIT_MASK(8));
/* move the allocated entry to the out list */
list_move(&rxb->list, &local_allocated);
i++;
}
pending--;
if (!pending) {
pending = atomic_xchg(&rba->req_pending, 0);
IWL_DEBUG_RX(trans,
"Pending allocation requests = %d\n",
pending);
}
spin_lock(&rba->lock);
/* add the allocated rbds to the allocator allocated list */
list_splice_tail(&local_allocated, &rba->rbd_allocated);
/* get more empty RBDs for current pending requests */
list_splice_tail_init(&rba->rbd_empty, &local_empty);
spin_unlock(&rba->lock);
atomic_inc(&rba->req_ready);
}
spin_lock(&rba->lock);
/* return unused rbds to the allocator empty list */
list_splice_tail(&local_empty, &rba->rbd_empty);
spin_unlock(&rba->lock);
}