/*
* row_add_request() - Add request to the scheduler
* @q: requests queue
* @rq: request to add
*
*/
static void row_add_request(struct request_queue *q,
struct request *rq)
{
struct row_data *rd = (struct row_data *)q->elevator->elevator_data;
struct row_queue *rqueue = RQ_ROWQ(rq);
s64 diff_ms;
bool queue_was_empty = list_empty(&rqueue->fifo);
list_add_tail(&rq->queuelist, &rqueue->fifo);
rd->nr_reqs[rq_data_dir(rq)]++;
rqueue->nr_req++;
rq_set_fifo_time(rq, jiffies); /* for statistics*/
if (rq->cmd_flags & REQ_URGENT) {
WARN_ON(1);
blk_dump_rq_flags(rq, "");
rq->cmd_flags &= ~REQ_URGENT;
}
if (row_queues_def[rqueue->prio].idling_enabled) {
if (rd->rd_idle_data.idling_queue_idx == rqueue->prio &&
hrtimer_active(&rd->rd_idle_data.hr_timer)) {
if (hrtimer_try_to_cancel(
&rd->rd_idle_data.hr_timer) >= 0) {
row_log_rowq(rd, rqueue->prio,
"Canceled delayed work on %d",
rd->rd_idle_data.idling_queue_idx);
rd->rd_idle_data.idling_queue_idx =
ROWQ_MAX_PRIO;
}
}
diff_ms = ktime_to_ms(ktime_sub(ktime_get(),
rqueue->idle_data.last_insert_time));
if (unlikely(diff_ms < 0)) {
pr_err("%s(): time delta error: diff_ms < 0",
__func__);
rqueue->idle_data.begin_idling = false;
return;
}
if (diff_ms < rd->rd_idle_data.freq_ms) {
rqueue->idle_data.begin_idling = true;
row_log_rowq(rd, rqueue->prio, "Enable idling");
} else {
rqueue->idle_data.begin_idling = false;
row_log_rowq(rd, rqueue->prio, "Disable idling (%ldms)",
(long)diff_ms);
}
rqueue->idle_data.last_insert_time = ktime_get();
}
if (row_queues_def[rqueue->prio].is_urgent &&
!rd->pending_urgent_rq && !rd->urgent_in_flight) {
/* Handle High Priority queues */
if (rqueue->prio < ROWQ_REG_PRIO_IDX &&
rd->last_served_ioprio_class != IOPRIO_CLASS_RT &&
queue_was_empty) {
row_log_rowq(rd, rqueue->prio,
"added (high prio) urgent request");
rq->cmd_flags |= REQ_URGENT;
rd->pending_urgent_rq = rq;
} else if (row_rowq_unserved(rd, rqueue->prio)) {
/* Handle Regular priotity queues */
row_log_rowq(rd, rqueue->prio,
"added urgent request (total on queue=%d)",
rqueue->nr_req);
rq->cmd_flags |= REQ_URGENT;
rd->pending_urgent_rq = rq;
}
} else
row_log_rowq(rd, rqueue->prio,
"added request (total on queue=%d)", rqueue->nr_req);
}
/*
* row_dispatch_requests() - selects the next request to dispatch
* @q: requests queue
* @force: ignored
*
* Return 0 if no requests were moved to the dispatch queue.
* 1 otherwise
*
*/
static int row_dispatch_requests(struct request_queue *q, int force)
{
struct row_data *rd = (struct row_data *)q->elevator->elevator_data;
int ret = 0, currq, i;
currq = rd->curr_queue;
/*
* Find the first unserved queue (with higher priority then currq)
* that is not empty
*/
for (i = 0; i < currq; i++) {
if (row_rowq_unserved(rd, i) &&
!list_empty(&rd->row_queues[i].rqueue.fifo)) {
row_log_rowq(rd, currq,
" Preemting for unserved rowq%d", i);
rd->curr_queue = i;
row_dispatch_insert(rd);
ret = 1;
goto done;
}
}
if (rd->row_queues[currq].rqueue.nr_dispatched >=
rd->row_queues[currq].disp_quantum) {
rd->row_queues[currq].rqueue.nr_dispatched = 0;
row_log_rowq(rd, currq, "Expiring rqueue");
ret = row_choose_queue(rd);
if (ret)
row_dispatch_insert(rd);
goto done;
}
/* Dispatch from curr_queue */
if (list_empty(&rd->row_queues[currq].rqueue.fifo)) {
/* check idling */
if (delayed_work_pending(&rd->read_idle.idle_work)) {
if (force) {
(void)cancel_delayed_work(
&rd->read_idle.idle_work);
row_log_rowq(rd, currq,
"Canceled delayed work - forced dispatch");
} else {
row_log_rowq(rd, currq,
"Delayed work pending. Exiting");
goto done;
}
}
if (!force && queue_idling_enabled[currq] &&
rd->row_queues[currq].rqueue.idle_data.begin_idling) {
if (!queue_delayed_work(rd->read_idle.idle_workqueue,
&rd->read_idle.idle_work,
rd->read_idle.idle_time)) {
row_log_rowq(rd, currq,
"Work already on queue!");
pr_err("ROW_BUG: Work already on queue!");
} else
row_log_rowq(rd, currq,
"Scheduled delayed work. exiting");
goto done;
} else {
row_log_rowq(rd, currq,
"Currq empty. Choose next queue");
ret = row_choose_queue(rd);
if (!ret)
goto done;
}
}
ret = 1;
row_dispatch_insert(rd);
done:
return ret;
}
/*
* row_get_ioprio_class_to_serve() - Return the next I/O priority
* class to dispatch requests from
* @rd: pointer to struct row_data
* @force: flag indicating if forced dispatch
*
* This function returns the next I/O priority class to serve
* {IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE}.
* If there are no more requests in scheduler or if we're idling on some queue
* IOPRIO_CLASS_NONE will be returned.
* If idling is scheduled on a lower priority queue than the one that needs
* to be served, it will be canceled.
*
*/
static int row_get_ioprio_class_to_serve(struct row_data *rd, int force)
{
int i;
int ret = IOPRIO_CLASS_NONE;
if (!rd->nr_reqs[READ] && !rd->nr_reqs[WRITE]) {
row_log(rd->dispatch_queue, "No more requests in scheduler");
goto check_idling;
}
/* First, go over the high priority queues */
for (i = 0; i < ROWQ_REG_PRIO_IDX; i++) {
if (!list_empty(&rd->row_queues[i].fifo)) {
if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
if (hrtimer_try_to_cancel(
&rd->rd_idle_data.hr_timer) >= 0) {
row_log(rd->dispatch_queue,
"Canceling delayed work on %d. RT pending",
rd->rd_idle_data.idling_queue_idx);
rd->rd_idle_data.idling_queue_idx =
ROWQ_MAX_PRIO;
}
}
if (row_regular_req_pending(rd) &&
(rd->reg_prio_starvation.starvation_counter >=
rd->reg_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_BE;
else if (row_low_req_pending(rd) &&
(rd->low_prio_starvation.starvation_counter >=
rd->low_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_IDLE;
else
ret = IOPRIO_CLASS_RT;
goto done;
}
}
/*
* At the moment idling is implemented only for READ queues.
* If enabled on WRITE, this needs updating
*/
if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
row_log(rd->dispatch_queue, "Delayed work pending. Exiting");
goto done;
}
check_idling:
/* Check for (high priority) idling and enable if needed */
for (i = 0; i < ROWQ_REG_PRIO_IDX && !force; i++) {
if (rd->row_queues[i].idle_data.begin_idling &&
row_queues_def[i].idling_enabled)
goto initiate_idling;
}
/* Regular priority queues */
for (i = ROWQ_REG_PRIO_IDX; i < ROWQ_LOW_PRIO_IDX; i++) {
if (list_empty(&rd->row_queues[i].fifo)) {
/* We can idle only if this is not a forced dispatch */
if (rd->row_queues[i].idle_data.begin_idling &&
!force && row_queues_def[i].idling_enabled)
goto initiate_idling;
} else {
if (row_low_req_pending(rd) &&
(rd->low_prio_starvation.starvation_counter >=
rd->low_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_IDLE;
else
ret = IOPRIO_CLASS_BE;
goto done;
}
}
if (rd->nr_reqs[READ] || rd->nr_reqs[WRITE])
ret = IOPRIO_CLASS_IDLE;
goto done;
initiate_idling:
hrtimer_start(&rd->rd_idle_data.hr_timer,
ktime_set(0, rd->rd_idle_data.idle_time_ms * NSEC_PER_MSEC),
HRTIMER_MODE_REL);
rd->rd_idle_data.idling_queue_idx = i;
row_log_rowq(rd, i, "Scheduled delayed work on %d. exiting", i);
done:
return ret;
}
static int row_get_ioprio_class_to_serve(struct row_data *rd, int force)
{
int i;
int ret = IOPRIO_CLASS_NONE;
if (!rd->nr_reqs[READ] && !rd->nr_reqs[WRITE]) {
row_log(rd->dispatch_queue, "No more requests in scheduler");
goto check_idling;
}
for (i = 0; i < ROWQ_REG_PRIO_IDX; i++) {
if (!list_empty(&rd->row_queues[i].fifo)) {
if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
if (hrtimer_try_to_cancel(
&rd->rd_idle_data.hr_timer) >= 0) {
row_log(rd->dispatch_queue,
"Canceling delayed work on %d. RT pending",
rd->rd_idle_data.idling_queue_idx);
rd->rd_idle_data.idling_queue_idx =
ROWQ_MAX_PRIO;
}
}
if (row_regular_req_pending(rd) &&
(rd->reg_prio_starvation.starvation_counter >=
rd->reg_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_BE;
else if (row_low_req_pending(rd) &&
(rd->low_prio_starvation.starvation_counter >=
rd->low_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_IDLE;
else
ret = IOPRIO_CLASS_RT;
goto done;
}
}
if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
row_log(rd->dispatch_queue, "Delayed work pending. Exiting");
goto done;
}
check_idling:
for (i = 0; i < ROWQ_REG_PRIO_IDX && !force; i++) {
if (rd->row_queues[i].idle_data.begin_idling &&
row_queues_def[i].idling_enabled)
goto initiate_idling;
}
for (i = ROWQ_REG_PRIO_IDX; i < ROWQ_LOW_PRIO_IDX; i++) {
if (list_empty(&rd->row_queues[i].fifo)) {
if (rd->row_queues[i].idle_data.begin_idling &&
!force && row_queues_def[i].idling_enabled)
goto initiate_idling;
} else {
if (row_low_req_pending(rd) &&
(rd->low_prio_starvation.starvation_counter >=
rd->low_prio_starvation.starvation_limit))
ret = IOPRIO_CLASS_IDLE;
else
ret = IOPRIO_CLASS_BE;
goto done;
}
}
if (rd->nr_reqs[READ] || rd->nr_reqs[WRITE])
ret = IOPRIO_CLASS_IDLE;
goto done;
initiate_idling:
hrtimer_start(&rd->rd_idle_data.hr_timer,
ktime_set(0, rd->rd_idle_data.idle_time_ms * NSEC_PER_MSEC),
HRTIMER_MODE_REL);
rd->rd_idle_data.idling_queue_idx = i;
row_log_rowq(rd, i, "Scheduled delayed work on %d. exiting", i);
done:
return ret;
}