static void dm_start_request(struct mapped_device *md, struct request *orig)
{
if (!orig->q->mq_ops)
blk_start_request(orig);
else
blk_mq_start_request(orig);
atomic_inc(&md->pending[rq_data_dir(orig)]);
if (md->seq_rq_merge_deadline_usecs) {
md->last_rq_pos = rq_end_sector(orig);
md->last_rq_rw = rq_data_dir(orig);
md->last_rq_start_time = ktime_get();
}
if (unlikely(dm_stats_used(&md->stats))) {
struct dm_rq_target_io *tio = tio_from_request(orig);
tio->duration_jiffies = jiffies;
tio->n_sectors = blk_rq_sectors(orig);
dm_stats_account_io(&md->stats, rq_data_dir(orig),
blk_rq_pos(orig), tio->n_sectors, false, 0,
&tio->stats_aux);
}
/*
* Hold the md reference here for the in-flight I/O.
* We can't rely on the reference count by device opener,
* because the device may be closed during the request completion
* when all bios are completed.
* See the comment in rq_completed() too.
*/
dm_get(md);
}
/*
* Called when an error was detected during the last packet command.
* We queue a request sense packet command at the head of the request
* queue.
*/
void ide_retry_pc(ide_drive_t *drive)
{
struct request *failed_rq = drive->hwif->rq;
struct request *sense_rq = &drive->sense_rq;
struct ide_atapi_pc *pc = &drive->request_sense_pc;
(void)ide_read_error(drive);
/* init pc from sense_rq */
ide_init_pc(pc);
memcpy(pc->c, sense_rq->cmd, 12);
if (drive->media == ide_tape)
drive->atapi_flags |= IDE_AFLAG_IGNORE_DSC;
/*
* Push back the failed request and put request sense on top
* of it. The failed command will be retried after sense data
* is acquired.
*/
drive->hwif->rq = NULL;
ide_requeue_and_plug(drive, failed_rq);
if (ide_queue_sense_rq(drive, pc)) {
blk_start_request(failed_rq);
ide_complete_rq(drive, -EIO, blk_rq_bytes(failed_rq));
}
}
/**
* blk_queue_start_tag - find a free tag and assign it
* @q: the request queue for the device
* @rq: the block request that needs tagging
*
* Description:
* This can either be used as a stand-alone helper, or possibly be
* assigned as the queue &prep_rq_fn (in which case &struct request
* automagically gets a tag assigned). Note that this function
* assumes that any type of request can be queued! if this is not
* true for your device, you must check the request type before
* calling this function. The request will also be removed from
* the request queue, so it's the drivers responsibility to readd
* it if it should need to be restarted for some reason.
*
* Notes:
* queue lock must be held.
**/
int blk_queue_start_tag(struct request_queue *q, struct request *rq)
{
struct blk_queue_tag *bqt = q->queue_tags;
unsigned max_depth;
int tag;
if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
printk(KERN_ERR
"%s: request %p for device [%s] already tagged %d",
__func__, rq,
rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
BUG();
}
/*
* Protect against shared tag maps, as we may not have exclusive
* access to the tag map.
*
* We reserve a few tags just for sync IO, since we don't want
* to starve sync IO on behalf of flooding async IO.
*/
max_depth = bqt->max_depth;
if (!rq_is_sync(rq) && max_depth > 1) {
switch (max_depth) {
case 2:
max_depth = 1;
break;
case 3:
max_depth = 2;
break;
default:
max_depth -= 2;
}
if (q->in_flight[BLK_RW_ASYNC] > max_depth)
return 1;
}
do {
tag = find_first_zero_bit(bqt->tag_map, max_depth);
if (tag >= max_depth)
return 1;
} while (test_and_set_bit_lock(tag, bqt->tag_map));
/*
* We need lock ordering semantics given by test_and_set_bit_lock.
* See blk_queue_end_tag for details.
*/
rq->cmd_flags |= REQ_QUEUED;
rq->tag = tag;
bqt->tag_index[tag] = rq;
blk_start_request(rq);
list_add(&rq->queuelist, &q->tag_busy_list);
return 0;
}
/* Get the next read/write request; ending requests that we don't handle */
struct request *ace_get_next_request(struct request_queue * q)
{
struct request *req;
while ((req = blk_peek_request(q)) != NULL) {
if (req->cmd_type == REQ_TYPE_FS)
break;
blk_start_request(req);
__blk_end_request_all(req, -EIO);
}
return req;
}
/* Get the next read/write request; ending requests that we don't handle */
struct request *ace_get_next_request(struct request_queue * q)
{
struct request *req;
while ((req = blk_peek_request(q)) != NULL) {
if (blk_fs_request(req))
break;
blk_start_request(req);
__blk_end_request_all(req, -EIO);
}
return req;
}
/*
*****************************************************************************
* READ/WRITE REQUESTS
*****************************************************************************
*/
static void skd_fail_all_pending(struct skd_device *skdev)
{
struct request_queue *q = skdev->queue;
struct request *req;
for (;; ) {
req = blk_peek_request(q);
if (req == NULL)
break;
blk_start_request(req);
__blk_end_request_all(req, -EIO);
}
}
/*
* do_blkif_request
* read a block; request is in a request queue
*/
static void do_blkif_request(struct request_queue *rq)
{
struct blkfront_info *info = NULL;
struct request *req;
int queued;
pr_debug("Entered do_blkif_request\n");
queued = 0;
while ((req = blk_peek_request(rq)) != NULL) {
info = req->rq_disk->private_data;
if (RING_FULL(&info->ring))
goto wait;
blk_start_request(req);
if (!blk_fs_request(req)) {
__blk_end_request_all(req, -EIO);
continue;
}
pr_debug("do_blk_req %p: cmd %p, sec %lx, "
"(%u/%u) buffer:%p [%s]\n",
req, req->cmd, (unsigned long)blk_rq_pos(req),
blk_rq_cur_sectors(req), blk_rq_sectors(req),
req->buffer, rq_data_dir(req) ? "write" : "read");
if (blkif_queue_request(req)) {
blk_requeue_request(rq, req);
wait:
/* Avoid pointless unplugs. */
blk_stop_queue(rq);
break;
}
queued++;
}
if (queued != 0)
flush_requests(info);
}
void
aoedev_downdev(struct aoedev *d)
{
struct aoetgt *t, **tt, **te;
struct list_head *head, *pos, *nx;
struct request *rq;
int i;
d->flags &= ~DEVFL_UP;
/* clean out active and to-be-retransmitted buffers */
for (i = 0; i < NFACTIVE; i++) {
head = &d->factive[i];
list_for_each_safe(pos, nx, head)
downdev_frame(pos);
}
head = &d->rexmitq;
list_for_each_safe(pos, nx, head)
downdev_frame(pos);
/* reset window dressings */
tt = d->targets;
te = tt + d->ntargets;
for (; tt < te && (t = *tt); tt++) {
aoecmd_wreset(t);
t->nout = 0;
}
/* clean out the in-process request (if any) */
aoe_failip(d);
/* fast fail all pending I/O */
if (d->blkq) {
while ((rq = blk_peek_request(d->blkq))) {
blk_start_request(rq);
aoe_end_request(d, rq, 1);
}
}
if (d->gd)
set_capacity(d->gd, 0);
}
static void
aoeblk_request(struct request_queue *q)
{
struct aoedev *d;
struct request *rq;
d = q->queuedata;
if ((d->flags & DEVFL_UP) == 0) {
printk(KERN_INFO "aoe: device %ld.%d is not up\n",
d->aoemajor, d->aoeminor);
while ((rq = blk_peek_request(q))) {
blk_start_request(rq);
aoe_end_request(d, rq, 1);
}
return;
}
aoecmd_work(d);
return;
}
/*
* The simple form of the request function.
*/
static void sbull_request(struct request_queue *q)
{
struct request *req;
while ((req = blk_peek_request(q)) != NULL) {
struct sbull_dev *dev = req->rq_disk->private_data;
blk_start_request(req);
if (! req->cmd_type != REQ_TYPE_FS) {
printk (KERN_NOTICE "Skip non-fs request\n");
__blk_end_request_all(req, -EIO);
continue;
}
// printk (KERN_NOTICE "Req dev %d dir %ld sec %ld, nr %d f %lx\n",
// dev - Devices, rq_data_dir(req),
// req->sector, req->current_nr_sectors,
// req->flags);
sbull_transfer(dev, blk_rq_pos(req), blk_rq_cur_bytes(req),
req->buffer, rq_data_dir(req));
__blk_end_request_all(req, 0);
}
}
static void ace_fsm_dostate(struct ace_device *ace)
{
struct request *req;
u32 status;
u16 val;
int count;
#if defined(DEBUG)
dev_dbg(ace->dev, "fsm_state=%i, id_req_count=%i\n",
ace->fsm_state, ace->id_req_count);
#endif
/* Verify that there is actually a CF in the slot. If not, then
* bail out back to the idle state and wake up all the waiters */
status = ace_in32(ace, ACE_STATUS);
if ((status & ACE_STATUS_CFDETECT) == 0) {
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->media_change = 1;
set_capacity(ace->gd, 0);
dev_info(ace->dev, "No CF in slot\n");
/* Drop all in-flight and pending requests */
if (ace->req) {
__blk_end_request_all(ace->req, -EIO);
ace->req = NULL;
}
while ((req = blk_fetch_request(ace->queue)) != NULL)
__blk_end_request_all(req, -EIO);
/* Drop back to IDLE state and notify waiters */
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->id_result = -EIO;
while (ace->id_req_count) {
complete(&ace->id_completion);
ace->id_req_count--;
}
}
switch (ace->fsm_state) {
case ACE_FSM_STATE_IDLE:
/* See if there is anything to do */
if (ace->id_req_count || ace_get_next_request(ace->queue)) {
ace->fsm_iter_num++;
ace->fsm_state = ACE_FSM_STATE_REQ_LOCK;
mod_timer(&ace->stall_timer, jiffies + HZ);
if (!timer_pending(&ace->stall_timer))
add_timer(&ace->stall_timer);
break;
}
del_timer(&ace->stall_timer);
ace->fsm_continue_flag = 0;
break;
case ACE_FSM_STATE_REQ_LOCK:
if (ace_in(ace, ACE_STATUS) & ACE_STATUS_MPULOCK) {
/* Already have the lock, jump to next state */
ace->fsm_state = ACE_FSM_STATE_WAIT_CFREADY;
break;
}
/* Request the lock */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_LOCKREQ);
ace->fsm_state = ACE_FSM_STATE_WAIT_LOCK;
break;
case ACE_FSM_STATE_WAIT_LOCK:
if (ace_in(ace, ACE_STATUS) & ACE_STATUS_MPULOCK) {
/* got the lock; move to next state */
ace->fsm_state = ACE_FSM_STATE_WAIT_CFREADY;
break;
}
/* wait a bit for the lock */
ace_fsm_yield(ace);
break;
case ACE_FSM_STATE_WAIT_CFREADY:
status = ace_in32(ace, ACE_STATUS);
if (!(status & ACE_STATUS_RDYFORCFCMD) ||
(status & ACE_STATUS_CFBSY)) {
/* CF card isn't ready; it needs to be polled */
ace_fsm_yield(ace);
break;
}
/* Device is ready for command; determine what to do next */
if (ace->id_req_count)
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_PREPARE;
else
ace->fsm_state = ACE_FSM_STATE_REQ_PREPARE;
break;
case ACE_FSM_STATE_IDENTIFY_PREPARE:
/* Send identify command */
ace->fsm_task = ACE_TASK_IDENTIFY;
ace->data_ptr = ace->cf_id;
ace->data_count = ACE_BUF_PER_SECTOR;
ace_out(ace, ACE_SECCNTCMD, ACE_SECCNTCMD_IDENTIFY);
/* As per datasheet, put config controller in reset */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_CFGRESET);
/* irq handler takes over from this point; wait for the
* transfer to complete */
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_TRANSFER;
ace_fsm_yieldirq(ace);
break;
case ACE_FSM_STATE_IDENTIFY_TRANSFER:
/* Check that the sysace is ready to receive data */
status = ace_in32(ace, ACE_STATUS);
if (status & ACE_STATUS_CFBSY) {
dev_dbg(ace->dev, "CFBSY set; t=%i iter=%i dc=%i\n",
ace->fsm_task, ace->fsm_iter_num,
ace->data_count);
ace_fsm_yield(ace);
break;
}
if (!(status & ACE_STATUS_DATABUFRDY)) {
ace_fsm_yield(ace);
break;
}
/* Transfer the next buffer */
ace->reg_ops->datain(ace);
ace->data_count--;
/* If there are still buffers to be transfers; jump out here */
if (ace->data_count != 0) {
ace_fsm_yieldirq(ace);
break;
}
/* transfer finished; kick state machine */
dev_dbg(ace->dev, "identify finished\n");
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_COMPLETE;
break;
case ACE_FSM_STATE_IDENTIFY_COMPLETE:
ace_fix_driveid(ace->cf_id);
ace_dump_mem(ace->cf_id, 512); /* Debug: Dump out disk ID */
if (ace->data_result) {
/* Error occured, disable the disk */
ace->media_change = 1;
set_capacity(ace->gd, 0);
dev_err(ace->dev, "error fetching CF id (%i)\n",
ace->data_result);
} else {
ace->media_change = 0;
/* Record disk parameters */
set_capacity(ace->gd,
ata_id_u32(ace->cf_id, ATA_ID_LBA_CAPACITY));
dev_info(ace->dev, "capacity: %i sectors\n",
ata_id_u32(ace->cf_id, ATA_ID_LBA_CAPACITY));
}
/* We're done, drop to IDLE state and notify waiters */
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->id_result = ace->data_result;
while (ace->id_req_count) {
complete(&ace->id_completion);
ace->id_req_count--;
}
break;
case ACE_FSM_STATE_REQ_PREPARE:
req = ace_get_next_request(ace->queue);
if (!req) {
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
}
blk_start_request(req);
/* Okay, it's a data request, set it up for transfer */
dev_dbg(ace->dev,
"request: sec=%llx hcnt=%x, ccnt=%x, dir=%i\n",
(unsigned long long)blk_rq_pos(req),
blk_rq_sectors(req), blk_rq_cur_sectors(req),
rq_data_dir(req));
ace->req = req;
ace->data_ptr = req->buffer;
ace->data_count = blk_rq_cur_sectors(req) * ACE_BUF_PER_SECTOR;
ace_out32(ace, ACE_MPULBA, blk_rq_pos(req) & 0x0FFFFFFF);
count = blk_rq_sectors(req);
if (rq_data_dir(req)) {
/* Kick off write request */
dev_dbg(ace->dev, "write data\n");
ace->fsm_task = ACE_TASK_WRITE;
ace_out(ace, ACE_SECCNTCMD,
count | ACE_SECCNTCMD_WRITE_DATA);
} else {
/* Kick off read request */
dev_dbg(ace->dev, "read data\n");
ace->fsm_task = ACE_TASK_READ;
ace_out(ace, ACE_SECCNTCMD,
count | ACE_SECCNTCMD_READ_DATA);
}
/* As per datasheet, put config controller in reset */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_CFGRESET);
/* Move to the transfer state. The systemace will raise
* an interrupt once there is something to do
*/
ace->fsm_state = ACE_FSM_STATE_REQ_TRANSFER;
if (ace->fsm_task == ACE_TASK_READ)
ace_fsm_yieldirq(ace); /* wait for data ready */
break;
case ACE_FSM_STATE_REQ_TRANSFER:
/* Check that the sysace is ready to receive data */
status = ace_in32(ace, ACE_STATUS);
if (status & ACE_STATUS_CFBSY) {
dev_dbg(ace->dev,
"CFBSY set; t=%i iter=%i c=%i dc=%i irq=%i\n",
ace->fsm_task, ace->fsm_iter_num,
blk_rq_cur_sectors(ace->req) * 16,
ace->data_count, ace->in_irq);
ace_fsm_yield(ace); /* need to poll CFBSY bit */
break;
}
if (!(status & ACE_STATUS_DATABUFRDY)) {
dev_dbg(ace->dev,
"DATABUF not set; t=%i iter=%i c=%i dc=%i irq=%i\n",
ace->fsm_task, ace->fsm_iter_num,
blk_rq_cur_sectors(ace->req) * 16,
ace->data_count, ace->in_irq);
ace_fsm_yieldirq(ace);
break;
}
/* Transfer the next buffer */
if (ace->fsm_task == ACE_TASK_WRITE)
ace->reg_ops->dataout(ace);
else
ace->reg_ops->datain(ace);
ace->data_count--;
/* If there are still buffers to be transfers; jump out here */
if (ace->data_count != 0) {
ace_fsm_yieldirq(ace);
break;
}
/* bio finished; is there another one? */
if (__blk_end_request_cur(ace->req, 0)) {
/* dev_dbg(ace->dev, "next block; h=%u c=%u\n",
* blk_rq_sectors(ace->req),
* blk_rq_cur_sectors(ace->req));
*/
ace->data_ptr = ace->req->buffer;
ace->data_count = blk_rq_cur_sectors(ace->req) * 16;
ace_fsm_yieldirq(ace);
break;
}
ace->fsm_state = ACE_FSM_STATE_REQ_COMPLETE;
break;
case ACE_FSM_STATE_REQ_COMPLETE:
ace->req = NULL;
/* Finished request; go to idle state */
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
default:
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
}
}
static void ace_fsm_dostate(struct ace_device *ace)
{
struct request *req;
u32 status;
u16 val;
int count;
#if defined(DEBUG)
dev_dbg(ace->dev, "fsm_state=%i, id_req_count=%i\n",
ace->fsm_state, ace->id_req_count);
#endif
/*
*/
status = ace_in32(ace, ACE_STATUS);
if ((status & ACE_STATUS_CFDETECT) == 0) {
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->media_change = 1;
set_capacity(ace->gd, 0);
dev_info(ace->dev, "No CF in slot\n");
/* */
if (ace->req) {
__blk_end_request_all(ace->req, -EIO);
ace->req = NULL;
}
while ((req = blk_fetch_request(ace->queue)) != NULL)
__blk_end_request_all(req, -EIO);
/* */
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->id_result = -EIO;
while (ace->id_req_count) {
complete(&ace->id_completion);
ace->id_req_count--;
}
}
switch (ace->fsm_state) {
case ACE_FSM_STATE_IDLE:
/* */
if (ace->id_req_count || ace_get_next_request(ace->queue)) {
ace->fsm_iter_num++;
ace->fsm_state = ACE_FSM_STATE_REQ_LOCK;
mod_timer(&ace->stall_timer, jiffies + HZ);
if (!timer_pending(&ace->stall_timer))
add_timer(&ace->stall_timer);
break;
}
del_timer(&ace->stall_timer);
ace->fsm_continue_flag = 0;
break;
case ACE_FSM_STATE_REQ_LOCK:
if (ace_in(ace, ACE_STATUS) & ACE_STATUS_MPULOCK) {
/* */
ace->fsm_state = ACE_FSM_STATE_WAIT_CFREADY;
break;
}
/* */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_LOCKREQ);
ace->fsm_state = ACE_FSM_STATE_WAIT_LOCK;
break;
case ACE_FSM_STATE_WAIT_LOCK:
if (ace_in(ace, ACE_STATUS) & ACE_STATUS_MPULOCK) {
/* */
ace->fsm_state = ACE_FSM_STATE_WAIT_CFREADY;
break;
}
/* */
ace_fsm_yield(ace);
break;
case ACE_FSM_STATE_WAIT_CFREADY:
status = ace_in32(ace, ACE_STATUS);
if (!(status & ACE_STATUS_RDYFORCFCMD) ||
(status & ACE_STATUS_CFBSY)) {
/* */
ace_fsm_yield(ace);
break;
}
/* */
if (ace->id_req_count)
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_PREPARE;
else
ace->fsm_state = ACE_FSM_STATE_REQ_PREPARE;
break;
case ACE_FSM_STATE_IDENTIFY_PREPARE:
/* */
ace->fsm_task = ACE_TASK_IDENTIFY;
ace->data_ptr = ace->cf_id;
ace->data_count = ACE_BUF_PER_SECTOR;
ace_out(ace, ACE_SECCNTCMD, ACE_SECCNTCMD_IDENTIFY);
/* */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_CFGRESET);
/*
*/
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_TRANSFER;
ace_fsm_yieldirq(ace);
break;
case ACE_FSM_STATE_IDENTIFY_TRANSFER:
/* */
status = ace_in32(ace, ACE_STATUS);
if (status & ACE_STATUS_CFBSY) {
dev_dbg(ace->dev, "CFBSY set; t=%i iter=%i dc=%i\n",
ace->fsm_task, ace->fsm_iter_num,
ace->data_count);
ace_fsm_yield(ace);
break;
}
if (!(status & ACE_STATUS_DATABUFRDY)) {
ace_fsm_yield(ace);
break;
}
/* */
ace->reg_ops->datain(ace);
ace->data_count--;
/* */
if (ace->data_count != 0) {
ace_fsm_yieldirq(ace);
break;
}
/* */
dev_dbg(ace->dev, "identify finished\n");
ace->fsm_state = ACE_FSM_STATE_IDENTIFY_COMPLETE;
break;
case ACE_FSM_STATE_IDENTIFY_COMPLETE:
ace_fix_driveid(ace->cf_id);
ace_dump_mem(ace->cf_id, 512); /* */
if (ace->data_result) {
/* */
ace->media_change = 1;
set_capacity(ace->gd, 0);
dev_err(ace->dev, "error fetching CF id (%i)\n",
ace->data_result);
} else {
ace->media_change = 0;
/* */
set_capacity(ace->gd,
ata_id_u32(ace->cf_id, ATA_ID_LBA_CAPACITY));
dev_info(ace->dev, "capacity: %i sectors\n",
ata_id_u32(ace->cf_id, ATA_ID_LBA_CAPACITY));
}
/* */
ace->fsm_state = ACE_FSM_STATE_IDLE;
ace->id_result = ace->data_result;
while (ace->id_req_count) {
complete(&ace->id_completion);
ace->id_req_count--;
}
break;
case ACE_FSM_STATE_REQ_PREPARE:
req = ace_get_next_request(ace->queue);
if (!req) {
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
}
blk_start_request(req);
/* */
dev_dbg(ace->dev,
"request: sec=%llx hcnt=%x, ccnt=%x, dir=%i\n",
(unsigned long long)blk_rq_pos(req),
blk_rq_sectors(req), blk_rq_cur_sectors(req),
rq_data_dir(req));
ace->req = req;
ace->data_ptr = req->buffer;
ace->data_count = blk_rq_cur_sectors(req) * ACE_BUF_PER_SECTOR;
ace_out32(ace, ACE_MPULBA, blk_rq_pos(req) & 0x0FFFFFFF);
count = blk_rq_sectors(req);
if (rq_data_dir(req)) {
/* */
dev_dbg(ace->dev, "write data\n");
ace->fsm_task = ACE_TASK_WRITE;
ace_out(ace, ACE_SECCNTCMD,
count | ACE_SECCNTCMD_WRITE_DATA);
} else {
/* */
dev_dbg(ace->dev, "read data\n");
ace->fsm_task = ACE_TASK_READ;
ace_out(ace, ACE_SECCNTCMD,
count | ACE_SECCNTCMD_READ_DATA);
}
/* */
val = ace_in(ace, ACE_CTRL);
ace_out(ace, ACE_CTRL, val | ACE_CTRL_CFGRESET);
/*
*/
ace->fsm_state = ACE_FSM_STATE_REQ_TRANSFER;
if (ace->fsm_task == ACE_TASK_READ)
ace_fsm_yieldirq(ace); /* */
break;
case ACE_FSM_STATE_REQ_TRANSFER:
/* */
status = ace_in32(ace, ACE_STATUS);
if (status & ACE_STATUS_CFBSY) {
dev_dbg(ace->dev,
"CFBSY set; t=%i iter=%i c=%i dc=%i irq=%i\n",
ace->fsm_task, ace->fsm_iter_num,
blk_rq_cur_sectors(ace->req) * 16,
ace->data_count, ace->in_irq);
ace_fsm_yield(ace); /* */
break;
}
if (!(status & ACE_STATUS_DATABUFRDY)) {
dev_dbg(ace->dev,
"DATABUF not set; t=%i iter=%i c=%i dc=%i irq=%i\n",
ace->fsm_task, ace->fsm_iter_num,
blk_rq_cur_sectors(ace->req) * 16,
ace->data_count, ace->in_irq);
ace_fsm_yieldirq(ace);
break;
}
/* */
if (ace->fsm_task == ACE_TASK_WRITE)
ace->reg_ops->dataout(ace);
else
ace->reg_ops->datain(ace);
ace->data_count--;
/* */
if (ace->data_count != 0) {
ace_fsm_yieldirq(ace);
break;
}
/* */
if (__blk_end_request_cur(ace->req, 0)) {
/*
*/
ace->data_ptr = ace->req->buffer;
ace->data_count = blk_rq_cur_sectors(ace->req) * 16;
ace_fsm_yieldirq(ace);
break;
}
ace->fsm_state = ACE_FSM_STATE_REQ_COMPLETE;
break;
case ACE_FSM_STATE_REQ_COMPLETE:
ace->req = NULL;
/* */
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
default:
ace->fsm_state = ACE_FSM_STATE_IDLE;
break;
}
}
static void skd_request_fn(struct request_queue *q)
{
struct skd_device *skdev = q->queuedata;
struct skd_fitmsg_context *skmsg = NULL;
struct fit_msg_hdr *fmh = NULL;
struct skd_request_context *skreq;
struct request *req = NULL;
struct skd_scsi_request *scsi_req;
struct page *page;
unsigned long io_flags;
int error;
u32 lba;
u32 count;
int data_dir;
u32 be_lba;
u32 be_count;
u64 be_dmaa;
u64 cmdctxt;
u32 timo_slot;
void *cmd_ptr;
int flush, fua;
if (skdev->state != SKD_DRVR_STATE_ONLINE) {
skd_request_fn_not_online(q);
return;
}
if (blk_queue_stopped(skdev->queue)) {
if (skdev->skmsg_free_list == NULL ||
skdev->skreq_free_list == NULL ||
skdev->in_flight >= skdev->queue_low_water_mark)
/* There is still some kind of shortage */
return;
queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
}
/*
* Stop conditions:
* - There are no more native requests
* - There are already the maximum number of requests in progress
* - There are no more skd_request_context entries
* - There are no more FIT msg buffers
*/
for (;; ) {
flush = fua = 0;
req = blk_peek_request(q);
/* Are there any native requests to start? */
if (req == NULL)
break;
lba = (u32)blk_rq_pos(req);
count = blk_rq_sectors(req);
data_dir = rq_data_dir(req);
io_flags = req->cmd_flags;
if (io_flags & REQ_FLUSH)
flush++;
if (io_flags & REQ_FUA)
fua++;
pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
"count=%u(0x%x) dir=%d\n",
skdev->name, __func__, __LINE__,
req, lba, lba, count, count, data_dir);
/* At this point we know there is a request */
/* Are too many requets already in progress? */
if (skdev->in_flight >= skdev->cur_max_queue_depth) {
pr_debug("%s:%s:%d qdepth %d, limit %d\n",
skdev->name, __func__, __LINE__,
skdev->in_flight, skdev->cur_max_queue_depth);
break;
}
/* Is a skd_request_context available? */
skreq = skdev->skreq_free_list;
if (skreq == NULL) {
pr_debug("%s:%s:%d Out of req=%p\n",
skdev->name, __func__, __LINE__, q);
break;
}
SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
/* Now we check to see if we can get a fit msg */
if (skmsg == NULL) {
if (skdev->skmsg_free_list == NULL) {
pr_debug("%s:%s:%d Out of msg\n",
skdev->name, __func__, __LINE__);
break;
}
}
skreq->flush_cmd = 0;
skreq->n_sg = 0;
skreq->sg_byte_count = 0;
skreq->discard_page = 0;
/*
* OK to now dequeue request from q.
*
* At this point we are comitted to either start or reject
* the native request. Note that skd_request_context is
* available but is still at the head of the free list.
*/
blk_start_request(req);
skreq->req = req;
skreq->fitmsg_id = 0;
/* Either a FIT msg is in progress or we have to start one. */
if (skmsg == NULL) {
/* Are there any FIT msg buffers available? */
skmsg = skdev->skmsg_free_list;
if (skmsg == NULL) {
pr_debug("%s:%s:%d Out of msg skdev=%p\n",
skdev->name, __func__, __LINE__,
skdev);
break;
}
SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
skdev->skmsg_free_list = skmsg->next;
skmsg->state = SKD_MSG_STATE_BUSY;
skmsg->id += SKD_ID_INCR;
/* Initialize the FIT msg header */
fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
memset(fmh, 0, sizeof(*fmh));
fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
skmsg->length = sizeof(*fmh);
}
skreq->fitmsg_id = skmsg->id;
/*
* Note that a FIT msg may have just been started
* but contains no SoFIT requests yet.
*/
/*
* Transcode the request, checking as we go. The outcome of
* the transcoding is represented by the error variable.
*/
cmd_ptr = &skmsg->msg_buf[skmsg->length];
memset(cmd_ptr, 0, 32);
be_lba = cpu_to_be32(lba);
be_count = cpu_to_be32(count);
be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
cmdctxt = skreq->id + SKD_ID_INCR;
scsi_req = cmd_ptr;
scsi_req->hdr.tag = cmdctxt;
scsi_req->hdr.sg_list_dma_address = be_dmaa;
if (data_dir == READ)
skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
else
skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
if (io_flags & REQ_DISCARD) {
page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
if (!page) {
pr_err("request_fn:Page allocation failed.\n");
skd_end_request(skdev, skreq, -ENOMEM);
break;
}
skreq->discard_page = 1;
req->completion_data = page;
skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
} else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
skd_prep_zerosize_flush_cdb(scsi_req, skreq);
SKD_ASSERT(skreq->flush_cmd == 1);
} else {
skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
}
if (fua)
scsi_req->cdb[1] |= SKD_FUA_NV;
if (!req->bio)
goto skip_sg;
error = skd_preop_sg_list(skdev, skreq);
if (error != 0) {
/*
* Complete the native request with error.
* Note that the request context is still at the
* head of the free list, and that the SoFIT request
* was encoded into the FIT msg buffer but the FIT
* msg length has not been updated. In short, the
* only resource that has been allocated but might
* not be used is that the FIT msg could be empty.
*/
pr_debug("%s:%s:%d error Out\n",
skdev->name, __func__, __LINE__);
skd_end_request(skdev, skreq, error);
continue;
}
skip_sg:
scsi_req->hdr.sg_list_len_bytes =
cpu_to_be32(skreq->sg_byte_count);
/* Complete resource allocations. */
skdev->skreq_free_list = skreq->next;
skreq->state = SKD_REQ_STATE_BUSY;
skreq->id += SKD_ID_INCR;
skmsg->length += sizeof(struct skd_scsi_request);
fmh->num_protocol_cmds_coalesced++;
/*
* Update the active request counts.
* Capture the timeout timestamp.
*/
skreq->timeout_stamp = skdev->timeout_stamp;
timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
skdev->timeout_slot[timo_slot]++;
skdev->in_flight++;
pr_debug("%s:%s:%d req=0x%x busy=%d\n",
skdev->name, __func__, __LINE__,
skreq->id, skdev->in_flight);
/*
* If the FIT msg buffer is full send it.
*/
if (skmsg->length >= SKD_N_FITMSG_BYTES ||
fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
skd_send_fitmsg(skdev, skmsg);
skmsg = NULL;
fmh = NULL;
}
}
/*
* Is a FIT msg in progress? If it is empty put the buffer back
* on the free list. If it is non-empty send what we got.
* This minimizes latency when there are fewer requests than
* what fits in a FIT msg.
*/
if (skmsg != NULL) {
/* Bigger than just a FIT msg header? */
if (skmsg->length > sizeof(struct fit_msg_hdr)) {
pr_debug("%s:%s:%d sending msg=%p, len %d\n",
skdev->name, __func__, __LINE__,
skmsg, skmsg->length);
skd_send_fitmsg(skdev, skmsg);
} else {
/*
* The FIT msg is empty. It means we got started
* on the msg, but the requests were rejected.
*/
skmsg->state = SKD_MSG_STATE_IDLE;
skmsg->id += SKD_ID_INCR;
skmsg->next = skdev->skmsg_free_list;
skdev->skmsg_free_list = skmsg;
}
skmsg = NULL;
fmh = NULL;
}
/*
* If req is non-NULL it means there is something to do but
* we are out of a resource.
*/
if (req)
blk_stop_queue(skdev->queue);
}
