static void punt_bios_to_rescuer(struct bio_set *bs)
{
struct bio_list punt, nopunt;
struct bio *bio;
/*
* In order to guarantee forward progress we must punt only bios that
* were allocated from this bio_set; otherwise, if there was a bio on
* there for a stacking driver higher up in the stack, processing it
* could require allocating bios from this bio_set, and doing that from
* our own rescuer would be bad.
*
* Since bio lists are singly linked, pop them all instead of trying to
* remove from the middle of the list:
*/
bio_list_init(&punt);
bio_list_init(&nopunt);
while ((bio = bio_list_pop(current->bio_list)))
bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
*current->bio_list = nopunt;
spin_lock(&bs->rescue_lock);
bio_list_merge(&bs->rescue_list, &punt);
spin_unlock(&bs->rescue_lock);
queue_work(bs->rescue_workqueue, &bs->rescue_work);
}
/*
* @inmates must have been initialised prior to this call
*/
static void __cell_release(struct dm_bio_prison_cell *cell,
struct bio_list *inmates)
{
hlist_del(&cell->list);
if (inmates) {
if (cell->holder)
bio_list_add(inmates, cell->holder);
bio_list_merge(inmates, &cell->bios);
}
}
/*
* Add the bio to the list of deferred io.
*/
static int queue_io(struct mapped_device *md, struct bio *bio)
{
down_write(&md->lock);
if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
up_write(&md->lock);
return 1;
}
bio_list_add(&md->deferred, bio);
up_write(&md->lock);
return 0; /* deferred successfully */
}
static int __bio_detain(struct dm_bio_prison *prison,
struct dm_cell_key *key,
struct bio *inmate,
struct dm_bio_prison_cell *cell_prealloc,
struct dm_bio_prison_cell **cell_result)
{
uint32_t hash = hash_key(prison, key);
struct dm_bio_prison_cell *cell;
cell = __search_bucket(prison->cells + hash, key);
if (cell) {
if (inmate)
bio_list_add(&cell->bios, inmate);
*cell_result = cell;
return 1;
}
__setup_new_cell(prison, key, inmate, hash, cell_prealloc);
*cell_result = cell_prealloc;
return 0;
}
static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio)
{
struct rrpc *rrpc = q->queuedata;
struct nvm_rq *rqd;
int err;
if (bio->bi_rw & REQ_DISCARD) {
rrpc_discard(rrpc, bio);
return BLK_QC_T_NONE;
}
rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
if (!rqd) {
pr_err_ratelimited("rrpc: not able to queue bio.");
bio_io_error(bio);
return BLK_QC_T_NONE;
}
memset(rqd, 0, sizeof(struct nvm_rq));
err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE);
switch (err) {
case NVM_IO_OK:
return BLK_QC_T_NONE;
case NVM_IO_ERR:
bio_io_error(bio);
break;
case NVM_IO_DONE:
bio_endio(bio);
break;
case NVM_IO_REQUEUE:
spin_lock(&rrpc->bio_lock);
bio_list_add(&rrpc->requeue_bios, bio);
spin_unlock(&rrpc->bio_lock);
queue_work(rrpc->kgc_wq, &rrpc->ws_requeue);
break;
}
mempool_free(rqd, rrpc->rq_pool);
return BLK_QC_T_NONE;
}
/*
* Decrements the number of outstanding ios that a bio has been
* cloned into, completing the original io if necc.
*/
static void dec_pending(struct dm_io *io, int error)
{
unsigned long flags;
/* Push-back supersedes any I/O errors */
if (error && !(io->error > 0 && __noflush_suspending(io->md)))
io->error = error;
if (atomic_dec_and_test(&io->io_count)) {
if (io->error == DM_ENDIO_REQUEUE) {
/*
* Target requested pushing back the I/O.
* This must be handled before the sleeper on
* suspend queue merges the pushback list.
*/
spin_lock_irqsave(&io->md->pushback_lock, flags);
if (__noflush_suspending(io->md))
bio_list_add(&io->md->pushback, io->bio);
else
/* noflush suspend was interrupted. */
io->error = -EIO;
spin_unlock_irqrestore(&io->md->pushback_lock, flags);
}
if (end_io_acct(io))
/* nudge anyone waiting on suspend queue */
wake_up(&io->md->wait);
if (io->error != DM_ENDIO_REQUEUE) {
blk_add_trace_bio(io->md->queue, io->bio,
BLK_TA_COMPLETE);
bio_endio(io->bio, io->bio->bi_size, io->error);
}
free_io(io->md, io);
}
}
/* static void stackbd_make_request(struct request_queue *q, struct bio *bio) */
static int stackbd_make_request(struct request_queue *q, struct bio *bio)
{
printk("stackbd: make request %-5s block %-12llu #pages %-4hu total-size "
"%-10u\n", bio_data_dir(bio) == WRITE ? "write" : "read",
(unsigned long long) bio->bi_sector, bio->bi_vcnt, bio->bi_size);
// printk("<%p> Make request %s %s %s\n", bio,
// bio->bi_rw & REQ_SYNC ? "SYNC" : "",
// bio->bi_rw & REQ_FLUSH ? "FLUSH" : "",
// bio->bi_rw & REQ_NOIDLE ? "NOIDLE" : "");
//
spin_lock_irq(&stackbd.lock);
if (!stackbd.bdev_raw)
{
printk("stackbd: Request before bdev_raw is ready, aborting\n");
goto abort;
}
if (!stackbd.is_active)
{
printk("stackbd: Device not active yet, aborting\n");
goto abort;
}
bio_list_add(&stackbd.bio_list, bio);
wake_up(&req_event);
spin_unlock_irq(&stackbd.lock);
/* FIXME:VER return; */
return 0;
abort:
spin_unlock_irq(&stackbd.lock);
printk("<%p> Abort request\n\n", bio);
bio_io_error(bio);
/*FIXME:VER return; */
return 0;
}
static int iblock_execute_rw(struct se_cmd *cmd)
{
struct scatterlist *sgl = cmd->t_data_sg;
u32 sgl_nents = cmd->t_data_nents;
enum dma_data_direction data_direction = cmd->data_direction;
struct se_device *dev = cmd->se_dev;
struct iblock_req *ibr;
struct bio *bio;
struct bio_list list;
struct scatterlist *sg;
u32 sg_num = sgl_nents;
sector_t block_lba;
unsigned bio_cnt;
int rw;
int i;
if (data_direction == DMA_TO_DEVICE) {
/*
* Force data to disk if we pretend to not have a volatile
* write cache, or the initiator set the Force Unit Access bit.
*/
if (dev->se_sub_dev->se_dev_attrib.emulate_write_cache == 0 ||
(dev->se_sub_dev->se_dev_attrib.emulate_fua_write > 0 &&
(cmd->se_cmd_flags & SCF_FUA)))
rw = WRITE_FUA;
else
rw = WRITE;
} else {
rw = READ;
}
/*
* Convert the blocksize advertised to the initiator to the 512 byte
* units unconditionally used by the Linux block layer.
*/
if (dev->se_sub_dev->se_dev_attrib.block_size == 4096)
block_lba = (cmd->t_task_lba << 3);
else if (dev->se_sub_dev->se_dev_attrib.block_size == 2048)
block_lba = (cmd->t_task_lba << 2);
else if (dev->se_sub_dev->se_dev_attrib.block_size == 1024)
block_lba = (cmd->t_task_lba << 1);
else if (dev->se_sub_dev->se_dev_attrib.block_size == 512)
block_lba = cmd->t_task_lba;
else {
pr_err("Unsupported SCSI -> BLOCK LBA conversion:"
" %u\n", dev->se_sub_dev->se_dev_attrib.block_size);
cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
return -ENOSYS;
}
ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
if (!ibr)
goto fail;
cmd->priv = ibr;
bio = iblock_get_bio(cmd, block_lba, sgl_nents);
if (!bio)
goto fail_free_ibr;
bio_list_init(&list);
bio_list_add(&list, bio);
atomic_set(&ibr->pending, 2);
bio_cnt = 1;
for_each_sg(sgl, sg, sgl_nents, i) {
/*
* XXX: if the length the device accepts is shorter than the
* length of the S/G list entry this will cause and
* endless loop. Better hope no driver uses huge pages.
*/
while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
!= sg->length) {
if (bio_cnt >= IBLOCK_MAX_BIO_PER_TASK) {
iblock_submit_bios(&list, rw);
bio_cnt = 0;
}
bio = iblock_get_bio(cmd, block_lba, sg_num);
if (!bio)
goto fail_put_bios;
atomic_inc(&ibr->pending);
bio_list_add(&list, bio);
bio_cnt++;
}
/* Always in 512 byte units for Linux/Block */
block_lba += sg->length >> IBLOCK_LBA_SHIFT;
sg_num--;
}
iblock_submit_bios(&list, rw);
iblock_complete_cmd(cmd);
return 0;
fail_put_bios:
while ((bio = bio_list_pop(&list)))
bio_put(bio);
fail_free_ibr:
kfree(ibr);
cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
fail:
return -ENOMEM;
}
static sense_reason_t
iblock_execute_write_same(struct se_cmd *cmd)
{
struct iblock_req *ibr;
struct scatterlist *sg;
struct bio *bio;
struct bio_list list;
sector_t block_lba = cmd->t_task_lba;
sector_t sectors = sbc_get_write_same_sectors(cmd);
sg = &cmd->t_data_sg[0];
if (cmd->t_data_nents > 1 ||
sg->length != cmd->se_dev->dev_attrib.block_size) {
pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u"
" block_size: %u\n", cmd->t_data_nents, sg->length,
cmd->se_dev->dev_attrib.block_size);
return TCM_INVALID_CDB_FIELD;
}
ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
if (!ibr)
goto fail;
cmd->priv = ibr;
bio = iblock_get_bio(cmd, block_lba, 1);
if (!bio)
goto fail_free_ibr;
bio_list_init(&list);
bio_list_add(&list, bio);
atomic_set(&ibr->pending, 1);
while (sectors) {
while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
!= sg->length) {
bio = iblock_get_bio(cmd, block_lba, 1);
if (!bio)
goto fail_put_bios;
atomic_inc(&ibr->pending);
bio_list_add(&list, bio);
}
/* Always in 512 byte units for Linux/Block */
block_lba += sg->length >> IBLOCK_LBA_SHIFT;
sectors -= 1;
}
iblock_submit_bios(&list, WRITE);
return 0;
fail_put_bios:
while ((bio = bio_list_pop(&list)))
bio_put(bio);
fail_free_ibr:
kfree(ibr);
fail:
return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
}