static struct dm_rq_target_io *dm_old_prep_tio(struct request *rq, struct mapped_device *md, gfp_t gfp_mask) { struct dm_rq_target_io *tio; int srcu_idx; struct dm_table *table; tio = alloc_old_rq_tio(md, gfp_mask); if (!tio) return NULL; init_tio(tio, rq, md); table = dm_get_live_table(md, &srcu_idx); /* * Must clone a request if this .request_fn DM device * is stacked on .request_fn device(s). */ if (!dm_table_mq_request_based(table)) { if (!clone_old_rq(rq, md, tio, gfp_mask)) { dm_put_live_table(md, srcu_idx); free_old_rq_tio(tio); return NULL; } } dm_put_live_table(md, srcu_idx); return tio; }
/* * q->request_fn for old request-based dm. * Called with the queue lock held. */ static void dm_old_request_fn(struct request_queue *q) { struct mapped_device *md = q->queuedata; struct dm_target *ti = md->immutable_target; struct request *rq; struct dm_rq_target_io *tio; sector_t pos = 0; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); if (unlikely(!map)) { dm_put_live_table(md, srcu_idx); return; } ti = dm_table_find_target(map, pos); dm_put_live_table(md, srcu_idx); } /* * For suspend, check blk_queue_stopped() and increment * ->pending within a single queue_lock not to increment the * number of in-flight I/Os after the queue is stopped in * dm_suspend(). */ while (!blk_queue_stopped(q)) { rq = blk_peek_request(q); if (!rq) return; /* always use block 0 to find the target for flushes for now */ pos = 0; if (req_op(rq) != REQ_OP_FLUSH) pos = blk_rq_pos(rq); if ((dm_old_request_peeked_before_merge_deadline(md) && md_in_flight(md) && rq->bio && !bio_multiple_segments(rq->bio) && md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) || (ti->type->busy && ti->type->busy(ti))) { blk_delay_queue(q, 10); return; } dm_start_request(md, rq); tio = tio_from_request(rq); init_tio(tio, rq, md); /* Establish tio->ti before queuing work (map_tio_request) */ tio->ti = ti; kthread_queue_work(&md->kworker, &tio->work); BUG_ON(!irqs_disabled()); } }
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); struct mapped_device *md = tio->md; struct dm_target *ti = md->immutable_target; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); } /* * On suspend dm_stop_queue() handles stopping the blk-mq * request_queue BUT: even though the hw_queues are marked * BLK_MQ_S_STOPPED at that point there is still a race that * is allowing block/blk-mq.c to call ->queue_rq against a * hctx that it really shouldn't. The following check guards * against this rarity (albeit _not_ race-free). */ if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state))) return BLK_MQ_RQ_QUEUE_BUSY; if (ti->type->busy && ti->type->busy(ti)) return BLK_MQ_RQ_QUEUE_BUSY; dm_start_request(md, rq); /* Init tio using md established in .init_request */ init_tio(tio, rq, md); /* * Establish tio->ti before calling map_request(). */ tio->ti = ti; /* Direct call is fine since .queue_rq allows allocations */ if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { /* Undo dm_start_request() before requeuing */ rq_end_stats(md, rq); rq_completed(md, rq_data_dir(rq), false); return BLK_MQ_RQ_QUEUE_BUSY; } return BLK_MQ_RQ_QUEUE_OK; }
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); struct mapped_device *md = tio->md; struct dm_target *ti = md->immutable_target; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); } if (ti->type->busy && ti->type->busy(ti)) return BLK_MQ_RQ_QUEUE_BUSY; dm_start_request(md, rq); /* Init tio using md established in .init_request */ init_tio(tio, rq, md); /* * Establish tio->ti before calling map_request(). */ tio->ti = ti; /* Direct call is fine since .queue_rq allows allocations */ if (map_request(tio) == DM_MAPIO_REQUEUE) { /* Undo dm_start_request() before requeuing */ rq_end_stats(md, rq); rq_completed(md, rq_data_dir(rq), false); blk_mq_delay_run_hw_queue(hctx, 100/*ms*/); return BLK_MQ_RQ_QUEUE_BUSY; } return BLK_MQ_RQ_QUEUE_OK; }