static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, bool reserved) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; struct socket *sock = NULL; spin_lock(&nbd->sock_lock); set_bit(NBD_TIMEDOUT, &nbd->runtime_flags); if (nbd->sock) { sock = nbd->sock; get_file(sock->file); } spin_unlock(&nbd->sock_lock); if (sock) { kernel_sock_shutdown(sock, SHUT_RDWR); sockfd_put(sock); } req->errors++; dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n"); return BLK_EH_HANDLED; }
static void null_softirq_done_fn(struct request *rq) { if (queue_mode == NULL_Q_MQ) end_cmd(blk_mq_rq_to_pdu(rq)); else end_cmd(rq->special); }
static void bsg_exit_rq(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx) { struct bsg_job *job = blk_mq_rq_to_pdu(req); kfree(job->reply); }
static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) { struct request *rq; rq = blk_mq_alloc_request(dd->queue, 0, __GFP_RECLAIM, true); return blk_mq_rq_to_pdu(rq); }
/** * bsg_queue_rq - generic handler for bsg requests * @hctx: hardware queue * @bd: queue data * * On error the create_bsg_job function should return a -Exyz error value * that will be set to ->result. * * Drivers/subsys should pass this to the queue init function. */ static blk_status_t bsg_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request_queue *q = hctx->queue; struct device *dev = q->queuedata; struct request *req = bd->rq; struct bsg_set *bset = container_of(q->tag_set, struct bsg_set, tag_set); int ret; blk_mq_start_request(req); if (!get_device(dev)) return BLK_STS_IOERR; if (!bsg_prepare_job(dev, req)) return BLK_STS_IOERR; ret = bset->job_fn(blk_mq_rq_to_pdu(req)); if (ret) return BLK_STS_IOERR; put_device(dev); return BLK_STS_OK; }
/** * bsg_prepare_job - create the bsg_job structure for the bsg request * @dev: device that is being sent the bsg request * @req: BSG request that needs a job structure */ static bool bsg_prepare_job(struct device *dev, struct request *req) { struct bsg_job *job = blk_mq_rq_to_pdu(req); int ret; job->timeout = req->timeout; if (req->bio) { ret = bsg_map_buffer(&job->request_payload, req); if (ret) goto failjob_rls_job; } if (job->bidi_rq) { ret = bsg_map_buffer(&job->reply_payload, job->bidi_rq); if (ret) goto failjob_rls_rqst_payload; } job->dev = dev; /* take a reference for the request */ get_device(job->dev); kref_init(&job->kref); return true; failjob_rls_rqst_payload: kfree(job->request_payload.sg_list); failjob_rls_job: job->result = -ENOMEM; return false; }
static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, unsigned int tag) { struct request *rq = mtip_rq_from_tag(dd, tag); return blk_mq_rq_to_pdu(rq); }
static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq) { struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); cmd->rq = rq; cmd->nq = hctx->driver_data; null_handle_cmd(cmd); return BLK_MQ_RQ_QUEUE_OK; }
static inline void virtblk_scsi_reques_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); struct virtio_blk *vblk = req->q->queuedata; struct scsi_request *sreq = &vbr->sreq; sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual); sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len); req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors); }
/* called right before the request is given to the request_queue user */ static void bsg_initialize_rq(struct request *req) { struct bsg_job *job = blk_mq_rq_to_pdu(req); void *reply = job->reply; memset(job, 0, sizeof(*job)); job->reply = reply; job->reply_len = SCSI_SENSE_BUFFERSIZE; job->dd_data = job + 1; }
/* called right after the request is allocated for the request_queue */ static int bsg_init_rq(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node) { struct bsg_job *job = blk_mq_rq_to_pdu(req); job->reply = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); if (!job->reply) return -ENOMEM; return 0; }
static void bsg_transport_free_rq(struct request *rq) { struct bsg_job *job = blk_mq_rq_to_pdu(rq); if (job->bidi_rq) { blk_rq_unmap_user(job->bidi_bio); blk_put_request(job->bidi_rq); } kfree(job->request); }
static int null_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); cmd->rq = bd->rq; cmd->nq = hctx->driver_data; blk_mq_start_request(bd->rq); null_handle_cmd(cmd); return BLK_MQ_RQ_QUEUE_OK; }
static inline void virtblk_request_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); switch (req_op(req)) { case REQ_OP_SCSI_IN: case REQ_OP_SCSI_OUT: virtblk_scsi_request_done(req); break; } blk_mq_end_request(req, virtblk_result(vbr)); }
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); struct mapped_device *md = tio->md; struct dm_target *ti = md->immutable_target; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); } /* * On suspend dm_stop_queue() handles stopping the blk-mq * request_queue BUT: even though the hw_queues are marked * BLK_MQ_S_STOPPED at that point there is still a race that * is allowing block/blk-mq.c to call ->queue_rq against a * hctx that it really shouldn't. The following check guards * against this rarity (albeit _not_ race-free). */ if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state))) return BLK_MQ_RQ_QUEUE_BUSY; if (ti->type->busy && ti->type->busy(ti)) return BLK_MQ_RQ_QUEUE_BUSY; dm_start_request(md, rq); /* Init tio using md established in .init_request */ init_tio(tio, rq, md); /* * Establish tio->ti before calling map_request(). */ tio->ti = ti; /* Direct call is fine since .queue_rq allows allocations */ if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { /* Undo dm_start_request() before requeuing */ rq_end_stats(md, rq); rq_completed(md, rq_data_dir(rq), false); return BLK_MQ_RQ_QUEUE_BUSY; } return BLK_MQ_RQ_QUEUE_OK; }
static inline void virtblk_request_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); int error = virtblk_result(vbr); if (req->cmd_type == REQ_TYPE_BLOCK_PC) { req->resid_len = vbr->in_hdr.residual; req->sense_len = vbr->in_hdr.sense_len; req->errors = vbr->in_hdr.errors; } else if (req->cmd_type == REQ_TYPE_SPECIAL) { req->errors = (error != 0); } blk_mq_end_io(req, error); }
static int bsg_transport_complete_rq(struct request *rq, struct sg_io_v4 *hdr) { struct bsg_job *job = blk_mq_rq_to_pdu(rq); int ret = 0; /* * The assignments below don't make much sense, but are kept for * bug by bug backwards compatibility: */ hdr->device_status = job->result & 0xff; hdr->transport_status = host_byte(job->result); hdr->driver_status = driver_byte(job->result); hdr->info = 0; if (hdr->device_status || hdr->transport_status || hdr->driver_status) hdr->info |= SG_INFO_CHECK; hdr->response_len = 0; if (job->result < 0) { /* we're only returning the result field in the reply */ job->reply_len = sizeof(u32); ret = job->result; } if (job->reply_len && hdr->response) { int len = min(hdr->max_response_len, job->reply_len); if (copy_to_user(uptr64(hdr->response), job->reply, len)) ret = -EFAULT; else hdr->response_len = len; } /* we assume all request payload was transferred, residual == 0 */ hdr->dout_resid = 0; if (job->bidi_rq) { unsigned int rsp_len = job->reply_payload.payload_len; if (WARN_ON(job->reply_payload_rcv_len > rsp_len)) hdr->din_resid = 0; else hdr->din_resid = rsp_len - job->reply_payload_rcv_len; } else { hdr->din_resid = 0; } return ret; }
static inline void virtblk_request_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); struct virtio_blk *vblk = req->q->queuedata; int error = virtblk_result(vbr); if (req->cmd_type == REQ_TYPE_BLOCK_PC) { req->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual); req->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len); req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors); } else if (req->cmd_type == REQ_TYPE_DRV_PRIV) { req->errors = (error != 0); } blk_mq_end_request(req, error); }
static inline void virtblk_request_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); int error = virtblk_result(vbr); switch (req_op(req)) { case REQ_OP_SCSI_IN: case REQ_OP_SCSI_OUT: virtblk_scsi_reques_done(req); break; case REQ_OP_DRV_IN: req->errors = (error != 0); break; } blk_mq_end_request(req, error); }
static int __dm_rq_init_rq(struct mapped_device *md, struct request *rq) { struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); /* * Must initialize md member of tio, otherwise it won't * be available in dm_mq_queue_rq. */ tio->md = md; if (md->init_tio_pdu) { /* target-specific per-io data is immediately after the tio */ tio->info.ptr = tio + 1; } return 0; }
static int null_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); if (irqmode == NULL_IRQ_TIMER) { hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cmd->timer.function = null_cmd_timer_expired; } cmd->rq = bd->rq; cmd->nq = hctx->driver_data; blk_mq_start_request(bd->rq); null_handle_cmd(cmd); return BLK_MQ_RQ_QUEUE_OK; }
/* return id (s/n) string for *disk to *id_str */ static int virtblk_get_id(struct gendisk *disk, char *id_str) { struct virtio_blk *vblk = disk->private_data; struct request_queue *q = vblk->disk->queue; struct request *req; int err; req = blk_get_request(q, REQ_OP_DRV_IN, GFP_KERNEL); if (IS_ERR(req)) return PTR_ERR(req); err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); if (err) goto out; blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); out: blk_put_request(req); return err; }
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); struct mapped_device *md = tio->md; struct dm_target *ti = md->immutable_target; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); } if (ti->type->busy && ti->type->busy(ti)) return BLK_MQ_RQ_QUEUE_BUSY; dm_start_request(md, rq); /* Init tio using md established in .init_request */ init_tio(tio, rq, md); /* * Establish tio->ti before calling map_request(). */ tio->ti = ti; /* Direct call is fine since .queue_rq allows allocations */ if (map_request(tio) == DM_MAPIO_REQUEUE) { /* Undo dm_start_request() before requeuing */ rq_end_stats(md, rq); rq_completed(md, rq_data_dir(rq), false); blk_mq_delay_run_hw_queue(hctx, 100/*ms*/); return BLK_MQ_RQ_QUEUE_BUSY; } return BLK_MQ_RQ_QUEUE_OK; }
static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, bool reserved) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n"); set_bit(NBD_TIMEDOUT, &nbd->runtime_flags); req->errors++; /* * If our disconnect packet times out then we're already holding the * config_lock and could deadlock here, so just set an error and return, * we'll handle shutting everything down later. */ if (req->cmd_type == REQ_TYPE_DRV_PRIV) return BLK_EH_HANDLED; mutex_lock(&nbd->config_lock); sock_shutdown(nbd); mutex_unlock(&nbd->config_lock); return BLK_EH_HANDLED; }
static int bsg_transport_fill_hdr(struct request *rq, struct sg_io_v4 *hdr, fmode_t mode) { struct bsg_job *job = blk_mq_rq_to_pdu(rq); int ret; job->request_len = hdr->request_len; job->request = memdup_user(uptr64(hdr->request), hdr->request_len); if (IS_ERR(job->request)) return PTR_ERR(job->request); if (hdr->dout_xfer_len && hdr->din_xfer_len) { job->bidi_rq = blk_get_request(rq->q, REQ_OP_SCSI_IN, 0); if (IS_ERR(job->bidi_rq)) { ret = PTR_ERR(job->bidi_rq); goto out; } ret = blk_rq_map_user(rq->q, job->bidi_rq, NULL, uptr64(hdr->din_xferp), hdr->din_xfer_len, GFP_KERNEL); if (ret) goto out_free_bidi_rq; job->bidi_bio = job->bidi_rq->bio; } else { job->bidi_rq = NULL; job->bidi_bio = NULL; } return 0; out_free_bidi_rq: if (job->bidi_rq) blk_put_request(job->bidi_rq); out: kfree(job->request); return ret; }
static struct dm_rq_target_io *tio_from_request(struct request *rq) { return blk_mq_rq_to_pdu(rq); }
static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct virtio_blk *vblk = hctx->queue->queuedata; struct request *req = bd->rq; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); unsigned long flags; unsigned int num; int qid = hctx->queue_num; int err; bool notify = false; BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); vbr->req = req; if (req->cmd_flags & REQ_FLUSH) { vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_FLUSH); vbr->out_hdr.sector = 0; vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); } else { switch (req->cmd_type) { case REQ_TYPE_FS: vbr->out_hdr.type = 0; vbr->out_hdr.sector = cpu_to_virtio64(vblk->vdev, blk_rq_pos(vbr->req)); vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); break; case REQ_TYPE_BLOCK_PC: vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_SCSI_CMD); vbr->out_hdr.sector = 0; vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); break; case REQ_TYPE_DRV_PRIV: vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID); vbr->out_hdr.sector = 0; vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); break; default: /* We don't put anything else in the queue. */ BUG(); } } blk_mq_start_request(req); num = blk_rq_map_sg(hctx->queue, vbr->req, vbr->sg); if (num) { if (rq_data_dir(vbr->req) == WRITE) vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); else vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); } spin_lock_irqsave(&vblk->vqs[qid].lock, flags); err = __virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); if (err) { virtqueue_kick(vblk->vqs[qid].vq); blk_mq_stop_hw_queue(hctx); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); /* Out of mem doesn't actually happen, since we fall back * to direct descriptors */ if (err == -ENOMEM || err == -ENOSPC) return BLK_MQ_RQ_QUEUE_BUSY; return BLK_MQ_RQ_QUEUE_ERROR; } if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) notify = true; spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); if (notify) virtqueue_notify(vblk->vqs[qid].vq); return BLK_MQ_RQ_QUEUE_OK; }
static void null_softirq_done_fn(struct request *rq) { end_cmd(blk_mq_rq_to_pdu(rq)); }
static struct dm_rq_target_io *tio_from_request(struct request *rq) { return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); }
static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct virtio_blk *vblk = hctx->queue->queuedata; struct request *req = bd->rq; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); unsigned long flags; unsigned int num; int qid = hctx->queue_num; int err; bool notify = false; u32 type; BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); switch (req_op(req)) { case REQ_OP_READ: case REQ_OP_WRITE: type = 0; break; case REQ_OP_FLUSH: type = VIRTIO_BLK_T_FLUSH; break; case REQ_OP_SCSI_IN: case REQ_OP_SCSI_OUT: type = VIRTIO_BLK_T_SCSI_CMD; break; case REQ_OP_DRV_IN: type = VIRTIO_BLK_T_GET_ID; break; default: WARN_ON_ONCE(1); return BLK_MQ_RQ_QUEUE_ERROR; } vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type); vbr->out_hdr.sector = type ? 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req)); vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req)); blk_mq_start_request(req); num = blk_rq_map_sg(hctx->queue, req, vbr->sg); if (num) { if (rq_data_dir(req) == WRITE) vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); else vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); } spin_lock_irqsave(&vblk->vqs[qid].lock, flags); if (req_op(req) == REQ_OP_SCSI_IN || req_op(req) == REQ_OP_SCSI_OUT) err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num); else err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); if (err) { virtqueue_kick(vblk->vqs[qid].vq); blk_mq_stop_hw_queue(hctx); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); /* Out of mem doesn't actually happen, since we fall back * to direct descriptors */ if (err == -ENOMEM || err == -ENOSPC) return BLK_MQ_RQ_QUEUE_BUSY; return BLK_MQ_RQ_QUEUE_ERROR; } if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) notify = true; spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); if (notify) virtqueue_notify(vblk->vqs[qid].vq); return BLK_MQ_RQ_QUEUE_OK; }