/* Register user buffer memory and initialize passive rdma
* dto descriptor. Total data size is stored in
* ctask->data[ISER_DIR_OUT].data_len
*/
static int
iser_prepare_write_cmd(struct iscsi_cmd_task *ctask,
unsigned int imm_sz,
unsigned int unsol_sz,
unsigned int edtl)
{
struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
struct iser_regd_buf *regd_buf;
int err;
struct iser_dto *send_dto = &iser_ctask->desc.dto;
struct iser_hdr *hdr = &iser_ctask->desc.iser_header;
struct iser_data_buf *buf_out = &iser_ctask->data[ISER_DIR_OUT];
err = iser_dma_map_task_data(iser_ctask,
buf_out,
ISER_DIR_OUT,
DMA_TO_DEVICE);
if (err)
return err;
if (edtl > iser_ctask->data[ISER_DIR_OUT].data_len) {
iser_err("Total data length: %ld, less than EDTL: %d, "
"in WRITE cmd BHS itt: %d, conn: 0x%p\n",
iser_ctask->data[ISER_DIR_OUT].data_len,
edtl, ctask->itt, ctask->conn);
return -EINVAL;
}
err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_OUT);
if (err != 0) {
iser_err("Failed to register write cmd RDMA mem\n");
return err;
}
regd_buf = &iser_ctask->rdma_regd[ISER_DIR_OUT];
if (unsol_sz < edtl) {
hdr->flags |= ISER_WSV;
hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey);
hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz);
iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
"VA:%#llX + unsol:%d\n",
ctask->itt, regd_buf->reg.rkey,
(unsigned long long)regd_buf->reg.va, unsol_sz);
}
if (imm_sz > 0) {
iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
ctask->itt, imm_sz);
iser_dto_add_regd_buff(send_dto,
regd_buf,
0,
imm_sz);
}
return 0;
}
/* Register user buffer memory and initialize passive rdma
* dto descriptor. Data size is stored in
* task->data[ISER_DIR_IN].data_len, Protection size
* os stored in task->prot[ISER_DIR_IN].data_len
*/
static int iser_prepare_read_cmd(struct iscsi_task *task)
{
struct iscsi_iser_task *iser_task = task->dd_data;
struct iser_mem_reg *mem_reg;
int err;
struct iser_hdr *hdr = &iser_task->desc.iser_header;
struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN];
err = iser_dma_map_task_data(iser_task,
buf_in,
ISER_DIR_IN,
DMA_FROM_DEVICE);
if (err)
return err;
if (scsi_prot_sg_count(iser_task->sc)) {
struct iser_data_buf *pbuf_in = &iser_task->prot[ISER_DIR_IN];
err = iser_dma_map_task_data(iser_task,
pbuf_in,
ISER_DIR_IN,
DMA_FROM_DEVICE);
if (err)
return err;
}
err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
if (err) {
iser_err("Failed to set up Data-IN RDMA\n");
return err;
}
mem_reg = &iser_task->rdma_reg[ISER_DIR_IN];
hdr->flags |= ISER_RSV;
hdr->read_stag = cpu_to_be32(mem_reg->rkey);
hdr->read_va = cpu_to_be64(mem_reg->sge.addr);
iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
task->itt, mem_reg->rkey,
(unsigned long long)mem_reg->sge.addr);
return 0;
}
/* Register user buffer memory and initialize passive rdma
* dto descriptor. Total data size is stored in
* iser_ctask->data[ISER_DIR_IN].data_len
*/
static int iser_prepare_read_cmd(struct iscsi_cmd_task *ctask,
unsigned int edtl)
{
struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
struct iser_regd_buf *regd_buf;
int err;
struct iser_hdr *hdr = &iser_ctask->desc.iser_header;
struct iser_data_buf *buf_in = &iser_ctask->data[ISER_DIR_IN];
err = iser_dma_map_task_data(iser_ctask,
buf_in,
ISER_DIR_IN,
DMA_FROM_DEVICE);
if (err)
return err;
if (edtl > iser_ctask->data[ISER_DIR_IN].data_len) {
iser_err("Total data length: %ld, less than EDTL: "
"%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
iser_ctask->data[ISER_DIR_IN].data_len, edtl,
ctask->itt, iser_ctask->iser_conn);
return -EINVAL;
}
err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_IN);
if (err) {
iser_err("Failed to set up Data-IN RDMA\n");
return err;
}
regd_buf = &iser_ctask->rdma_regd[ISER_DIR_IN];
hdr->flags |= ISER_RSV;
hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey);
hdr->read_va = cpu_to_be64(regd_buf->reg.va);
iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
ctask->itt, regd_buf->reg.rkey,
(unsigned long long)regd_buf->reg.va);
return 0;
}
/* Register user buffer memory and initialize passive rdma
* dto descriptor. Data size is stored in
* task->data[ISER_DIR_OUT].data_len, Protection size
* is stored at task->prot[ISER_DIR_OUT].data_len
*/
static int
iser_prepare_write_cmd(struct iscsi_task *task,
unsigned int imm_sz,
unsigned int unsol_sz,
unsigned int edtl)
{
struct iscsi_iser_task *iser_task = task->dd_data;
struct iser_mem_reg *mem_reg;
int err;
struct iser_hdr *hdr = &iser_task->desc.iser_header;
struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT];
struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1];
err = iser_dma_map_task_data(iser_task,
buf_out,
ISER_DIR_OUT,
DMA_TO_DEVICE);
if (err)
return err;
if (scsi_prot_sg_count(iser_task->sc)) {
struct iser_data_buf *pbuf_out = &iser_task->prot[ISER_DIR_OUT];
err = iser_dma_map_task_data(iser_task,
pbuf_out,
ISER_DIR_OUT,
DMA_TO_DEVICE);
if (err)
return err;
}
err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
if (err != 0) {
iser_err("Failed to register write cmd RDMA mem\n");
return err;
}
mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT];
if (unsol_sz < edtl) {
hdr->flags |= ISER_WSV;
hdr->write_stag = cpu_to_be32(mem_reg->rkey);
hdr->write_va = cpu_to_be64(mem_reg->sge.addr + unsol_sz);
iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
"VA:%#llX + unsol:%d\n",
task->itt, mem_reg->rkey,
(unsigned long long)mem_reg->sge.addr, unsol_sz);
}
if (imm_sz > 0) {
iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
task->itt, imm_sz);
tx_dsg->addr = mem_reg->sge.addr;
tx_dsg->length = imm_sz;
tx_dsg->lkey = mem_reg->sge.lkey;
iser_task->desc.num_sge = 2;
}
return 0;
}
/*
* iser_buf_setup() is invoked on the initiator in order to register memory
* on demand for use with the iSER layer.
*/
static idm_status_t
iser_buf_setup(idm_buf_t *idb)
{
iser_conn_t *iser_conn;
iser_chan_t *iser_chan;
iser_hca_t *iser_hca;
iser_buf_t *iser_buf;
int status;
ASSERT(idb->idb_buf != NULL);
iser_conn = (iser_conn_t *)idb->idb_ic->ic_transport_private;
ASSERT(iser_conn != NULL);
iser_hca = iser_conn->ic_chan->ic_hca;
iser_chan = iser_conn->ic_chan;
ASSERT(iser_chan != NULL);
/*
* Memory registration is known to be slow, so for small
* transfers, use pre-registered memory buffers and just
* copy the data into/from them at the appropriate time
*/
if (idb->idb_buflen < ISER_BCOPY_THRESHOLD) {
iser_buf =
kmem_cache_alloc(iser_hca->iser_buf_cache, KM_NOSLEEP);
if (iser_buf == NULL) {
/* Fail over to dynamic registration */
status = iser_reg_rdma_mem(iser_chan->ic_hca, idb);
idb->idb_bufalloc = B_FALSE;
return (status);
}
/*
* Set the allocated data buffer pointer in the IDM buf handle
* Data is to be copied from/to this buffer using bcopy
*/
idb->idb_bufptr = idb->idb_buf;
idb->idb_bufbcopy = B_TRUE;
idb->idb_buf = iser_buf->buf;
/* Set the private buf and reg handles in the IDM buf handle */
idb->idb_buf_private = (void *)iser_buf;
idb->idb_reg_private = (void *)iser_buf->iser_mr;
/* Ensure bufalloc'd flag is set */
idb->idb_bufalloc = B_TRUE;
return (IDM_STATUS_SUCCESS);
} else {
/* Dynamically register the memory passed in on the idb */
status = iser_reg_rdma_mem(iser_chan->ic_hca, idb);
/* Ensure bufalloc'd flag is unset */
idb->idb_bufalloc = B_FALSE;
return (status);
}
}