/*
* srpt_ch_rsp_comp()
*
* Process a completion for an IB SEND message. A SEND completion
* is for a SRP response packet sent back to the initiator. It
* will not have a STMF SCSI task associated with it if it was
* sent for a rejected IU, or was a task management abort response.
*/
static void
srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu,
ibt_wc_status_t wc_status)
{
stmf_status_t st = STMF_SUCCESS;
ASSERT(iu->iu_ch == ch);
/*
* Process the completion regardless whether it's a failure or
* success. At this point, we've processed as far as we can and
* just need to complete the associated task.
*/
if (wc_status != IBT_SUCCESS) {
SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)",
wc_status);
st = STMF_FAILURE;
if (wc_status != IBT_WC_WR_FLUSHED_ERR) {
srpt_ch_disconnect(ch);
}
}
/*
* If the IU response completion is not associated with
* with a SCSI task, release the IU to return the resource
* and the reference to the channel it holds.
*/
mutex_enter(&iu->iu_lock);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
return;
}
/*
* We should not get a SEND completion where the task has already
* completed aborting and STMF has been informed.
*/
ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0);
/*
* Let STMF know we are done.
*/
mutex_exit(&iu->iu_lock);
stmf_send_status_done(iu->iu_stmf_task, st, STMF_IOF_LPORT_DONE);
}
/*
* srpt_ch_rcq_hdlr()
*/
static void
srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg)
{
ibt_status_t status;
srpt_channel_t *ch = arg;
ibt_wc_t wc[SRPT_RECV_WC_POLL_SIZE];
ibt_wc_t *wcp;
int i;
uint32_t entries;
srpt_iu_t *iu;
uint_t cq_rearmed = 0;
/*
* The channel object will exists while the CQ handler call-back
* is installed.
*/
ASSERT(ch != NULL);
srpt_ch_add_ref(ch);
/*
* If we know a channel disconnect has started do nothing
* and let channel cleanup code recover resources from the CQ.
* We are not concerned about races with the state transition
* since the code will do the correct thing either way. This
* is simply to circumvent rearming the CQ, and it will
* catch the state next time.
*/
rw_enter(&ch->ch_rwlock, RW_READER);
if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) {
SRPT_DPRINTF_L2("ch_rcq_hdlr, channel disconnecting");
rw_exit(&ch->ch_rwlock);
srpt_ch_release_ref(ch, 0);
return;
}
rw_exit(&ch->ch_rwlock);
for (;;) {
status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_RECV_WC_POLL_SIZE,
&entries);
if (status != IBT_SUCCESS) {
if (status != IBT_CQ_EMPTY) {
/*
* This error should not happen. It indicates
* something abnormal has gone wrong and means
* either a hardware or programming logic error.
*/
SRPT_DPRINTF_L2(
"ch_rcq_hdlr, unexpected CQ err(%d)",
status);
srpt_ch_disconnect(ch);
break;
}
/*
* If we have not rearmed the CQ do so now and poll to
* eliminate race; otherwise we are done.
*/
if (cq_rearmed == 0) {
(void) ibt_enable_cq_notify(ch->ch_rcq_hdl,
IBT_NEXT_COMPLETION);
cq_rearmed = 1;
continue;
} else {
break;
}
}
for (wcp = wc, i = 0; i < entries; i++, wcp++) {
/*
* Check wc_status before proceeding. If the
* status indicates a channel problem, stop processing.
*/
if (wcp->wc_status != IBT_WC_SUCCESS) {
if (wcp->wc_status == IBT_WC_WR_FLUSHED_ERR) {
SRPT_DPRINTF_L2(
"ch_rcq, unexpected"
" wc_status err(%d)",
wcp->wc_status);
srpt_ch_disconnect(ch);
goto done;
} else {
/* skip IUs with errors */
SRPT_DPRINTF_L2(
"ch_rcq, ERROR comp(%d)",
wcp->wc_status);
/* XXX - verify not leaking IUs */
continue;
}
}
iu = (srpt_iu_t *)(uintptr_t)wcp->wc_id;
ASSERT(iu != NULL);
/*
* Process the IU.
*/
ASSERT(wcp->wc_type == IBT_WRC_RECV);
srpt_ch_process_iu(ch, iu);
}
}
done:
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_scq_hdlr()
*/
static void
srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg)
{
ibt_status_t status;
srpt_channel_t *ch = arg;
ibt_wc_t wc[SRPT_SEND_WC_POLL_SIZE];
ibt_wc_t *wcp;
int i;
uint32_t cq_rearmed = 0;
uint32_t entries;
srpt_swqe_t *swqe;
ASSERT(ch != NULL);
/* Reference channel for the duration of this call */
srpt_ch_add_ref(ch);
for (;;) {
status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_SEND_WC_POLL_SIZE,
&entries);
if (status != IBT_SUCCESS) {
if (status != IBT_CQ_EMPTY) {
/*
* This error should not happen. It indicates
* something abnormal has gone wrong and means
* either a hardware or programming logic error.
*/
SRPT_DPRINTF_L2(
"ch_scq_hdlr, unexpected CQ err(%d)",
status);
srpt_ch_disconnect(ch);
}
/*
* If we have not rearmed the CQ do so now and poll to
* eliminate race; otherwise we are done.
*/
if (cq_rearmed == 0) {
(void) ibt_enable_cq_notify(ch->ch_scq_hdl,
IBT_NEXT_COMPLETION);
cq_rearmed = 1;
continue;
} else {
break;
}
}
for (wcp = wc, i = 0; i < entries; i++, wcp++) {
/*
* A zero work ID indicates this CQE is associated
* with an intermediate post of a RDMA data transfer
* operation. Since intermediate data requests are
* unsignaled, we should only get these if there was
* an error. No action is required.
*/
if (wcp->wc_id == 0) {
continue;
}
swqe = ch->ch_swqe + wcp->wc_id;
switch (swqe->sw_type) {
case SRPT_SWQE_TYPE_RESP:
srpt_ch_rsp_comp(ch, (srpt_iu_t *)
swqe->sw_addr, wcp->wc_status);
break;
case SRPT_SWQE_TYPE_DATA:
srpt_ch_data_comp(ch, (stmf_data_buf_t *)
swqe->sw_addr, wcp->wc_status);
break;
default:
SRPT_DPRINTF_L2("ch_scq_hdlr, bad type(%d)",
swqe->sw_type);
ASSERT(0);
}
srpt_ch_free_swqe_wrid(ch, wcp->wc_id);
}
}
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_data_comp()
*
* Process an IB completion for a RDMA operation. This completion
* should be associated with the last RDMA operation for any
* data buffer transfer.
*/
static void
srpt_ch_data_comp(srpt_channel_t *ch, stmf_data_buf_t *stmf_dbuf,
ibt_wc_status_t wc_status)
{
srpt_ds_dbuf_t *dbuf;
srpt_iu_t *iu;
stmf_status_t status;
ASSERT(stmf_dbuf != NULL);
dbuf = (srpt_ds_dbuf_t *)stmf_dbuf->db_port_private;
ASSERT(dbuf != NULL);
iu = dbuf->db_iu;
ASSERT(iu != NULL);
ASSERT(iu->iu_ch == ch);
/*
* If work completion indicates non-flush failure, then
* start a channel disconnect (asynchronous) and release
* the reference to the IU. The task will be cleaned
* up with STMF during channel shutdown processing.
*/
if (wc_status != IBT_SUCCESS) {
SRPT_DPRINTF_L2("ch_data_comp, WC status err(%d)",
wc_status);
if (wc_status != IBT_WC_WR_FLUSHED_ERR) {
srpt_ch_disconnect(ch);
}
atomic_dec_32(&iu->iu_sq_posted_cnt);
return;
}
/*
* If STMF has requested this task be aborted, then if this is the
* last I/O operation outstanding, notify STMF the task has been
* aborted and ignore the completion.
*/
mutex_enter(&iu->iu_lock);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) {
scsi_task_t *abort_task = iu->iu_stmf_task;
mutex_exit(&iu->iu_lock);
stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task,
STMF_ABORTED, NULL);
return;
}
/*
* We should not get an RDMA completion where the task has already
* completed aborting and STMF has been informed.
*/
ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0);
/*
* Good completion for last RDMA op associated with a data buffer
* I/O, if specified initiate status otherwise let STMF know we are
* done.
*/
stmf_dbuf->db_xfer_status = STMF_SUCCESS;
mutex_exit(&iu->iu_lock);
DTRACE_SRP_8(xfer__done, srpt_channel_t, ch,
ibt_wr_ds_t, &(dbuf->db_sge), srpt_iu_t, iu,
ibt_send_wr_t, 0, uint32_t, stmf_dbuf->db_data_size,
uint32_t, 0, uint32_t, 0,
uint32_t, (stmf_dbuf->db_flags & DB_DIRECTION_TO_RPORT) ? 1 : 0);
if ((stmf_dbuf->db_flags & DB_SEND_STATUS_GOOD) != 0) {
status = srpt_stp_send_status(dbuf->db_iu->iu_stmf_task, 0);
if (status == STMF_SUCCESS) {
return;
}
stmf_dbuf->db_xfer_status = STMF_FAILURE;
}
stmf_data_xfer_done(dbuf->db_iu->iu_stmf_task, stmf_dbuf, 0);
}
/*
* srpt_ch_rsp_comp()
*
* Process a completion for an IB SEND message. A SEND completion
* is for a SRP response packet sent back to the initiator. It
* will not have a STMF SCSI task associated with it if it was
* sent for a rejected IU, or was a task management abort response.
*/
static void
srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu,
ibt_wc_status_t wc_status)
{
ASSERT(iu->iu_ch == ch);
/*
* If work completion indicates failure, decrement the
* send posted count. If it is a flush error, we are
* done; for all other errors start a channel disconnect.
*/
if (wc_status != IBT_SUCCESS) {
SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)",
wc_status);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (wc_status != IBT_WC_WR_FLUSHED_ERR) {
srpt_ch_disconnect(ch);
}
mutex_enter(&iu->iu_lock);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
} else {
/* cleanup handled in task_free */
mutex_exit(&iu->iu_lock);
}
return;
}
/*
* If the IU response completion is not associated with
* with a SCSI task, release the IU to return the resource
* and the reference to the channel it holds.
*/
mutex_enter(&iu->iu_lock);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
return;
}
/*
* If STMF has requested the IU task be aborted, then notify STMF
* the command is now aborted.
*/
if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) {
scsi_task_t *abort_task = iu->iu_stmf_task;
mutex_exit(&iu->iu_lock);
stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task,
STMF_ABORTED, NULL);
return;
}
/*
* We should not get a SEND completion where the task has already
* completed aborting and STMF has been informed.
*/
ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0);
/*
* Successful status response completion for SCSI task.
* Let STMF know we are done.
*/
mutex_exit(&iu->iu_lock);
stmf_send_status_done(iu->iu_stmf_task, STMF_SUCCESS,
STMF_IOF_LPORT_DONE);
}
