/*
* srpt_ch_rsp_comp()
*
* Process a completion for an IB SEND message. A SEND completion
* is for a SRP response packet sent back to the initiator. It
* will not have a STMF SCSI task associated with it if it was
* sent for a rejected IU, or was a task management abort response.
*/
static void
srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu,
ibt_wc_status_t wc_status)
{
stmf_status_t st = STMF_SUCCESS;
ASSERT(iu->iu_ch == ch);
/*
* Process the completion regardless whether it's a failure or
* success. At this point, we've processed as far as we can and
* just need to complete the associated task.
*/
if (wc_status != IBT_SUCCESS) {
SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)",
wc_status);
st = STMF_FAILURE;
if (wc_status != IBT_WC_WR_FLUSHED_ERR) {
srpt_ch_disconnect(ch);
}
}
/*
* If the IU response completion is not associated with
* with a SCSI task, release the IU to return the resource
* and the reference to the channel it holds.
*/
mutex_enter(&iu->iu_lock);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
return;
}
/*
* We should not get a SEND completion where the task has already
* completed aborting and STMF has been informed.
*/
ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0);
/*
* Let STMF know we are done.
*/
mutex_exit(&iu->iu_lock);
stmf_send_status_done(iu->iu_stmf_task, st, STMF_IOF_LPORT_DONE);
}
/*
* srpt_ch_rcq_hdlr()
*/
static void
srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg)
{
ibt_status_t status;
srpt_channel_t *ch = arg;
ibt_wc_t wc[SRPT_RECV_WC_POLL_SIZE];
ibt_wc_t *wcp;
int i;
uint32_t entries;
srpt_iu_t *iu;
uint_t cq_rearmed = 0;
/*
* The channel object will exists while the CQ handler call-back
* is installed.
*/
ASSERT(ch != NULL);
srpt_ch_add_ref(ch);
/*
* If we know a channel disconnect has started do nothing
* and let channel cleanup code recover resources from the CQ.
* We are not concerned about races with the state transition
* since the code will do the correct thing either way. This
* is simply to circumvent rearming the CQ, and it will
* catch the state next time.
*/
rw_enter(&ch->ch_rwlock, RW_READER);
if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) {
SRPT_DPRINTF_L2("ch_rcq_hdlr, channel disconnecting");
rw_exit(&ch->ch_rwlock);
srpt_ch_release_ref(ch, 0);
return;
}
rw_exit(&ch->ch_rwlock);
for (;;) {
status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_RECV_WC_POLL_SIZE,
&entries);
if (status != IBT_SUCCESS) {
if (status != IBT_CQ_EMPTY) {
/*
* This error should not happen. It indicates
* something abnormal has gone wrong and means
* either a hardware or programming logic error.
*/
SRPT_DPRINTF_L2(
"ch_rcq_hdlr, unexpected CQ err(%d)",
status);
srpt_ch_disconnect(ch);
break;
}
/*
* If we have not rearmed the CQ do so now and poll to
* eliminate race; otherwise we are done.
*/
if (cq_rearmed == 0) {
(void) ibt_enable_cq_notify(ch->ch_rcq_hdl,
IBT_NEXT_COMPLETION);
cq_rearmed = 1;
continue;
} else {
break;
}
}
for (wcp = wc, i = 0; i < entries; i++, wcp++) {
/*
* Check wc_status before proceeding. If the
* status indicates a channel problem, stop processing.
*/
if (wcp->wc_status != IBT_WC_SUCCESS) {
if (wcp->wc_status == IBT_WC_WR_FLUSHED_ERR) {
SRPT_DPRINTF_L2(
"ch_rcq, unexpected"
" wc_status err(%d)",
wcp->wc_status);
srpt_ch_disconnect(ch);
goto done;
} else {
/* skip IUs with errors */
SRPT_DPRINTF_L2(
"ch_rcq, ERROR comp(%d)",
wcp->wc_status);
/* XXX - verify not leaking IUs */
continue;
}
}
iu = (srpt_iu_t *)(uintptr_t)wcp->wc_id;
ASSERT(iu != NULL);
/*
* Process the IU.
*/
ASSERT(wcp->wc_type == IBT_WRC_RECV);
srpt_ch_process_iu(ch, iu);
}
}
done:
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_scq_hdlr()
*/
static void
srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg)
{
ibt_status_t status;
srpt_channel_t *ch = arg;
ibt_wc_t wc[SRPT_SEND_WC_POLL_SIZE];
ibt_wc_t *wcp;
int i;
uint32_t cq_rearmed = 0;
uint32_t entries;
srpt_swqe_t *swqe;
ASSERT(ch != NULL);
/* Reference channel for the duration of this call */
srpt_ch_add_ref(ch);
for (;;) {
status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_SEND_WC_POLL_SIZE,
&entries);
if (status != IBT_SUCCESS) {
if (status != IBT_CQ_EMPTY) {
/*
* This error should not happen. It indicates
* something abnormal has gone wrong and means
* either a hardware or programming logic error.
*/
SRPT_DPRINTF_L2(
"ch_scq_hdlr, unexpected CQ err(%d)",
status);
srpt_ch_disconnect(ch);
}
/*
* If we have not rearmed the CQ do so now and poll to
* eliminate race; otherwise we are done.
*/
if (cq_rearmed == 0) {
(void) ibt_enable_cq_notify(ch->ch_scq_hdl,
IBT_NEXT_COMPLETION);
cq_rearmed = 1;
continue;
} else {
break;
}
}
for (wcp = wc, i = 0; i < entries; i++, wcp++) {
/*
* A zero work ID indicates this CQE is associated
* with an intermediate post of a RDMA data transfer
* operation. Since intermediate data requests are
* unsignaled, we should only get these if there was
* an error. No action is required.
*/
if (wcp->wc_id == 0) {
continue;
}
swqe = ch->ch_swqe + wcp->wc_id;
switch (swqe->sw_type) {
case SRPT_SWQE_TYPE_RESP:
srpt_ch_rsp_comp(ch, (srpt_iu_t *)
swqe->sw_addr, wcp->wc_status);
break;
case SRPT_SWQE_TYPE_DATA:
srpt_ch_data_comp(ch, (stmf_data_buf_t *)
swqe->sw_addr, wcp->wc_status);
break;
default:
SRPT_DPRINTF_L2("ch_scq_hdlr, bad type(%d)",
swqe->sw_type);
ASSERT(0);
}
srpt_ch_free_swqe_wrid(ch, wcp->wc_id);
}
}
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_cleanup()
*/
void
srpt_ch_cleanup(srpt_channel_t *ch)
{
srpt_iu_t *iu;
srpt_iu_t *next;
ibt_wc_t wc;
srpt_target_port_t *tgt;
srpt_channel_t *tgt_ch;
scsi_task_t *iutask;
SRPT_DPRINTF_L3("ch_cleanup, invoked for ch(%p), state(%d)",
(void *)ch, ch->ch_state);
/* add a ref for the channel until we're done */
srpt_ch_add_ref(ch);
tgt = ch->ch_tgt;
ASSERT(tgt != NULL);
/*
* Make certain the channel is in the target ports list of
* known channels and remove it (releasing the target
* ports reference to the channel).
*/
mutex_enter(&tgt->tp_ch_list_lock);
tgt_ch = list_head(&tgt->tp_ch_list);
while (tgt_ch != NULL) {
if (tgt_ch == ch) {
list_remove(&tgt->tp_ch_list, tgt_ch);
srpt_ch_release_ref(tgt_ch, 0);
break;
}
tgt_ch = list_next(&tgt->tp_ch_list, tgt_ch);
}
mutex_exit(&tgt->tp_ch_list_lock);
if (tgt_ch == NULL) {
SRPT_DPRINTF_L2("ch_cleanup, target channel no"
"longer known to target");
srpt_ch_release_ref(ch, 0);
return;
}
rw_enter(&ch->ch_rwlock, RW_WRITER);
ch->ch_state = SRPT_CHANNEL_DISCONNECTING;
rw_exit(&ch->ch_rwlock);
/*
* Don't accept any further incoming requests, and clean
* up the receive queue. The send queue is left alone
* so tasks can finish and clean up (whether normally
* or via abort).
*/
if (ch->ch_rcq_hdl) {
ibt_set_cq_handler(ch->ch_rcq_hdl, NULL, NULL);
while (ibt_poll_cq(ch->ch_rcq_hdl, &wc, 1, NULL) ==
IBT_SUCCESS) {
iu = (srpt_iu_t *)(uintptr_t)wc.wc_id;
SRPT_DPRINTF_L4("ch_cleanup, recovering"
" outstanding RX iu(%p)", (void *)iu);
mutex_enter(&iu->iu_lock);
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
/*
* Channel reference has not yet been added for this
* IU, so do not decrement.
*/
mutex_exit(&iu->iu_lock);
}
}
/*
* Go through the list of outstanding IU for the channel's SCSI
* session and for each either abort or complete an abort.
*/
rw_enter(&ch->ch_rwlock, RW_READER);
if (ch->ch_session != NULL) {
rw_enter(&ch->ch_session->ss_rwlock, RW_READER);
iu = list_head(&ch->ch_session->ss_task_list);
while (iu != NULL) {
next = list_next(&ch->ch_session->ss_task_list, iu);
mutex_enter(&iu->iu_lock);
if (ch == iu->iu_ch) {
if (iu->iu_stmf_task == NULL) {
cmn_err(CE_NOTE,
"ch_cleanup, NULL stmf task");
ASSERT(0);
}
iutask = iu->iu_stmf_task;
} else {
iutask = NULL;
}
mutex_exit(&iu->iu_lock);
if (iutask != NULL) {
SRPT_DPRINTF_L4("ch_cleanup, aborting "
"task(%p)", (void *)iutask);
stmf_abort(STMF_QUEUE_TASK_ABORT, iutask,
STMF_ABORTED, NULL);
}
iu = next;
}
rw_exit(&ch->ch_session->ss_rwlock);
}
rw_exit(&ch->ch_rwlock);
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_process_iu()
*/
static void
srpt_ch_process_iu(srpt_channel_t *ch, srpt_iu_t *iu)
{
srpt_iu_data_t *iud;
int status = 1;
/*
* IU adds reference to channel which will represent a
* a reference by STMF. If for whatever reason the IU
* is not handed off to STMF, then this reference will be
* released. Otherwise, the reference will be released when
* SRP informs STMF that the associated SCSI task is done.
*/
srpt_ch_add_ref(ch);
/*
* Validate login RC channel state. Normally active, if
* not active then we need to handle a possible race between the
* receipt of a implied RTU and CM calling back to notify of the
* state transition.
*/
rw_enter(&ch->ch_rwlock, RW_READER);
if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) {
rw_exit(&ch->ch_rwlock);
goto repost_iu;
}
rw_exit(&ch->ch_rwlock);
iud = iu->iu_buf;
switch (iud->rx_iu.srp_op) {
case SRP_IU_CMD:
status = srpt_ch_srp_cmd(ch, iu);
break;
case SRP_IU_TASK_MGMT:
status = srpt_ch_srp_task_mgmt(ch, iu);
return;
case SRP_IU_I_LOGOUT:
SRPT_DPRINTF_L3("ch_process_iu, SRP INITIATOR LOGOUT");
/*
* Initiators should logout by issuing a CM disconnect
* request (DREQ) with the logout IU in the private data;
* however some initiators have been known to send the
* IU in-band, if this happens just initiate the logout.
* Note that we do not return a response as per the
* specification.
*/
srpt_stp_logout(ch);
break;
case SRP_IU_AER_RSP:
case SRP_IU_CRED_RSP:
default:
/*
* We don't send asynchronous events or ask for credit
* adjustments, so nothing need be done. Log we got an
* unexpected IU but then just repost the IU to the SRQ.
*/
SRPT_DPRINTF_L2("ch_process_iu, invalid IU from initiator,"
" IU opcode(%d)", iud->rx_iu.srp_op);
break;
}
if (status == 0) {
return;
}
repost_iu:
SRPT_DPRINTF_L4("process_iu: reposting iu %p", (void *)iu);
mutex_enter(&iu->iu_lock);
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
}
/*
* srpt_ch_rsp_comp()
*
* Process a completion for an IB SEND message. A SEND completion
* is for a SRP response packet sent back to the initiator. It
* will not have a STMF SCSI task associated with it if it was
* sent for a rejected IU, or was a task management abort response.
*/
static void
srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu,
ibt_wc_status_t wc_status)
{
ASSERT(iu->iu_ch == ch);
/*
* If work completion indicates failure, decrement the
* send posted count. If it is a flush error, we are
* done; for all other errors start a channel disconnect.
*/
if (wc_status != IBT_SUCCESS) {
SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)",
wc_status);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (wc_status != IBT_WC_WR_FLUSHED_ERR) {
srpt_ch_disconnect(ch);
}
mutex_enter(&iu->iu_lock);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
} else {
/* cleanup handled in task_free */
mutex_exit(&iu->iu_lock);
}
return;
}
/*
* If the IU response completion is not associated with
* with a SCSI task, release the IU to return the resource
* and the reference to the channel it holds.
*/
mutex_enter(&iu->iu_lock);
atomic_dec_32(&iu->iu_sq_posted_cnt);
if (iu->iu_stmf_task == NULL) {
srpt_ioc_repost_recv_iu(iu->iu_ioc, iu);
mutex_exit(&iu->iu_lock);
srpt_ch_release_ref(ch, 0);
return;
}
/*
* If STMF has requested the IU task be aborted, then notify STMF
* the command is now aborted.
*/
if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) {
scsi_task_t *abort_task = iu->iu_stmf_task;
mutex_exit(&iu->iu_lock);
stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task,
STMF_ABORTED, NULL);
return;
}
/*
* We should not get a SEND completion where the task has already
* completed aborting and STMF has been informed.
*/
ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0);
/*
* Successful status response completion for SCSI task.
* Let STMF know we are done.
*/
mutex_exit(&iu->iu_lock);
stmf_send_status_done(iu->iu_stmf_task, STMF_SUCCESS,
STMF_IOF_LPORT_DONE);
}