/*
* ibmf_i_issue_pkt():
* Post an IB packet on the specified QP's send queue
*/
int
ibmf_i_issue_pkt(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
ibmf_qp_handle_t ibmf_qp_handle, ibmf_send_wqe_t *send_wqep)
{
int ret;
ibt_status_t status;
ibt_wr_ds_t sgl[1];
ibt_qp_hdl_t ibt_qp_handle;
_NOTE(ASSUMING_PROTECTED(*send_wqep))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))
IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_issue_pkt_start, IBMF_TNF_TRACE, "",
"ibmf_i_issue_pkt() enter, clientp = %p, msg = %p, "
"qp_hdl = %p, swqep = %p\n", tnf_opaque, clientp, clientp,
tnf_opaque, msg, msgimplp, tnf_opaque, ibmf_qp_handle,
ibmf_qp_handle, tnf_opaque, send_wqep, send_wqep);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
ASSERT(MUTEX_NOT_HELD(&clientp->ic_mutex));
/*
* if the qp handle provided in ibmf_send_pkt()
* is not the default qp handle for this client,
* then the wqe must be sent on this qp,
* else use the default qp handle set up during ibmf_register()
*/
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
ibt_qp_handle = clientp->ic_qp->iq_qp_handle;
} else {
ibt_qp_handle =
((ibmf_alt_qp_t *)ibmf_qp_handle)->isq_qp_handle;
}
/* initialize the send WQE */
ibmf_i_init_send_wqe(clientp, msgimplp, sgl, send_wqep,
msgimplp->im_ud_dest, ibt_qp_handle, ibmf_qp_handle);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))
/*
* Issue the wqe to the transport.
* NOTE: ibt_post_send() will not block, so, it is ok
* to hold the msgimpl mutex across this call.
*/
status = ibt_post_send(send_wqep->send_qp_handle, &send_wqep->send_wr,
1, NULL);
if (status != IBT_SUCCESS) {
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_pkt_failed, 1);
mutex_exit(&clientp->ic_kstat_mutex);
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_issue_pkt_err, IBMF_TNF_TRACE, "",
"ibmf_i_issue_pkt(): %s, status = %d\n",
tnf_string, msg, "post send failure",
tnf_uint, ibt_status, status);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(() exit\n");
return (IBMF_TRANSPORT_FAILURE);
}
ret = IBMF_SUCCESS;
/* bump the number of active sends */
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
clientp->ic_sends_active++;
mutex_exit(&clientp->ic_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
} else {
ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;
mutex_enter(&qpp->isq_mutex);
qpp->isq_sends_active++;
mutex_exit(&qpp->isq_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() exit\n");
return (ret);
}
/*
* ibmf_i_client_add_msg():
* Add the message to the client message list
*/
void
ibmf_i_client_add_msg(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp)
{
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_add_msg_start, IBMF_TNF_TRACE, "",
"ibmf_i_client_add_msg(): clientp = 0x%p, msgp = 0x%p\n",
tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp);
ASSERT(MUTEX_NOT_HELD(&msgimplp->im_mutex));
mutex_enter(&clientp->ic_msg_mutex);
/*
* If this is a termination message, add the message to
* the termination message list else add the message
* to the regular message list.
*/
mutex_enter(&msgimplp->im_mutex);
if (msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) {
mutex_exit(&msgimplp->im_mutex);
/* Put the message on the list */
if (clientp->ic_term_msg_list == NULL) {
clientp->ic_term_msg_list = clientp->ic_term_msg_last =
msgimplp;
} else {
msgimplp->im_msg_prev = clientp->ic_term_msg_last;
clientp->ic_term_msg_last->im_msg_next = msgimplp;
clientp->ic_term_msg_last = msgimplp;
}
} else {
mutex_exit(&msgimplp->im_mutex);
/*
* Increment the counter and kstats for active messages
*/
clientp->ic_msgs_active++;
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
/* Put the message on the list */
if (clientp->ic_msg_list == NULL) {
clientp->ic_msg_list = clientp->ic_msg_last = msgimplp;
} else {
msgimplp->im_msg_prev = clientp->ic_msg_last;
clientp->ic_msg_last->im_msg_next = msgimplp;
clientp->ic_msg_last = msgimplp;
}
}
msgimplp->im_msg_next = NULL;
/* Set the message flags to indicate the message is on the list */
mutex_enter(&msgimplp->im_mutex);
msgimplp->im_flags |= IBMF_MSG_FLAGS_ON_LIST;
mutex_exit(&msgimplp->im_mutex);
mutex_exit(&clientp->ic_msg_mutex);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_add_msg_end, IBMF_TNF_TRACE, "",
"ibmf_i_client_add_msg() exit\n");
}
/* ARGSUSED */
void
ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp)
{
ibmf_client_t *clientp, *cclientp;
ibmf_send_wqe_t *send_wqep;
ibmf_qp_handle_t ibmf_qp_handle;
ibmf_alt_qp_t *qpp;
int ret;
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n",
tnf_opaque, cip, cip, tnf_opaque, wcp, wcp);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))
ASSERT(wcp->wc_id != NULL);
ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id));
/* get the IBMF send WQE context */
IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep);
ASSERT(send_wqep != NULL);
/* get the client context */
cclientp = clientp = send_wqep->send_client;
/* Check if this is a completion for a BUSY MAD sent by IBMF */
if (clientp == NULL) {
ibmf_msg_impl_t *msgimplp;
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion(): NULL client\n");
msgimplp = send_wqep->send_msg;
/*
* Deregister registered memory and free it, and
* free up the send WQE context
*/
(void) ibt_deregister_mr(cip->ci_ci_handle,
send_wqep->send_mem_hdl);
kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE);
kmem_free(send_wqep, sizeof (ibmf_send_wqe_t));
/* Free up the message context */
ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest);
ibmf_i_clean_ud_dest_list(cip, B_FALSE);
kmem_free(msgimplp, sizeof (ibmf_msg_impl_t));
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
return;
}
/* get the QP handle */
ibmf_qp_handle = send_wqep->send_ibmf_qp_handle;
qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;
ASSERT(clientp != NULL);
/* decrement the number of active sends */
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
clientp->ic_sends_active--;
mutex_exit(&clientp->ic_mutex);
} else {
mutex_enter(&qpp->isq_mutex);
qpp->isq_sends_active--;
mutex_exit(&qpp->isq_mutex);
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status);
/*
* issue the callback using taskq. If no taskq or if the
* dispatch fails, we do the send processing in the callback context
* which is the interrupt context
*/
if (cclientp->ic_send_taskq == NULL) {
/* Do the processing in callback context */
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
ibmf_i_do_send_cb((void *)send_wqep);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
"ibmf_i_handle_send_completion(): %s\n",
tnf_string, msg, "ci_send_taskq == NULL");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
return;
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
/* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */
if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) {
ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb,
send_wqep, TQ_NOSLEEP);
if (ret == 0) {
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
"ibmf_i_handle_send_completion(): %s\n",
tnf_string, msg, "send: dispatch failed");
ibmf_i_do_send_cb((void *)send_wqep);
}
} else {
ibmf_i_do_send_cb((void *)send_wqep);
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
}
/*
* ibmf_i_notify_client():
* If the transaction is done, call the appropriate callback
*/
void
ibmf_i_notify_client(ibmf_msg_impl_t *msgimplp)
{
ibmf_client_t *clientp;
ibmf_msg_cb_t async_cb;
void *async_cb_arg;
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_start,
IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): msgp = 0x%p\n",
tnf_opaque, msgimplp, msgimplp);
clientp = msgimplp->im_client;
/*
* message is removed so no more threads will find message;
* wait for any current clients to finish
*/
mutex_enter(&msgimplp->im_mutex);
ASSERT(msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE);
/*
* If the message reference count is not zero, then some duplicate
* MAD has arrived for this message. The thread processing the MAD
* found the message on the client's list before this thread was able
* to remove the message from the list. Since, we should not notify
* the client of the transaction completion until all the threads
* working on this message have completed (we don't want the client
* to free the message while a thread is working on it), we let one
* of the other threads notify the client of the completion once
* the message reference count is zero.
*/
if (msgimplp->im_ref_count != 0) {
mutex_exit(&msgimplp->im_mutex);
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client_err, IBMF_TNF_TRACE,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg, "message reference count != 0");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
mutex_exit(&msgimplp->im_mutex);
/*
* Free up the UD dest resource so it is not tied down by
* the message in case the message is not freed immediately.
* Clean up the UD dest list as well so that excess UD dest
* resources are returned to the CI.
*/
if (msgimplp->im_ibmf_ud_dest != NULL) {
ibmf_i_free_ud_dest(clientp, msgimplp);
ibmf_i_clean_ud_dest_list(clientp->ic_myci, B_FALSE);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msgimplp))
if (msgimplp->im_unsolicited == B_TRUE) {
/*
* Do nothing if error status
*/
if (msgimplp->im_msg_status != IBMF_SUCCESS) {
if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
} else {
ibmf_alt_qp_t *qpp =
(ibmf_alt_qp_t *)msgimplp->im_qp_hdl;
mutex_enter(&qpp->isq_mutex);
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
}
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR, "",
"ibmf_i_notify_client(): %s, status = %d\n",
tnf_string, msg, "message status not success",
tnf_opaque, status, msgimplp->im_msg_status);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
/*
* Check to see if
* a callback has been resgistered with the client
* for this unsolicited message.
* If one has been registered, up the recvs active
* count to get the teardown routine to wait until
* this callback is complete.
*/
if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
if ((clientp->ic_recv_cb == NULL) ||
(clientp->ic_flags & IBMF_CLIENT_TEAR_DOWN_CB)) {
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg,
"ibmf_tear_down_recv_cb already occurred");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
clientp->ic_msgs_alloced++;
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1);
mutex_exit(&clientp->ic_kstat_mutex);
async_cb = clientp->ic_recv_cb;
async_cb_arg = clientp->ic_recv_cb_arg;
mutex_exit(&clientp->ic_mutex);
async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp,
async_cb_arg);
mutex_enter(&clientp->ic_mutex);
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
} else {
ibmf_alt_qp_t *qpp =
(ibmf_alt_qp_t *)msgimplp->im_qp_hdl;
mutex_enter(&qpp->isq_mutex);
if ((qpp->isq_recv_cb == NULL) ||
(qpp->isq_flags & IBMF_CLIENT_TEAR_DOWN_CB)) {
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg,
"ibmf_tear_down_recv_cb already occurred");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
async_cb = qpp->isq_recv_cb;
async_cb_arg = qpp->isq_recv_cb_arg;
mutex_exit(&qpp->isq_mutex);
mutex_enter(&clientp->ic_mutex);
clientp->ic_msgs_alloced++;
mutex_exit(&clientp->ic_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1);
mutex_exit(&clientp->ic_kstat_mutex);
async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp,
async_cb_arg);
mutex_enter(&qpp->isq_mutex);
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
}
} else {
/* Solicited transaction processing */
if (msgimplp->im_trans_cb == NULL) {
/* Processing for a blocking transaction */
mutex_enter(&msgimplp->im_mutex);
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_WAIT) {
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "Awaking thread",
tnf_opaque, msgimplp, msgimplp);
cv_signal(&msgimplp->im_trans_cv);
} else {
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "Notify client, no wait",
tnf_opaque, msgimplp, msgimplp);
}
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SIGNALED;
mutex_exit(&msgimplp->im_mutex);
} else {
/* Processing for a non-blocking transaction */
mutex_enter(&msgimplp->im_mutex);
msgimplp->im_flags &= ~IBMF_MSG_FLAGS_BUSY;
mutex_exit(&msgimplp->im_mutex);
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "No thread is blocking",
tnf_opaque, msgimplp, msgimplp);
if (msgimplp->im_trans_cb != NULL) {
msgimplp->im_trans_cb(
(ibmf_handle_t)clientp,
(ibmf_msg_t *)msgimplp,
msgimplp->im_trans_cb_arg);
}
}
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n");
}