/*
* ibmf_i_send_single_pkt():
* Send a single IB packet. Only used to send non-RMPP packets.
*/
int
ibmf_i_send_single_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle,
ibmf_msg_impl_t *msgimplp, int block)
{
int status;
IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_start,
IBMF_TNF_TRACE, "",
"ibmf_i_send_single_pkt(): clientp = 0x%p, qp_hdl = 0x%p, "
"msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp,
tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp,
tnf_uint, block, block);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
status = ibmf_i_send_pkt(clientp, ibmf_qp_handle, msgimplp, block);
if (status != IBMF_SUCCESS) {
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_send_single_pkt_err, IBMF_TNF_ERROR, "",
"ibmf_i_send_single_pkt(): %s, msgp = 0x%p\n",
tnf_string, msg, "unable to send packet",
tnf_uint, status, status);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_send_single_pkt_end, IBMF_TNF_TRACE, "",
"ibmf_i_send_single_pkt() exit\n");
return (status);
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt() exit\n");
return (IBMF_SUCCESS);
}
/* ARGSUSED */
static void
ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp,
ibmf_send_wqe_t *send_wqep)
{
IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_start,
IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): ibmf_hdl = 0x%p "
"msgp = %p, send_wqep = 0x%p, msg_flags = 0x%x\n",
tnf_opaque, ibmf_hdl, ibmf_handle, tnf_opaque, msgimplp, msgimplp,
tnf_opaque, send_wqep, send_wqep,
tnf_opaque, msg_flags, msgimplp->im_flags);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
/*
* For RMPP transactions, we only care about the final packet of the
* transaction. For others, the code does not need to wait for the send
* completion (although bad things can happen if it never occurs).
* The final packets of a transaction are sent when the state is either
* ABORT or RECEVR_TERMINATE.
* Don't mark the transaction as send_done if there are still more
* packets to be sent, including doing the second part of a double-sided
* transaction.
*/
if ((msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) ||
(msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)) {
IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_compl, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_compl(): %s msgp = %p, rmpp_state = 0x%x\n",
tnf_string, msg, "Received send callback for RMPP trans",
tnf_opaque, msg, msgimplp,
tnf_opaque, rmpp_state, msgimplp->im_rmpp_ctx.rmpp_state);
/*
* For ABORT state, we should not return control to
* the client from the send completion handler.
* Control should be returned in the error timeout handler.
*
* The exception is when the IBMF_TRANS_STATE_FLAG_RECV_DONE
* flag has already been set. This flag is set when
* ibmf_i_terminate_transaction is called from one of the
* three timeout handlers. In this case return control from
* here.
*/
if (msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_ABORT) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SEND_DONE;
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_RECV_DONE) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_DONE;
}
}
if ((msgimplp->im_rmpp_ctx.rmpp_state ==
IBMF_RMPP_STATE_RECEVR_TERMINATE) ||
(msgimplp->im_rmpp_ctx.rmpp_state ==
IBMF_RMPP_STATE_DONE)) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SEND_DONE;
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_RECV_DONE) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_DONE;
}
}
/*
* If the transaction is a send-only RMPP, then
* set the SEND_DONE flag on every send completion
* as long as there are no outstanding ones.
* This is needed so that the transaction can return
* in the receive path, where ibmf_i_terminate_transaction
* is called from ibmf_i_rmpp_sender_active_flow,
* after checking if the SEND_DONE flag is set.
* When a new MAD is sent as part of the RMPP transaction,
* the SEND_DONE flag will get reset.
* The RECV_DONE indicates that the last ACK was received.
*/
if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) {
if (msgimplp->im_pending_send_compls == 0) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SEND_DONE;
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_RECV_DONE) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_DONE;
}
}
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_compl() exit\n");
return;
}
/*
* Only non-RMPP send completion gets here.
* If the send is a single-packet send that does not use RMPP, and if
* the transaction is not a sequenced transaction, call the transaction
* callback handler after flagging the transaction as done. If the
* message is sequenced, start a timer to bound the wait for the first
* data packet of the response.
*/
if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) {
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_compl, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_compl(): %s msgp = %p\n", tnf_string, msg,
"Sequenced transaction, setting response timer",
tnf_opaque, msg, msgimplp);
/*
* Check if the send completion already occured,
* which could imply that this is a send completion
* for some previous transaction that has come in very late.
* In this case exit here.
*/
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_SEND_DONE) {
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_compl() exit, "
"Duplicate SEND completion\n");
return;
}
/* mark as send_compl happened */
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SEND_DONE;
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_RECV_DONE) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_DONE;
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_compl() exit, RECV_DONE\n");
return;
}
/*
* check if response was received before send
* completion
*/
if (((msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_DONE) == 0) &&
((msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_RECV_ACTIVE) == 0)) {
/* set timer for first packet of response */
ibmf_i_set_timer(ibmf_i_send_timeout, msgimplp,
IBMF_RESP_TIMER);
}
} else {
msgimplp->im_msg_status = IBMF_SUCCESS;
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SEND_DONE;
msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end,
IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit\n");
}
/*
* ibmf_i_issue_pkt():
* Post an IB packet on the specified QP's send queue
*/
int
ibmf_i_issue_pkt(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
ibmf_qp_handle_t ibmf_qp_handle, ibmf_send_wqe_t *send_wqep)
{
int ret;
ibt_status_t status;
ibt_wr_ds_t sgl[1];
ibt_qp_hdl_t ibt_qp_handle;
_NOTE(ASSUMING_PROTECTED(*send_wqep))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))
IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_issue_pkt_start, IBMF_TNF_TRACE, "",
"ibmf_i_issue_pkt() enter, clientp = %p, msg = %p, "
"qp_hdl = %p, swqep = %p\n", tnf_opaque, clientp, clientp,
tnf_opaque, msg, msgimplp, tnf_opaque, ibmf_qp_handle,
ibmf_qp_handle, tnf_opaque, send_wqep, send_wqep);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
ASSERT(MUTEX_NOT_HELD(&clientp->ic_mutex));
/*
* if the qp handle provided in ibmf_send_pkt()
* is not the default qp handle for this client,
* then the wqe must be sent on this qp,
* else use the default qp handle set up during ibmf_register()
*/
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
ibt_qp_handle = clientp->ic_qp->iq_qp_handle;
} else {
ibt_qp_handle =
((ibmf_alt_qp_t *)ibmf_qp_handle)->isq_qp_handle;
}
/* initialize the send WQE */
ibmf_i_init_send_wqe(clientp, msgimplp, sgl, send_wqep,
msgimplp->im_ud_dest, ibt_qp_handle, ibmf_qp_handle);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))
/*
* Issue the wqe to the transport.
* NOTE: ibt_post_send() will not block, so, it is ok
* to hold the msgimpl mutex across this call.
*/
status = ibt_post_send(send_wqep->send_qp_handle, &send_wqep->send_wr,
1, NULL);
if (status != IBT_SUCCESS) {
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_pkt_failed, 1);
mutex_exit(&clientp->ic_kstat_mutex);
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_issue_pkt_err, IBMF_TNF_TRACE, "",
"ibmf_i_issue_pkt(): %s, status = %d\n",
tnf_string, msg, "post send failure",
tnf_uint, ibt_status, status);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(() exit\n");
return (IBMF_TRANSPORT_FAILURE);
}
ret = IBMF_SUCCESS;
/* bump the number of active sends */
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
clientp->ic_sends_active++;
mutex_exit(&clientp->ic_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
} else {
ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;
mutex_enter(&qpp->isq_mutex);
qpp->isq_sends_active++;
mutex_exit(&qpp->isq_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() exit\n");
return (ret);
}
/*
* ibmf_i_do_send_cb():
* Do the send completion processing
*/
static void
ibmf_i_do_send_cb(void *taskq_arg)
{
ibmf_ci_t *cip;
ibmf_msg_impl_t *msgimplp;
ibmf_client_t *clientp;
ibmf_send_wqe_t *send_wqep;
boolean_t found;
int msg_trans_state_flags, msg_flags;
uint_t ref_cnt;
ibmf_qp_handle_t ibmf_qp_handle;
struct kmem_cache *kmem_cachep;
timeout_id_t msg_rp_unset_id, msg_tr_unset_id;
timeout_id_t msg_rp_set_id, msg_tr_set_id;
ibmf_alt_qp_t *altqp;
boolean_t inc_refcnt;
send_wqep = taskq_arg;
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_do_send_cb_start, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb() enter, send_wqep = %p\n",
tnf_opaque, send_wqep, send_wqep);
clientp = send_wqep->send_client;
cip = clientp->ic_myci;
msgimplp = send_wqep->send_msg;
/* get the QP handle */
ibmf_qp_handle = send_wqep->send_ibmf_qp_handle;
/* Get the WQE kmem cache pointer based on the QP type */
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT)
kmem_cachep = cip->ci_send_wqes_cache;
else {
altqp = (ibmf_alt_qp_t *)ibmf_qp_handle;
kmem_cachep = altqp->isq_send_wqes_cache;
}
/* Look for a message in the client's message list */
inc_refcnt = B_TRUE;
found = ibmf_i_find_msg_client(clientp, msgimplp, inc_refcnt);
/*
* If the message context was not found, then it's likely
* been freed up. So, do nothing in this timeout handler
*/
if (found == B_FALSE) {
kmem_cache_free(kmem_cachep, send_wqep);
mutex_enter(&cip->ci_mutex);
IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
mutex_exit(&cip->ci_mutex);
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&cip->ci_mutex);
cip->ci_wqes_alloced--;
if (cip->ci_wqes_alloced == 0)
cv_signal(&cip->ci_wqes_cv);
mutex_exit(&cip->ci_mutex);
} else {
mutex_enter(&altqp->isq_mutex);
altqp->isq_wqes_alloced--;
if (altqp->isq_wqes_alloced == 0)
cv_signal(&altqp->isq_wqes_cv);
mutex_exit(&altqp->isq_mutex);
}
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb(): %s\n", tnf_string, msg,
"Message not found, return without processing send cb");
return;
}
/* Grab the message context lock */
mutex_enter(&msgimplp->im_mutex);
/*
* Decrement the count of pending send completions for
* this transaction
*/
msgimplp->im_pending_send_compls -= 1;
/*
* If the pending send completions is not zero, then we must
* not attempt to notify the client of a transaction completion
* in this instance of the send completion handler. Notification
* of transaction completion should be provided only by the
* last send completion so that all send completions are accounted
* for before the client is notified and subsequently attempts to
* reuse the message for an other transaction.
* If this is not done, the message may be reused while the
* send WR from the old transaction is still active in the QP's WQ.
* This could result in an attempt to modify the address handle with
* information for the new transaction which could be potentially
* incompatible, such as an incorrect port number. Such an
* incompatible modification of the address handle of the old
* transaction could result in a QP error.
*/
if (msgimplp->im_pending_send_compls != 0) {
IBMF_MSG_DECR_REFCNT(msgimplp);
mutex_exit(&msgimplp->im_mutex);
kmem_cache_free(kmem_cachep, send_wqep);
mutex_enter(&cip->ci_mutex);
IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
mutex_exit(&cip->ci_mutex);
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&cip->ci_mutex);
cip->ci_wqes_alloced--;
if (cip->ci_wqes_alloced == 0)
cv_signal(&cip->ci_wqes_cv);
mutex_exit(&cip->ci_mutex);
} else {
mutex_enter(&altqp->isq_mutex);
altqp->isq_wqes_alloced--;
if (altqp->isq_wqes_alloced == 0)
cv_signal(&altqp->isq_wqes_cv);
mutex_exit(&altqp->isq_mutex);
}
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb(): %s\n", tnf_string, msg,
"Message found with pending send completions, "
"return without processing send cb");
return;
}
/*
* If the message has been marked unitialized or done
* release the message mutex and return
*/
if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) ||
(msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) {
IBMF_MSG_DECR_REFCNT(msgimplp);
msg_trans_state_flags = msgimplp->im_trans_state_flags;
msg_flags = msgimplp->im_flags;
ref_cnt = msgimplp->im_ref_count;
mutex_exit(&msgimplp->im_mutex);
/*
* This thread may notify the client only if the
* transaction is done, the message has been removed
* from the client's message list, and the message
* reference count is 0.
* If the transaction is done, and the message reference
* count = 0, there is still a possibility that a
* packet could arrive for the message and its reference
* count increased if the message is still on the list.
* If the message is still on the list, it will be
* removed by a call to ibmf_i_client_rem_msg() at
* the completion point of the transaction.
* So, the reference count should be checked after the
* message has been removed.
*/
if ((msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) &&
!(msg_flags & IBMF_MSG_FLAGS_ON_LIST) &&
(ref_cnt == 0)) {
ibmf_i_notify_sequence(clientp, msgimplp, msg_flags);
}
kmem_cache_free(kmem_cachep, send_wqep);
mutex_enter(&cip->ci_mutex);
IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
mutex_exit(&cip->ci_mutex);
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&cip->ci_mutex);
cip->ci_wqes_alloced--;
if (cip->ci_wqes_alloced == 0)
cv_signal(&cip->ci_wqes_cv);
mutex_exit(&cip->ci_mutex);
} else {
mutex_enter(&altqp->isq_mutex);
altqp->isq_wqes_alloced--;
if (altqp->isq_wqes_alloced == 0)
cv_signal(&altqp->isq_wqes_cv);
mutex_exit(&altqp->isq_mutex);
}
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb(): %s, msg = %p\n", tnf_string, msg,
"Message marked for removal, return without processing "
"send cb", tnf_opaque, msgimplp, msgimplp);
return;
}
/* Perform send completion processing of the message context */
ibmf_i_do_send_compl((ibmf_handle_t)clientp, msgimplp, send_wqep);
msg_rp_unset_id = msg_tr_unset_id = msg_rp_set_id = msg_tr_set_id = 0;
/* Save the message flags before releasing the mutex */
msg_trans_state_flags = msgimplp->im_trans_state_flags;
msg_flags = msgimplp->im_flags;
msg_rp_unset_id = msgimplp->im_rp_unset_timeout_id;
msg_tr_unset_id = msgimplp->im_tr_unset_timeout_id;
msgimplp->im_rp_unset_timeout_id = 0;
msgimplp->im_tr_unset_timeout_id = 0;
/*
* Decrement the message reference count
* This count was inceremented when the message was found on the
* client's message list
*/
IBMF_MSG_DECR_REFCNT(msgimplp);
if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) {
if (msgimplp->im_rp_timeout_id != 0) {
msg_rp_set_id = msgimplp->im_rp_timeout_id;
msgimplp->im_rp_timeout_id = 0;
}
if (msgimplp->im_tr_timeout_id != 0) {
msg_tr_set_id = msgimplp->im_tr_timeout_id;
msgimplp->im_tr_timeout_id = 0;
}
}
mutex_exit(&msgimplp->im_mutex);
if (msg_rp_unset_id != 0) {
(void) untimeout(msg_rp_unset_id);
}
if (msg_tr_unset_id != 0) {
(void) untimeout(msg_tr_unset_id);
}
if (msg_rp_set_id != 0) {
(void) untimeout(msg_rp_set_id);
}
if (msg_tr_set_id != 0) {
(void) untimeout(msg_tr_set_id);
}
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb(): %s, msg = %p\n",
tnf_string, msg, "Send callback done. Dec ref count",
tnf_opaque, msgimplp, msgimplp);
/*
* If the transaction is done, signal the block thread if the
* transaction is blocking, or call the client's transaction done
* notification callback
*/
if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) {
/* Remove the message from the client's message list */
ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);
/*
* Notify the client if the message reference count is zero.
* At this point, we know that the transaction is done and
* the message has been removed from the client's message list.
* So, we only need to make sure the reference count is zero
* before notifying the client.
*/
if (ref_cnt == 0) {
ibmf_i_notify_sequence(clientp, msgimplp, msg_flags);
}
}
kmem_cache_free(kmem_cachep, send_wqep);
mutex_enter(&cip->ci_mutex);
IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
mutex_exit(&cip->ci_mutex);
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&cip->ci_mutex);
cip->ci_wqes_alloced--;
if (cip->ci_wqes_alloced == 0)
cv_signal(&cip->ci_wqes_cv);
mutex_exit(&cip->ci_mutex);
} else {
mutex_enter(&altqp->isq_mutex);
altqp->isq_wqes_alloced--;
if (altqp->isq_wqes_alloced == 0)
cv_signal(&altqp->isq_wqes_cv);
mutex_exit(&altqp->isq_mutex);
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_do_send_cb_end, IBMF_TNF_TRACE, "",
"ibmf_i_do_send_cb() exit\n");
}
/* ARGSUSED */
void
ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp)
{
ibmf_client_t *clientp, *cclientp;
ibmf_send_wqe_t *send_wqep;
ibmf_qp_handle_t ibmf_qp_handle;
ibmf_alt_qp_t *qpp;
int ret;
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n",
tnf_opaque, cip, cip, tnf_opaque, wcp, wcp);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))
ASSERT(wcp->wc_id != NULL);
ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id));
/* get the IBMF send WQE context */
IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep);
ASSERT(send_wqep != NULL);
/* get the client context */
cclientp = clientp = send_wqep->send_client;
/* Check if this is a completion for a BUSY MAD sent by IBMF */
if (clientp == NULL) {
ibmf_msg_impl_t *msgimplp;
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion(): NULL client\n");
msgimplp = send_wqep->send_msg;
/*
* Deregister registered memory and free it, and
* free up the send WQE context
*/
(void) ibt_deregister_mr(cip->ci_ci_handle,
send_wqep->send_mem_hdl);
kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE);
kmem_free(send_wqep, sizeof (ibmf_send_wqe_t));
/* Free up the message context */
ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest);
ibmf_i_clean_ud_dest_list(cip, B_FALSE);
kmem_free(msgimplp, sizeof (ibmf_msg_impl_t));
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
return;
}
/* get the QP handle */
ibmf_qp_handle = send_wqep->send_ibmf_qp_handle;
qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;
ASSERT(clientp != NULL);
/* decrement the number of active sends */
if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
clientp->ic_sends_active--;
mutex_exit(&clientp->ic_mutex);
} else {
mutex_enter(&qpp->isq_mutex);
qpp->isq_sends_active--;
mutex_exit(&qpp->isq_mutex);
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, sends_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status);
/*
* issue the callback using taskq. If no taskq or if the
* dispatch fails, we do the send processing in the callback context
* which is the interrupt context
*/
if (cclientp->ic_send_taskq == NULL) {
/* Do the processing in callback context */
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
ibmf_i_do_send_cb((void *)send_wqep);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
"ibmf_i_handle_send_completion(): %s\n",
tnf_string, msg, "ci_send_taskq == NULL");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
return;
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
/* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */
if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) {
ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb,
send_wqep, TQ_NOSLEEP);
if (ret == 0) {
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
"ibmf_i_handle_send_completion(): %s\n",
tnf_string, msg, "send: dispatch failed");
ibmf_i_do_send_cb((void *)send_wqep);
}
} else {
ibmf_i_do_send_cb((void *)send_wqep);
}
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
"ibmf_i_handle_send_completion() exit\n");
}
/*
* ibmf_i_send_pkt()
* Send an IB packet after allocating send resources
*/
int
ibmf_i_send_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle,
ibmf_msg_impl_t *msgimplp, int block)
{
ibmf_send_wqe_t *send_wqep;
int status;
IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_start,
IBMF_TNF_TRACE, "",
"ibmf_i_send_pkt(): clientp = 0x%p, qp_hdl = 0x%p, "
"msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp,
tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp,
tnf_uint, block, block);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))
/*
* Reset send_done to indicate we have not received the completion
* for this send yet.
*/
msgimplp->im_trans_state_flags &= ~IBMF_TRANS_STATE_FLAG_SEND_DONE;
/*
* Allocate resources needed to send a UD packet including the
* send WQE context
*/
status = ibmf_i_alloc_send_resources(clientp->ic_myci,
msgimplp, block, &send_wqep);
if (status != IBMF_SUCCESS) {
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err,
IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n",
tnf_string, msg, "unable to allocate send resources",
tnf_uint, status, status);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n");
return (status);
}
/* Set the segment number in the send WQE context */
if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)
send_wqep->send_rmpp_segment = msgimplp->im_rmpp_ctx.rmpp_ns;
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))
/*
* Increment the count of pending send completions.
* Only when this count is zero should the client be notified
* of completion of the transaction.
*/
msgimplp->im_pending_send_compls += 1;
/* Send the packet */
status = ibmf_i_issue_pkt(clientp, msgimplp, ibmf_qp_handle, send_wqep);
if (status != IBMF_SUCCESS) {
ibmf_i_free_send_resources(clientp->ic_myci, msgimplp,
send_wqep);
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err,
IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n",
tnf_string, msg, "unable to issue packet",
tnf_uint, status, status);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n");
return (status);
}
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit, status = %d\n",
tnf_uint, status, status);
return (IBMF_SUCCESS);
}
/*
* ibmf_i_client_add_msg():
* Add the message to the client message list
*/
void
ibmf_i_client_add_msg(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp)
{
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_add_msg_start, IBMF_TNF_TRACE, "",
"ibmf_i_client_add_msg(): clientp = 0x%p, msgp = 0x%p\n",
tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp);
ASSERT(MUTEX_NOT_HELD(&msgimplp->im_mutex));
mutex_enter(&clientp->ic_msg_mutex);
/*
* If this is a termination message, add the message to
* the termination message list else add the message
* to the regular message list.
*/
mutex_enter(&msgimplp->im_mutex);
if (msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) {
mutex_exit(&msgimplp->im_mutex);
/* Put the message on the list */
if (clientp->ic_term_msg_list == NULL) {
clientp->ic_term_msg_list = clientp->ic_term_msg_last =
msgimplp;
} else {
msgimplp->im_msg_prev = clientp->ic_term_msg_last;
clientp->ic_term_msg_last->im_msg_next = msgimplp;
clientp->ic_term_msg_last = msgimplp;
}
} else {
mutex_exit(&msgimplp->im_mutex);
/*
* Increment the counter and kstats for active messages
*/
clientp->ic_msgs_active++;
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
/* Put the message on the list */
if (clientp->ic_msg_list == NULL) {
clientp->ic_msg_list = clientp->ic_msg_last = msgimplp;
} else {
msgimplp->im_msg_prev = clientp->ic_msg_last;
clientp->ic_msg_last->im_msg_next = msgimplp;
clientp->ic_msg_last = msgimplp;
}
}
msgimplp->im_msg_next = NULL;
/* Set the message flags to indicate the message is on the list */
mutex_enter(&msgimplp->im_mutex);
msgimplp->im_flags |= IBMF_MSG_FLAGS_ON_LIST;
mutex_exit(&msgimplp->im_mutex);
mutex_exit(&clientp->ic_msg_mutex);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_add_msg_end, IBMF_TNF_TRACE, "",
"ibmf_i_client_add_msg() exit\n");
}
/*
* ibmf_i_client_rem_msg():
* Remove the message from the client's message list
* The refcnt will hold the message reference count at the time
* the message was removed from the message list. Any packets
* arriving after this point for the message will be dropped.
* The message reference count is used by the threads processing
* the message to decide which one should notify the client
* (the one that decrements the reference count to zero).
*/
void
ibmf_i_client_rem_msg(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
uint_t *refcnt)
{
ibmf_msg_impl_t *tmpmsg, *prevmsg = NULL;
ASSERT(MUTEX_NOT_HELD(&msgimplp->im_mutex));
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_rem_msg_start, IBMF_TNF_TRACE, "",
"ibmf_i_client_rem_msg(): clientp = 0x%p, msgp = 0x%p\n",
tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp);
mutex_enter(&clientp->ic_msg_mutex);
/*
* If this is a termination message, remove the message from
* the termination message list else remove the message
* from the regular message list.
*/
mutex_enter(&msgimplp->im_mutex);
if (msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) {
mutex_exit(&msgimplp->im_mutex);
tmpmsg = clientp->ic_term_msg_list;
while (tmpmsg != NULL) {
if (tmpmsg == msgimplp)
break;
prevmsg = tmpmsg;
tmpmsg = tmpmsg->im_msg_next;
}
ASSERT(tmpmsg != NULL);
if (tmpmsg->im_msg_next == NULL)
clientp->ic_term_msg_last = prevmsg;
else
tmpmsg->im_msg_next->im_msg_prev = prevmsg;
if (prevmsg != NULL)
prevmsg->im_msg_next = tmpmsg->im_msg_next;
else
clientp->ic_term_msg_list = tmpmsg->im_msg_next;
} else {
mutex_exit(&msgimplp->im_mutex);
/*
* Decrement the counter and kstats for active messages
*/
ASSERT(clientp->ic_msgs_active != 0);
clientp->ic_msgs_active--;
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_SUB32_KSTATS(clientp, msgs_active, 1);
mutex_exit(&clientp->ic_kstat_mutex);
tmpmsg = clientp->ic_msg_list;
while (tmpmsg != NULL) {
if (tmpmsg == msgimplp)
break;
prevmsg = tmpmsg;
tmpmsg = tmpmsg->im_msg_next;
}
ASSERT(tmpmsg != NULL);
if (tmpmsg->im_msg_next == NULL)
clientp->ic_msg_last = prevmsg;
else
tmpmsg->im_msg_next->im_msg_prev = prevmsg;
if (prevmsg != NULL)
prevmsg->im_msg_next = tmpmsg->im_msg_next;
else
clientp->ic_msg_list = tmpmsg->im_msg_next;
}
/* Save away the message reference count and clear the list flag */
mutex_enter(&msgimplp->im_mutex);
*refcnt = msgimplp->im_ref_count;
msgimplp->im_flags &= ~IBMF_MSG_FLAGS_ON_LIST;
mutex_exit(&msgimplp->im_mutex);
mutex_exit(&clientp->ic_msg_mutex);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_client_rem_msg_end, IBMF_TNF_TRACE, "",
"ibmf_i_client_rem_msg() exit\n");
}
/*
* ibmf_i_terminate_transaction():
* Do transaction termination processing.
*/
void
ibmf_i_terminate_transaction(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
uint32_t status)
{
IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_terminate_transaction_start, IBMF_TNF_TRACE, "",
"ibmf_i_terminate_transaction(): clientp = 0x%p, msgp = 0x%p, "
"status = 0x%x\n", tnf_opaque, clientp, clientp,
tnf_opaque, msg, msgimplp, tnf_uint, status, status);
ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
msgimplp->im_msg_status = status;
/*
* Cancel the transaction timer. timer is probably only active if status
* was not success and this is a recv operation, but unset_timer() will
* check.
*/
ibmf_i_unset_timer(msgimplp, IBMF_TRANS_TIMER);
/*
* For unsolicited messages, do not notify the client
* if an error was encontered in the transfer.
* For solicited messages, call the transaction callback
* provided by the client in the message context.
*/
if (msgimplp->im_unsolicited == B_TRUE) {
msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
} else {
IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_terminate_transaction, IBMF_TNF_TRACE, "",
"ibmf_i_terminate_transaction(): %s, "
"trans_state_flags = 0x%x, msg_flags = 0x%x\n",
tnf_string, msg, "solicted message callback",
tnf_opaque, trans_state_flags,
msgimplp->im_trans_state_flags,
tnf_opaque, flags, msgimplp->im_flags);
/* mark as recv_compl happened */
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_RECV_DONE;
/*
* Check if last send is done before marking as done.
* We should get here for sequenced transactions and
* non-sequenced send RMPP transaction.
*/
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_SEND_DONE) {
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_DONE;
}
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_terminate_transaction_end, IBMF_TNF_TRACE, "",
"ibmf_i_terminate_transaction() exit\n");
}
/*
* ibmf_i_notify_client():
* If the transaction is done, call the appropriate callback
*/
void
ibmf_i_notify_client(ibmf_msg_impl_t *msgimplp)
{
ibmf_client_t *clientp;
ibmf_msg_cb_t async_cb;
void *async_cb_arg;
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_start,
IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): msgp = 0x%p\n",
tnf_opaque, msgimplp, msgimplp);
clientp = msgimplp->im_client;
/*
* message is removed so no more threads will find message;
* wait for any current clients to finish
*/
mutex_enter(&msgimplp->im_mutex);
ASSERT(msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE);
/*
* If the message reference count is not zero, then some duplicate
* MAD has arrived for this message. The thread processing the MAD
* found the message on the client's list before this thread was able
* to remove the message from the list. Since, we should not notify
* the client of the transaction completion until all the threads
* working on this message have completed (we don't want the client
* to free the message while a thread is working on it), we let one
* of the other threads notify the client of the completion once
* the message reference count is zero.
*/
if (msgimplp->im_ref_count != 0) {
mutex_exit(&msgimplp->im_mutex);
IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client_err, IBMF_TNF_TRACE,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg, "message reference count != 0");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
mutex_exit(&msgimplp->im_mutex);
/*
* Free up the UD dest resource so it is not tied down by
* the message in case the message is not freed immediately.
* Clean up the UD dest list as well so that excess UD dest
* resources are returned to the CI.
*/
if (msgimplp->im_ibmf_ud_dest != NULL) {
ibmf_i_free_ud_dest(clientp, msgimplp);
ibmf_i_clean_ud_dest_list(clientp->ic_myci, B_FALSE);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msgimplp))
if (msgimplp->im_unsolicited == B_TRUE) {
/*
* Do nothing if error status
*/
if (msgimplp->im_msg_status != IBMF_SUCCESS) {
if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
} else {
ibmf_alt_qp_t *qpp =
(ibmf_alt_qp_t *)msgimplp->im_qp_hdl;
mutex_enter(&qpp->isq_mutex);
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
}
IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR, "",
"ibmf_i_notify_client(): %s, status = %d\n",
tnf_string, msg, "message status not success",
tnf_opaque, status, msgimplp->im_msg_status);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
/*
* Check to see if
* a callback has been resgistered with the client
* for this unsolicited message.
* If one has been registered, up the recvs active
* count to get the teardown routine to wait until
* this callback is complete.
*/
if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) {
mutex_enter(&clientp->ic_mutex);
if ((clientp->ic_recv_cb == NULL) ||
(clientp->ic_flags & IBMF_CLIENT_TEAR_DOWN_CB)) {
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg,
"ibmf_tear_down_recv_cb already occurred");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
clientp->ic_msgs_alloced++;
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1);
mutex_exit(&clientp->ic_kstat_mutex);
async_cb = clientp->ic_recv_cb;
async_cb_arg = clientp->ic_recv_cb_arg;
mutex_exit(&clientp->ic_mutex);
async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp,
async_cb_arg);
mutex_enter(&clientp->ic_mutex);
IBMF_RECV_CB_CLEANUP(clientp);
mutex_exit(&clientp->ic_mutex);
} else {
ibmf_alt_qp_t *qpp =
(ibmf_alt_qp_t *)msgimplp->im_qp_hdl;
mutex_enter(&qpp->isq_mutex);
if ((qpp->isq_recv_cb == NULL) ||
(qpp->isq_flags & IBMF_CLIENT_TEAR_DOWN_CB)) {
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
ibmf_i_free_msg(msgimplp);
IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
ibmf_i_notify_client_err, IBMF_TNF_ERROR,
"", "ibmf_i_notify_client(): %s\n",
tnf_string, msg,
"ibmf_tear_down_recv_cb already occurred");
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "",
"ibmf_i_notify_client() exit\n");
return;
}
async_cb = qpp->isq_recv_cb;
async_cb_arg = qpp->isq_recv_cb_arg;
mutex_exit(&qpp->isq_mutex);
mutex_enter(&clientp->ic_mutex);
clientp->ic_msgs_alloced++;
mutex_exit(&clientp->ic_mutex);
mutex_enter(&clientp->ic_kstat_mutex);
IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1);
mutex_exit(&clientp->ic_kstat_mutex);
async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp,
async_cb_arg);
mutex_enter(&qpp->isq_mutex);
IBMF_ALT_RECV_CB_CLEANUP(qpp);
mutex_exit(&qpp->isq_mutex);
}
} else {
/* Solicited transaction processing */
if (msgimplp->im_trans_cb == NULL) {
/* Processing for a blocking transaction */
mutex_enter(&msgimplp->im_mutex);
if (msgimplp->im_trans_state_flags &
IBMF_TRANS_STATE_FLAG_WAIT) {
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "Awaking thread",
tnf_opaque, msgimplp, msgimplp);
cv_signal(&msgimplp->im_trans_cv);
} else {
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "Notify client, no wait",
tnf_opaque, msgimplp, msgimplp);
}
msgimplp->im_trans_state_flags |=
IBMF_TRANS_STATE_FLAG_SIGNALED;
mutex_exit(&msgimplp->im_mutex);
} else {
/* Processing for a non-blocking transaction */
mutex_enter(&msgimplp->im_mutex);
msgimplp->im_flags &= ~IBMF_MSG_FLAGS_BUSY;
mutex_exit(&msgimplp->im_mutex);
IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
ibmf_i_notify_client, IBMF_TNF_TRACE, "",
"ibmf_i_notify_client(): %s, msg = 0x%p\n",
tnf_string, msg, "No thread is blocking",
tnf_opaque, msgimplp, msgimplp);
if (msgimplp->im_trans_cb != NULL) {
msgimplp->im_trans_cb(
(ibmf_handle_t)clientp,
(ibmf_msg_t *)msgimplp,
msgimplp->im_trans_cb_arg);
}
}
}
IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end,
IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n");
}