/* ARGSUSED */ void ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp) { ibmf_client_t *clientp, *cclientp; ibmf_send_wqe_t *send_wqep; ibmf_qp_handle_t ibmf_qp_handle; ibmf_alt_qp_t *qpp; int ret; IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n", tnf_opaque, cip, cip, tnf_opaque, wcp, wcp); _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) ASSERT(wcp->wc_id != NULL); ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id)); /* get the IBMF send WQE context */ IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep); ASSERT(send_wqep != NULL); /* get the client context */ cclientp = clientp = send_wqep->send_client; /* Check if this is a completion for a BUSY MAD sent by IBMF */ if (clientp == NULL) { ibmf_msg_impl_t *msgimplp; IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion(): NULL client\n"); msgimplp = send_wqep->send_msg; /* * Deregister registered memory and free it, and * free up the send WQE context */ (void) ibt_deregister_mr(cip->ci_ci_handle, send_wqep->send_mem_hdl); kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE); kmem_free(send_wqep, sizeof (ibmf_send_wqe_t)); /* Free up the message context */ ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest); ibmf_i_clean_ud_dest_list(cip, B_FALSE); kmem_free(msgimplp, sizeof (ibmf_msg_impl_t)); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); return; } /* get the QP handle */ ibmf_qp_handle = send_wqep->send_ibmf_qp_handle; qpp = (ibmf_alt_qp_t *)ibmf_qp_handle; ASSERT(clientp != NULL); /* decrement the number of active sends */ if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&clientp->ic_mutex); clientp->ic_sends_active--; mutex_exit(&clientp->ic_mutex); } else { mutex_enter(&qpp->isq_mutex); qpp->isq_sends_active--; mutex_exit(&qpp->isq_mutex); } mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, sends_active, 1); mutex_exit(&clientp->ic_kstat_mutex); send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status); /* * issue the callback using taskq. If no taskq or if the * dispatch fails, we do the send processing in the callback context * which is the interrupt context */ if (cclientp->ic_send_taskq == NULL) { /* Do the processing in callback context */ mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); ibmf_i_do_send_cb((void *)send_wqep); mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", "ibmf_i_handle_send_completion(): %s\n", tnf_string, msg, "ci_send_taskq == NULL"); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); return; } mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); /* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */ if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) { ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb, send_wqep, TQ_NOSLEEP); if (ret == 0) { IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", "ibmf_i_handle_send_completion(): %s\n", tnf_string, msg, "send: dispatch failed"); ibmf_i_do_send_cb((void *)send_wqep); } } else { ibmf_i_do_send_cb((void *)send_wqep); } mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); }
/* * ibmf_i_notify_client(): * If the transaction is done, call the appropriate callback */ void ibmf_i_notify_client(ibmf_msg_impl_t *msgimplp) { ibmf_client_t *clientp; ibmf_msg_cb_t async_cb; void *async_cb_arg; IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_start, IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): msgp = 0x%p\n", tnf_opaque, msgimplp, msgimplp); clientp = msgimplp->im_client; /* * message is removed so no more threads will find message; * wait for any current clients to finish */ mutex_enter(&msgimplp->im_mutex); ASSERT(msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE); /* * If the message reference count is not zero, then some duplicate * MAD has arrived for this message. The thread processing the MAD * found the message on the client's list before this thread was able * to remove the message from the list. Since, we should not notify * the client of the transaction completion until all the threads * working on this message have completed (we don't want the client * to free the message while a thread is working on it), we let one * of the other threads notify the client of the completion once * the message reference count is zero. */ if (msgimplp->im_ref_count != 0) { mutex_exit(&msgimplp->im_mutex); IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_notify_client_err, IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): %s\n", tnf_string, msg, "message reference count != 0"); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end, IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n"); return; } mutex_exit(&msgimplp->im_mutex); /* * Free up the UD dest resource so it is not tied down by * the message in case the message is not freed immediately. * Clean up the UD dest list as well so that excess UD dest * resources are returned to the CI. */ if (msgimplp->im_ibmf_ud_dest != NULL) { ibmf_i_free_ud_dest(clientp, msgimplp); ibmf_i_clean_ud_dest_list(clientp->ic_myci, B_FALSE); } _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msgimplp)) if (msgimplp->im_unsolicited == B_TRUE) { /* * Do nothing if error status */ if (msgimplp->im_msg_status != IBMF_SUCCESS) { if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&clientp->ic_mutex); IBMF_RECV_CB_CLEANUP(clientp); mutex_exit(&clientp->ic_mutex); } else { ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)msgimplp->im_qp_hdl; mutex_enter(&qpp->isq_mutex); IBMF_ALT_RECV_CB_CLEANUP(qpp); mutex_exit(&qpp->isq_mutex); } IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_notify_client_err, IBMF_TNF_ERROR, "", "ibmf_i_notify_client(): %s, status = %d\n", tnf_string, msg, "message status not success", tnf_opaque, status, msgimplp->im_msg_status); ibmf_i_free_msg(msgimplp); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end, IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n"); return; } /* * Check to see if * a callback has been resgistered with the client * for this unsolicited message. * If one has been registered, up the recvs active * count to get the teardown routine to wait until * this callback is complete. */ if (msgimplp->im_qp_hdl == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&clientp->ic_mutex); if ((clientp->ic_recv_cb == NULL) || (clientp->ic_flags & IBMF_CLIENT_TEAR_DOWN_CB)) { IBMF_RECV_CB_CLEANUP(clientp); mutex_exit(&clientp->ic_mutex); ibmf_i_free_msg(msgimplp); IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_notify_client_err, IBMF_TNF_ERROR, "", "ibmf_i_notify_client(): %s\n", tnf_string, msg, "ibmf_tear_down_recv_cb already occurred"); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end, IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n"); return; } clientp->ic_msgs_alloced++; mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1); mutex_exit(&clientp->ic_kstat_mutex); async_cb = clientp->ic_recv_cb; async_cb_arg = clientp->ic_recv_cb_arg; mutex_exit(&clientp->ic_mutex); async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp, async_cb_arg); mutex_enter(&clientp->ic_mutex); IBMF_RECV_CB_CLEANUP(clientp); mutex_exit(&clientp->ic_mutex); } else { ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)msgimplp->im_qp_hdl; mutex_enter(&qpp->isq_mutex); if ((qpp->isq_recv_cb == NULL) || (qpp->isq_flags & IBMF_CLIENT_TEAR_DOWN_CB)) { IBMF_ALT_RECV_CB_CLEANUP(qpp); mutex_exit(&qpp->isq_mutex); ibmf_i_free_msg(msgimplp); IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_notify_client_err, IBMF_TNF_ERROR, "", "ibmf_i_notify_client(): %s\n", tnf_string, msg, "ibmf_tear_down_recv_cb already occurred"); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end, IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n"); return; } async_cb = qpp->isq_recv_cb; async_cb_arg = qpp->isq_recv_cb_arg; mutex_exit(&qpp->isq_mutex); mutex_enter(&clientp->ic_mutex); clientp->ic_msgs_alloced++; mutex_exit(&clientp->ic_mutex); mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, msgs_alloced, 1); mutex_exit(&clientp->ic_kstat_mutex); async_cb((ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp, async_cb_arg); mutex_enter(&qpp->isq_mutex); IBMF_ALT_RECV_CB_CLEANUP(qpp); mutex_exit(&qpp->isq_mutex); } } else { /* Solicited transaction processing */ if (msgimplp->im_trans_cb == NULL) { /* Processing for a blocking transaction */ mutex_enter(&msgimplp->im_mutex); if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_WAIT) { IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_notify_client, IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): %s, msg = 0x%p\n", tnf_string, msg, "Awaking thread", tnf_opaque, msgimplp, msgimplp); cv_signal(&msgimplp->im_trans_cv); } else { IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_notify_client, IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): %s, msg = 0x%p\n", tnf_string, msg, "Notify client, no wait", tnf_opaque, msgimplp, msgimplp); } msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SIGNALED; mutex_exit(&msgimplp->im_mutex); } else { /* Processing for a non-blocking transaction */ mutex_enter(&msgimplp->im_mutex); msgimplp->im_flags &= ~IBMF_MSG_FLAGS_BUSY; mutex_exit(&msgimplp->im_mutex); IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_notify_client, IBMF_TNF_TRACE, "", "ibmf_i_notify_client(): %s, msg = 0x%p\n", tnf_string, msg, "No thread is blocking", tnf_opaque, msgimplp, msgimplp); if (msgimplp->im_trans_cb != NULL) { msgimplp->im_trans_cb( (ibmf_handle_t)clientp, (ibmf_msg_t *)msgimplp, msgimplp->im_trans_cb_arg); } } } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_notify_client_end, IBMF_TNF_TRACE, "", "ibmf_i_notify_client() exit\n"); }