/****f* opensm: SM/__osm_sm_mad_ctrl_disp_done_callback
* NAME
* __osm_sm_mad_ctrl_disp_done_callback
*
* DESCRIPTION
* This function is the Dispatcher callback that indicates
* a received MAD has been processed by the recipient.
*
* SYNOPSIS
*/
static void
__osm_sm_mad_ctrl_disp_done_callback(IN void *context, IN void *p_data)
{
osm_sm_mad_ctrl_t *const p_ctrl = (osm_sm_mad_ctrl_t *) context;
osm_madw_t *const p_madw = (osm_madw_t *) p_data;
ib_smp_t *p_smp;
OSM_LOG_ENTER(p_ctrl->p_log);
/*
If the MAD that just finished processing was a response,
then retire the transaction, since we must have generated
the request.
Otherwise, retire the transaction if a response was expected,
as in the case of a send failure. If a response was not expected,
just put the MAD back in the pool, because the MAD was a query
from some outside agent, e.g. Get(SMInfo) from another SM.
*/
p_smp = osm_madw_get_smp_ptr(p_madw);
if (ib_smp_is_response(p_smp)) {
CL_ASSERT(p_madw->resp_expected == FALSE);
__osm_sm_mad_ctrl_retire_trans_mad(p_ctrl, p_madw);
} else if (p_madw->resp_expected == TRUE)
__osm_sm_mad_ctrl_retire_trans_mad(p_ctrl, p_madw);
else
osm_mad_pool_put(p_ctrl->p_mad_pool, p_madw);
OSM_LOG_EXIT(p_ctrl->p_log);
}
static void
__osm_sm_mad_ctrl_retire_trans_mad(IN osm_sm_mad_ctrl_t * const p_ctrl,
IN osm_madw_t * const p_madw)
{
uint32_t outstanding;
OSM_LOG_ENTER(p_ctrl->p_log);
CL_ASSERT(p_madw);
/*
Return the MAD & wrapper to the pool.
*/
OSM_LOG(p_ctrl->p_log, OSM_LOG_DEBUG,
"Retiring MAD with TID 0x%" PRIx64 "\n",
cl_ntoh64(osm_madw_get_smp_ptr(p_madw)->trans_id));
osm_mad_pool_put(p_ctrl->p_mad_pool, p_madw);
outstanding = osm_stats_dec_qp0_outstanding(p_ctrl->p_stats);
OSM_LOG(p_ctrl->p_log, OSM_LOG_DEBUG, "%u QP0 MADs outstanding%s\n",
p_ctrl->p_stats->qp0_mads_outstanding,
outstanding ? "" : ": wire is clean.");
OSM_LOG_EXIT(p_ctrl->p_log);
}
/****f* opensm: SM/__osm_sm_mad_ctrl_process_set
* NAME
* __osm_sm_mad_ctrl_process_set
*
* DESCRIPTION
* This function handles method Set() for received MADs.
*
* SYNOPSIS
*/
static void
__osm_sm_mad_ctrl_process_set(IN osm_sm_mad_ctrl_t * const p_ctrl,
IN osm_madw_t * p_madw)
{
ib_smp_t *p_smp;
cl_status_t status;
cl_disp_msgid_t msg_id = CL_DISP_MSGID_NONE;
OSM_LOG_ENTER(p_ctrl->p_log);
p_smp = osm_madw_get_smp_ptr(p_madw);
/*
Note that attr_id (like the rest of the MAD) is in
network byte order.
*/
switch (p_smp->attr_id) {
case IB_MAD_ATTR_SM_INFO:
msg_id = OSM_MSG_MAD_SM_INFO;
break;
default:
cl_atomic_inc(&p_ctrl->p_stats->qp0_mads_rcvd_unknown);
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3107: "
"Unsupported attribute = 0x%X\n",
cl_ntoh16(p_smp->attr_id));
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_ERROR);
break;
}
if (msg_id == CL_DISP_MSGID_NONE) {
/*
There is an unknown MAD attribute type for which there is
no recipient. Simply retire the MAD here.
*/
osm_mad_pool_put(p_ctrl->p_mad_pool, p_madw);
goto Exit;
}
/*
Post this MAD to the dispatcher for asynchronous
processing by the appropriate controller.
*/
OSM_LOG(p_ctrl->p_log, OSM_LOG_DEBUG, "Posting Dispatcher message %s\n",
osm_get_disp_msg_str(msg_id));
status = cl_disp_post(p_ctrl->h_disp, msg_id, p_madw,
__osm_sm_mad_ctrl_disp_done_callback, p_ctrl);
if (status != CL_SUCCESS) {
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3108: "
"Dispatcher post message failed (%s)\n",
CL_STATUS_MSG(status));
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_ctrl->p_log);
}
void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool)
{
osm_madw_t *p_madw;
OSM_LOG_ENTER(p_vl->p_log);
/*
Signal our threads that we're leaving.
*/
p_vl->thread_state = OSM_THREAD_STATE_EXIT;
/*
Don't trigger unless event has been initialized.
Destroy the thread before we tear down the other objects.
*/
if (p_vl->state != OSM_VL15_STATE_INIT)
cl_event_signal(&p_vl->signal);
cl_thread_destroy(&p_vl->poller);
/*
Return the outstanding messages to the pool
*/
cl_spinlock_acquire(&p_vl->lock);
while (!cl_is_qlist_empty(&p_vl->rfifo)) {
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
osm_mad_pool_put(p_pool, p_madw);
}
while (!cl_is_qlist_empty(&p_vl->ufifo)) {
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
osm_mad_pool_put(p_pool, p_madw);
}
cl_spinlock_release(&p_vl->lock);
cl_event_destroy(&p_vl->signal);
p_vl->state = OSM_VL15_STATE_INIT;
cl_spinlock_destroy(&p_vl->lock);
OSM_LOG_EXIT(p_vl->p_log);
}
void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN osm_mad_pool_t * p_mad_pool)
{
osm_madw_t *p_madw;
OSM_LOG_ENTER(p_vl->p_log);
/* we only should get here after the VL15 interface was initialized */
CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
/* grap a lock on the object */
cl_spinlock_acquire(&p_vl->lock);
/* go over all outstanding MADs and retire their transactions */
/* first we handle the list of response MADs */
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) {
OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
"Releasing Response p_madw = %p\n", p_madw);
osm_mad_pool_put(p_mad_pool, p_madw);
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
}
/* Request MADs we send out */
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->rfifo)) {
OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
"Releasing Request p_madw = %p\n", p_madw);
osm_mad_pool_put(p_mad_pool, p_madw);
osm_stats_dec_qp0_outstanding(p_vl->p_stats);
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
}
/* free the lock */
cl_spinlock_release(&p_vl->lock);
OSM_LOG_EXIT(p_vl->p_log);
}
/**
* Return the mad query data.
*/
static void
_return_mad(void)
{
/*
* Return the IB query MAD to the pool as necessary.
*/
if( _openib_result.p_result_madw != NULL ) {
osm_mad_pool_put( &_openib_mad_pool, _openib_result.p_result_madw );
_openib_result.p_result_madw = NULL;
}
}
/****f* opensm: SM/__osm_sm_mad_ctrl_rcv_callback
* NAME
* __osm_sm_mad_ctrl_rcv_callback
*
* DESCRIPTION
* This is the callback from the transport layer for received MADs.
*
* SYNOPSIS
*/
static void
__osm_sm_mad_ctrl_rcv_callback(IN osm_madw_t * p_madw,
IN void *bind_context,
IN osm_madw_t * p_req_madw)
{
osm_sm_mad_ctrl_t *p_ctrl = (osm_sm_mad_ctrl_t *) bind_context;
ib_smp_t *p_smp;
ib_net16_t status;
OSM_LOG_ENTER(p_ctrl->p_log);
CL_ASSERT(p_madw);
/*
A MAD was received from the wire, possibly in response to a request.
*/
cl_atomic_inc(&p_ctrl->p_stats->qp0_mads_rcvd);
OSM_LOG(p_ctrl->p_log, OSM_LOG_DEBUG, "%u QP0 MADs received\n",
p_ctrl->p_stats->qp0_mads_rcvd);
p_smp = osm_madw_get_smp_ptr(p_madw);
/* if we are closing down simply do nothing */
if (osm_exit_flag) {
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR,
"Ignoring received mad - since we are exiting\n");
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_DEBUG);
/* retire the mad or put it back */
if (ib_smp_is_response(p_smp) ||
(p_smp->method == IB_MAD_METHOD_TRAP_REPRESS)) {
CL_ASSERT(p_madw->resp_expected == FALSE);
__osm_sm_mad_ctrl_retire_trans_mad(p_ctrl, p_madw);
} else if (p_madw->resp_expected == TRUE)
__osm_sm_mad_ctrl_retire_trans_mad(p_ctrl, p_madw);
else
osm_mad_pool_put(p_ctrl->p_mad_pool, p_madw);
goto Exit;
}
if (osm_log_is_active(p_ctrl->p_log, OSM_LOG_FRAMES))
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_FRAMES);
if (p_smp->mgmt_class == IB_MCLASS_SUBN_DIR)
status = ib_smp_get_status(p_smp);
else
status = p_smp->status;
if (status != 0) {
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3111: "
"Error status = 0x%X\n", status);
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_ERROR);
}
switch (p_smp->method) {
case IB_MAD_METHOD_GET_RESP:
CL_ASSERT(p_req_madw != NULL);
__osm_sm_mad_ctrl_process_get_resp(p_ctrl, p_madw, p_req_madw);
break;
case IB_MAD_METHOD_GET:
CL_ASSERT(p_req_madw == NULL);
__osm_sm_mad_ctrl_process_get(p_ctrl, p_madw);
break;
case IB_MAD_METHOD_TRAP:
CL_ASSERT(p_req_madw == NULL);
__osm_sm_mad_ctrl_process_trap(p_ctrl, p_madw);
break;
case IB_MAD_METHOD_SET:
CL_ASSERT(p_req_madw == NULL);
__osm_sm_mad_ctrl_process_set(p_ctrl, p_madw);
break;
case IB_MAD_METHOD_SEND:
case IB_MAD_METHOD_REPORT:
case IB_MAD_METHOD_REPORT_RESP:
case IB_MAD_METHOD_TRAP_REPRESS:
default:
cl_atomic_inc(&p_ctrl->p_stats->qp0_mads_rcvd_unknown);
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3112: "
"Unsupported method = 0x%X\n", p_smp->method);
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_ERROR);
osm_mad_pool_put(p_ctrl->p_mad_pool, p_madw);
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_ctrl->p_log);
}
/****f* opensm: SM/__osm_sm_mad_ctrl_process_get_resp
* NAME
* __osm_sm_mad_ctrl_process_get_resp
*
* DESCRIPTION
* This function handles method GetResp() for received MADs.
* This is the most common path for QP0 MADs.
*
* SYNOPSIS
*/
static void
__osm_sm_mad_ctrl_process_get_resp(IN osm_sm_mad_ctrl_t * const p_ctrl,
IN osm_madw_t * p_madw,
IN void *transaction_context)
{
ib_smp_t *p_smp;
cl_status_t status;
osm_madw_t *p_old_madw;
cl_disp_msgid_t msg_id = CL_DISP_MSGID_NONE;
OSM_LOG_ENTER(p_ctrl->p_log);
CL_ASSERT(p_madw);
CL_ASSERT(transaction_context);
p_smp = osm_madw_get_smp_ptr(p_madw);
if (p_smp->mgmt_class == IB_MCLASS_SUBN_DIR && !ib_smp_is_d(p_smp)) {
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3102: "
"'D' bit not set in returned SMP\n");
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_ERROR);
}
p_old_madw = (osm_madw_t *) transaction_context;
__osm_sm_mad_ctrl_update_wire_stats(p_ctrl);
/*
Copy the MAD Wrapper context from the requesting MAD
to the new MAD. This mechanism allows the recipient
controller to recover its own context regarding this
MAD transaction. Once we've copied the context, we
can return the original MAD to the pool.
*/
osm_madw_copy_context(p_madw, p_old_madw);
osm_mad_pool_put(p_ctrl->p_mad_pool, p_old_madw);
/*
Note that attr_id (like the rest of the MAD) is in
network byte order.
*/
switch (p_smp->attr_id) {
case IB_MAD_ATTR_NODE_DESC:
msg_id = OSM_MSG_MAD_NODE_DESC;
break;
case IB_MAD_ATTR_NODE_INFO:
msg_id = OSM_MSG_MAD_NODE_INFO;
break;
case IB_MAD_ATTR_SWITCH_INFO:
msg_id = OSM_MSG_MAD_SWITCH_INFO;
break;
case IB_MAD_ATTR_PORT_INFO:
msg_id = OSM_MSG_MAD_PORT_INFO;
break;
case IB_MAD_ATTR_LIN_FWD_TBL:
msg_id = OSM_MSG_MAD_LFT;
break;
case IB_MAD_ATTR_MCAST_FWD_TBL:
msg_id = OSM_MSG_MAD_MFT;
break;
case IB_MAD_ATTR_SM_INFO:
msg_id = OSM_MSG_MAD_SM_INFO;
break;
case IB_MAD_ATTR_SLVL_TABLE:
msg_id = OSM_MSG_MAD_SLVL;
break;
case IB_MAD_ATTR_VL_ARBITRATION:
msg_id = OSM_MSG_MAD_VL_ARB;
break;
case IB_MAD_ATTR_P_KEY_TABLE:
msg_id = OSM_MSG_MAD_PKEY;
break;
case IB_MAD_ATTR_GUID_INFO:
case IB_MAD_ATTR_CLASS_PORT_INFO:
case IB_MAD_ATTR_NOTICE:
case IB_MAD_ATTR_INFORM_INFO:
default:
cl_atomic_inc(&p_ctrl->p_stats->qp0_mads_rcvd_unknown);
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3103: "
"Unsupported attribute = 0x%X\n",
cl_ntoh16(p_smp->attr_id));
osm_dump_dr_smp(p_ctrl->p_log, p_smp, OSM_LOG_ERROR);
goto Exit;
}
if (msg_id == CL_DISP_MSGID_NONE)
goto Exit;
/*
Post this MAD to the dispatcher for asynchronous
processing by the appropriate controller.
*/
OSM_LOG(p_ctrl->p_log, OSM_LOG_DEBUG, "Posting Dispatcher message %s\n",
osm_get_disp_msg_str(msg_id));
status = cl_disp_post(p_ctrl->h_disp, msg_id, p_madw,
__osm_sm_mad_ctrl_disp_done_callback, p_ctrl);
if (status != CL_SUCCESS) {
OSM_LOG(p_ctrl->p_log, OSM_LOG_ERROR, "ERR 3104: "
"Dispatcher post message failed (%s) for attribute = 0x%X\n",
CL_STATUS_MSG(status), cl_ntoh16(p_smp->attr_id));
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_ctrl->p_log);
}
/*
Call back on new mad received:
We basically only need to set the context of the query.
Or report an error.
A pointer to the actual context of the request (a copy of the oriignal
request structure) is attached as the p_madw->context.ni_context.node_guid
*/
static void
__osmv_sa_mad_rcv_cb(IN osm_madw_t * p_madw,
IN void *bind_context, IN osm_madw_t * p_req_madw)
{
osmv_sa_bind_info_t *p_bind = (osmv_sa_bind_info_t *) bind_context;
osmv_query_req_t *p_query_req_copy = NULL;
osmv_query_res_t query_res;
ib_sa_mad_t *p_sa_mad;
ib_net16_t mad_status;
OSM_LOG_ENTER(p_bind->p_log);
if (!p_req_madw) {
OSM_LOG(p_bind->p_log, OSM_LOG_DEBUG,
"Ignoring a non-response mad\n");
osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
goto Exit;
}
/* obtain the sent context since we store it during send in the ni_ctx */
p_query_req_copy =
(osmv_query_req_t *) (long *)(long)(p_req_madw->context.ni_context.
node_guid);
/* provide the context of the original request in the result */
query_res.query_context = p_query_req_copy->query_context;
/* provide the resulting madw */
query_res.p_result_madw = p_madw;
/* update the req fields */
p_sa_mad = (ib_sa_mad_t *) p_madw->p_mad;
/* if we got a remote error track it in the status */
mad_status = (ib_net16_t) (p_sa_mad->status & IB_SMP_STATUS_MASK);
if (mad_status != IB_SUCCESS) {
OSM_LOG(p_bind->p_log, OSM_LOG_ERROR, "ERR 5501: "
"Remote error:0x%04X\n", cl_ntoh16(mad_status));
query_res.status = IB_REMOTE_ERROR;
} else
query_res.status = IB_SUCCESS;
/* what if we have got back an empty mad ? */
if (!p_madw->mad_size) {
OSM_LOG(p_bind->p_log, OSM_LOG_ERROR, "ERR 5502: "
"Got an empty mad\n");
query_res.status = IB_ERROR;
}
if (IB_SUCCESS == mad_status) {
/* if we are in not in a method response of an rmpp nature we must get only 1 */
/* HACK: in the future we might need to be smarter for other methods... */
if (p_sa_mad->method != IB_MAD_METHOD_GETTABLE_RESP) {
query_res.result_cnt = 1;
} else {
#ifndef VENDOR_RMPP_SUPPORT
if (mad_status != IB_SUCCESS)
query_res.result_cnt = 0;
else
query_res.result_cnt = 1;
#else
if (ib_get_attr_size(p_sa_mad->attr_offset)) {
/* we used the offset value to calculate the
number of records in here */
query_res.result_cnt = (uintn_t)
((p_madw->mad_size - IB_SA_MAD_HDR_SIZE) /
ib_get_attr_size(p_sa_mad->attr_offset));
OSM_LOG(p_bind->p_log, OSM_LOG_DEBUG,
"Count = %u = %zu / %u (%zu)\n",
query_res.result_cnt,
p_madw->mad_size - IB_SA_MAD_HDR_SIZE,
ib_get_attr_size(p_sa_mad->attr_offset),
(p_madw->mad_size -
IB_SA_MAD_HDR_SIZE) %
ib_get_attr_size(p_sa_mad->attr_offset));
} else
query_res.result_cnt = 0;
#endif
}
}
query_res.query_type = p_query_req_copy->query_type;
p_query_req_copy->pfn_query_cb(&query_res);
if ((p_query_req_copy->flags & OSM_SA_FLAGS_SYNC) == OSM_SA_FLAGS_SYNC)
cl_event_signal(&p_bind->sync_event);
Exit:
/* free the copied query request if found */
if (p_query_req_copy)
free(p_query_req_copy);
/* put back the request madw */
if (p_req_madw)
osm_mad_pool_put(p_bind->p_mad_pool, p_req_madw);
OSM_LOG_EXIT(p_bind->p_log);
}
/**********************************************************************
* TS Send callback : invoked after each send
*
**********************************************************************/
void
__osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p,
IN boolean_t is_resp,
IN osm_madw_t * madw_p, IN IB_comp_status_t status)
{
osm_log_t *const p_log = bind_info_p->p_vend->p_log;
osm_vend_wrap_t *p_vw;
OSM_LOG_ENTER(p_log);
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_ts_send_callback: INFO 1008: "
"Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
/* we need to handle requests and responses differently */
if (is_resp) {
if (status != IB_COMP_SUCCESS) {
osm_log(p_log, OSM_LOG_ERROR,
"__osm_ts_send_callback: ERR 5012: "
"Error Sending Response MADW:%p.\n", madw_p);
} else {
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_ts_send_callback: DBG 1008: "
"Completed Sending Response MADW:%p.\n",
madw_p);
}
/* if we are a response - we need to clean it up */
osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
} else {
/* this call back is invoked on completion of send - error or not */
if (status != IB_COMP_SUCCESS) {
osm_log(p_log, OSM_LOG_ERROR,
"__osm_ts_send_callback: ERR 5013: "
"Received an Error from IB_MGT Send (%d).\n",
status);
p_vw = osm_madw_get_vend_ptr(madw_p);
CL_ASSERT(p_vw);
/*
Return any wrappers to the pool that may have been
pre-emptively allocated to handle a receive.
*/
if (p_vw->p_resp_madw) {
osm_mad_pool_put(bind_info_p->p_osm_pool,
p_vw->p_resp_madw);
p_vw->p_resp_madw = NULL;
}
/* invoke the CB */
(*(osm_vend_mad_send_err_callback_t) bind_info_p->
send_err_callback)
(bind_info_p->client_context, madw_p);
} else {
/* successful request send - do nothing - the response will need the
out mad */
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_ts_send_callback: DBG 1008: "
"Completed Sending Request MADW:%p.\n", madw_p);
}
}
OSM_LOG_EXIT(p_log);
}
static void
__osm_al_send_callback(IN void *mad_svc_context, IN ib_mad_element_t * p_elem)
{
osm_al_bind_info_t *const p_bind =
(osm_al_bind_info_t *) mad_svc_context;
osm_vendor_t *const p_vend = p_bind->p_vend;
osm_madw_t *const p_madw = (osm_madw_t *) p_elem->context1;
osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
ib_mad_t *p_mad;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vw);
CL_ASSERT(p_vw->h_av);
/*
Destroy the address vector as necessary.
*/
if (p_vw->h_av != p_bind->h_dr_av) {
if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_al_send_callback: "
"Destroying av handle %p.\n", p_vw->h_av);
}
ib_destroy_av(p_vw->h_av);
}
p_mad = ib_get_mad_buf(p_elem);
if (p_elem->resp_expected) {
/*
If the send was unsuccessful, notify the user
for MADs that were expecting a response.
A NULL mad wrapper parameter is the user's clue
that the transaction turned sour.
Otherwise, do nothing for successful sends when a
reponse is expected. The mad will be returned to the
pool later.
*/
p_madw->status = __osm_al_convert_wcs(p_elem->status);
if (p_elem->status != IB_WCS_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_al_send_callback: "
"MAD completed with work queue error: %s.\n",
ib_get_wc_status_str(p_elem->status));
/*
Return any wrappers to the pool that may have been
pre-emptively allocated to handle a receive.
*/
if (p_vw->p_resp_madw) {
osm_mad_pool_put(p_bind->p_osm_pool,
p_vw->p_resp_madw);
p_vw->p_resp_madw = NULL;
}
p_bind->send_err_callback(p_bind->client_context,
p_madw);
}
} else {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_al_send_callback: "
"Returning MAD to pool, TID = 0x%" PRIx64 ".\n",
cl_ntoh64(p_mad->trans_id));
osm_mad_pool_put(p_bind->p_osm_pool, p_madw);
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
}
ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,
IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
{
osm_al_bind_info_t *const p_bind = h_bind;
osm_vendor_t *const p_vend = p_bind->p_vend;
osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
ib_api_status_t status;
ib_mad_element_t *p_elem;
ib_av_attr_t av;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vw->h_bind == h_bind);
CL_ASSERT(p_vw->p_elem);
p_elem = p_vw->p_elem;
/*
If a response is expected to this MAD, then preallocate
a mad wrapper to contain the wire MAD received in the
response. Allocating a wrapper here allows for easier
failure paths than after we already received the wire mad.
*/
if (resp_expected) {
p_vw->p_resp_madw =
osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
if (p_vw->p_resp_madw == NULL) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 3B27: "
"Unable to allocate MAD wrapper.\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
} else
p_vw->p_resp_madw = NULL;
/*
For all sends other than directed route SM MADs,
acquire an address vector for the destination.
*/
if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
memset(&av, 0, sizeof(av));
av.port_num = p_bind->port_num;
av.dlid = p_mad_addr->dest_lid;
av.static_rate = p_mad_addr->static_rate;
av.path_bits = p_mad_addr->path_bits;
if ((p_mad->mgmt_class != IB_MCLASS_SUBN_LID) &&
(p_mad->mgmt_class != IB_MCLASS_SUBN_DIR)) {
av.sl = p_mad_addr->addr_type.gsi.service_level;
if (p_mad_addr->addr_type.gsi.global_route) {
av.grh_valid = TRUE;
/* ANIL */
/* av.grh = p_mad_addr->addr_type.gsi.grh_info; */
}
}
if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_send: "
"av.port_num 0x%X, "
"av.dlid 0x%X, "
"av.static_rate %d, "
"av.path_bits %d.\n",
av.port_num, cl_ntoh16(av.dlid),
av.static_rate, av.path_bits);
}
status = ib_create_av(p_vend->h_pd, &av, &p_vw->h_av);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 3B28: "
"Unable to create address vector (%s).\n",
ib_get_err_str(status));
if (p_vw->p_resp_madw)
osm_mad_pool_put(p_bind->p_osm_pool,
p_vw->p_resp_madw);
goto Exit;
}
if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_send: "
"Allocating av handle %p.\n", p_vw->h_av);
}
} else {
p_vw->h_av = p_bind->h_dr_av;
}
p_elem->h_av = p_vw->h_av;
p_elem->context1 = p_madw;
p_elem->context2 = NULL;
p_elem->immediate_data = 0;
p_elem->p_grh = NULL;
p_elem->resp_expected = resp_expected;
p_elem->retry_cnt = OSM_DEFAULT_RETRY_COUNT;
p_elem->send_opt = IB_SEND_OPT_SIGNALED;
p_elem->timeout_ms = p_vend->timeout;
/* Completion information. */
p_elem->status = 0; /* Not trusting AL */
if ((p_mad->mgmt_class == IB_MCLASS_SUBN_LID) ||
(p_mad->mgmt_class == IB_MCLASS_SUBN_DIR)) {
p_elem->remote_qp = 0;
p_elem->remote_qkey = 0;
} else {
p_elem->remote_qp = p_mad_addr->addr_type.gsi.remote_qp;
p_elem->remote_qkey = p_mad_addr->addr_type.gsi.remote_qkey;
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_send: "
"remote qp = 0x%X, remote qkey = 0x%X.\n",
cl_ntoh32(p_elem->remote_qp),
cl_ntoh32(p_elem->remote_qkey));
}
status = ib_send_mad(p_bind->h_svc, p_elem, NULL);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 3B29: "
"Send failed (%s).\n", ib_get_err_str(status));
if (p_vw->p_resp_madw)
osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,
IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
{
ib_api_status_t ret = IB_SUCCESS;
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
boolean_t is_rmpp = FALSE, is_rmpp_ds = FALSE;
osmv_txn_ctx_t *p_txn = NULL;
ib_mad_t *p_mad;
osm_log_t *p_log = p_bo->p_vendor->p_log;
osm_mad_pool_t *p_mad_pool = p_bo->p_osm_pool;
OSM_LOG_ENTER(p_log);
if (NULL == h_bind || NULL == p_madw ||
NULL == (p_mad = osm_madw_get_mad_ptr(p_madw)) ||
NULL == osm_madw_get_mad_addr_ptr(p_madw)) {
return IB_INVALID_PARAMETER;
}
is_rmpp = (p_madw->mad_size > MAD_BLOCK_SIZE
|| osmv_mad_is_rmpp(p_mad));
/* is this rmpp double sided? This means we expect a response that can be
an rmpp or not */
is_rmpp_ds = (TRUE == is_rmpp && TRUE == resp_expected);
/* Make our operations with the send context atomic */
osmv_txn_lock(p_bo);
if (TRUE == p_bo->is_closing) {
osm_log(p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 7310: "
"The handle %p is being unbound, cannot send.\n",
h_bind);
ret = IB_INTERRUPTED;
/* When closing p_bo could be detroyed or is going to , thus could not refer to it */
goto send_done;
}
if (TRUE == resp_expected || TRUE == is_rmpp) {
/* We must run under a transaction framework.
* Get the transaction object (old or new) */
ret = __osmv_get_send_txn(h_bind, p_madw, is_rmpp,
resp_expected, &p_txn);
if (IB_SUCCESS != ret) {
goto send_done;
}
}
if (TRUE == is_rmpp) {
/* Do the job - RMPP!
* The call returns as all the packets are ACK'ed/upon error
* The txn lock will be released each time the function sleeps
* and re-acquired when it wakes up
*/
ret = osmv_rmpp_send_madw(h_bind, p_madw, p_txn, is_rmpp_ds);
} else {
/* Do the job - single MAD!
* The call returns as soon as the MAD is put on the wire
*/
ret = osmv_simple_send_madw(h_bind, p_madw, p_txn, FALSE);
}
if (IB_SUCCESS == ret) {
if ((TRUE == is_rmpp) && (FALSE == is_rmpp_ds)) {
/* For double-sided sends, the txn continues to live */
osmv_txn_done(h_bind, osmv_txn_get_key(p_txn),
FALSE /*not in callback */ );
}
if (FALSE == resp_expected) {
osm_mad_pool_put(p_mad_pool, p_madw);
}
} else if (IB_INTERRUPTED != ret) {
if (NULL != p_txn) {
osmv_txn_done(h_bind, osmv_txn_get_key(p_txn),
FALSE /*not in callback */ );
}
osm_log(p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 7311: failed to send MADW %p\n",
p_madw);
if (TRUE == resp_expected) {
/* Change the status on the p_madw */
p_madw->status = ret;
/* Only the requester expects the error callback */
p_bo->send_err_cb(p_bo->cb_context, p_madw);
} else {
/* put back the mad - it is useless ... */
osm_mad_pool_put(p_mad_pool, p_madw);
}
} else { /* the transaction was aborted due to p_bo exit */
osm_mad_pool_put(p_mad_pool, p_madw);
goto aborted;
}
send_done:
osmv_txn_unlock(p_bo);
aborted:
OSM_LOG_EXIT(p_log);
return ret;
}
/* this is the callback function of the timer */
void __osm_transaction_mgr_callback(IN void *context)
{
osm_transaction_mgr_t *trans_mgr_p;
osm_vendor_t *p_vend = (osm_vendor_t *) context;
cl_list_item_t *p_list_item;
cl_list_item_t *p_list_next_item;
osm_madw_req_t *osm_madw_req_p;
uint64_t current_time; /* [usec] */
uint32_t new_timeout; /* [msec] */
cl_status_t cl_status;
ib_mad_t *p_mad;
#ifdef OSM_VENDOR_INTF_MTL
osm_mtl_bind_info_t *p_bind;
#else
osm_ts_bind_info_t *p_bind;
#endif
cl_list_t tmp_madw_p_list; /* this list will include all the madw_p that should be removed. */
cl_list_t retry_madw_p_list; /* this list will include all the madw_p that were retried and need to be removed. */
osm_madw_t *madw_p;
OSM_LOG_ENTER(p_vend->p_log);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
/* initialize the tmp_madw_p_list */
cl_list_construct(&tmp_madw_p_list);
cl_status = cl_list_init(&tmp_madw_p_list, 50);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to create tmp_madw_p_list\n");
}
cl_list_construct(&retry_madw_p_list);
cl_status = cl_list_init(&retry_madw_p_list, 50);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to create retry_madw_p_list\n");
}
current_time = cl_get_time_stamp();
cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
p_list_item = cl_qlist_head(trans_mgr_p->madw_reqs_list_p);
if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* the list is empty - nothing to do */
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Nothing to do\n");
goto Exit;
}
/* non empty list: */
/* get the osm_madw_req_p */
osm_madw_req_p = PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
while (osm_madw_req_p->waking_time <= current_time) {
/* this object was supposed to have gotten a response */
/* we need to decide if we need to retry or done with it. */
if (osm_madw_req_p->retry_cnt > 0) {
/* add to the list of the retrys : */
cl_list_insert_tail(&retry_madw_p_list, osm_madw_req_p);
/* update wakeup time and retry count */
osm_madw_req_p->waking_time =
p_vend->timeout * 1000 + cl_get_time_stamp();
osm_madw_req_p->retry_cnt--;
/* make sure we will get some timer call if not earlier */
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Timer restart:%u\n",
p_vend->timeout);
cl_status =
cl_timer_start(&trans_mgr_p->madw_list_timer,
p_vend->timeout);
/* go to the next object and check if it also needs to be removed - didn't receive response */
/* we need to do it before we move current item to the end of the list */
p_list_next_item = cl_qlist_next(p_list_item);
/* remove from the head */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* insert the object to the qlist and the qmap */
cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
} else {
/* go to the next object and check if it also needs to be removed - didn't receive response */
p_list_next_item = cl_qlist_next(p_list_item);
/* remove from the head */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* add it to the tmp_madw_p_list to be removed */
cl_list_insert_tail(&tmp_madw_p_list,
osm_madw_req_p->p_madw);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Found failed transaction madw: %p\n",
osm_madw_req_p->p_madw);
}
/* Advance */
p_list_item = p_list_next_item;
if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* the list is empty - nothing to do */
break;
}
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
}
/* look at the current p_list_item. If it is not the end item - then we need to */
/* re-start the timer */
if (p_list_item != cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
/* we have the object that still didn't get response - re-start the timer */
/* start the timer to the timeout (in miliseconds) */
new_timeout =
(osm_madw_req_p->waking_time - cl_get_time_stamp()) / 1000 +
1;
cl_status =
cl_timer_start(&trans_mgr_p->madw_list_timer, new_timeout);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Timer restart:%u\n",
new_timeout);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to start timer\n");
}
}
/* if not empty - retry on retry list: */
if (!cl_is_list_empty(&retry_madw_p_list)) {
/* remove all elements that were retried: */
osm_madw_req_p =
(osm_madw_req_t
*) (cl_list_remove_head(&retry_madw_p_list));
while (osm_madw_req_p != NULL) {
/* resend: */
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : "
"Retry %d of madw %p\n",
OSM_DEFAULT_RETRY_COUNT -
osm_madw_req_p->retry_cnt,
osm_madw_req_p->p_madw);
/* actually send it */
#ifdef OSM_VENDOR_INTF_MTL
osm_mtl_send_mad((osm_mtl_bind_info_t *)
osm_madw_req_p->p_bind,
osm_madw_req_p->p_madw);
#else
ib_api_status_t
osm_ts_send_mad(osm_ts_bind_info_t * p_bind,
osm_madw_t * const p_madw);
osm_ts_send_mad((osm_ts_bind_info_t *) osm_madw_req_p->
p_bind, osm_madw_req_p->p_madw);
#endif
/* next one */
osm_madw_req_p =
(osm_madw_req_t
*) (cl_list_remove_head(&retry_madw_p_list));
}
}
/* if the tmp_madw_p_list has elements - need to call the send_err_callback */
madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
while (madw_p != NULL) {
/* need to remove it from pool */
/* obtain the madw_p stored as the wrid in the send call */
p_mad = osm_madw_get_mad_ptr(madw_p);
p_bind = madw_p->h_bind;
/*
Return any wrappers to the pool that may have been
pre-emptively allocated to handle a receive.
*/
if (madw_p->vend_wrap.p_resp_madw) {
#ifdef OSM_VENDOR_INTF_MTL
osm_mad_pool_put(p_bind->p_osm_pool,
madw_p->vend_wrap.p_resp_madw);
#else
osm_mad_pool_put(p_bind->p_osm_pool,
madw_p->vend_wrap.p_resp_madw);
#endif
madw_p->vend_wrap.p_resp_madw = NULL;
}
/* invoke the CB */
(*(osm_vend_mad_send_err_callback_t)
(p_bind->send_err_callback)) (p_bind->client_context, madw_p);
madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
}
