void osm_transaction_mgr_destroy(IN osm_vendor_t * const p_vend)
{
osm_transaction_mgr_t *trans_mgr_p;
cl_list_item_t *p_list_item;
cl_map_item_t *p_map_item;
osm_madw_req_t *osm_madw_req_p;
OSM_LOG_ENTER(p_vend->p_log);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
if (p_vend->p_transaction_mgr != NULL) {
/* we need to get a lock */
cl_spinlock_acquire(&trans_mgr_p->transaction_mgr_lock);
/* go over all the items in the list and remove them */
p_list_item =
cl_qlist_remove_head(trans_mgr_p->madw_reqs_list_p);
while (p_list_item !=
cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
osm_madw_req_p = (osm_madw_req_t *) p_list_item;
if (osm_madw_req_p->p_madw->p_mad)
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_destroy: "
"Found outstanding MADW:%p TID:<0x%"
PRIx64 ">.\n", osm_madw_req_p->p_madw,
osm_madw_req_p->p_madw->p_mad->
trans_id);
else
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_destroy: "
"Found outstanding MADW:%p TID:UNDEFINED.\n",
osm_madw_req_p->p_madw);
/* each item - remove it from the map */
p_map_item = &(osm_madw_req_p->map_item);
cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
p_map_item);
/* free the item */
free(osm_madw_req_p);
p_list_item =
cl_qlist_remove_head(trans_mgr_p->madw_reqs_list_p);
}
/* free the qlist and qmap */
free(trans_mgr_p->madw_reqs_list_p);
free(trans_mgr_p->madw_by_tid_map_p);
/* reliease and destroy the lock */
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
cl_spinlock_destroy(&(trans_mgr_p->transaction_mgr_lock));
/* destroy the timer */
cl_timer_trim(&trans_mgr_p->madw_list_timer, 1);
cl_timer_destroy(&trans_mgr_p->madw_list_timer);
/* free the transaction_manager object */
free(trans_mgr_p);
trans_mgr_p = NULL;
}
OSM_LOG_EXIT(p_vend->p_log);
}
static ib_api_status_t
__osmv_dispatch_accept_seg(IN osm_bind_handle_t h_bind,
IN osmv_txn_ctx_t * p_txn, IN const ib_mad_t * p_mad)
{
ib_api_status_t ret = IB_SUCCESS;
uint32_t seg_num = cl_ntoh32(((ib_rmpp_mad_t *) p_mad)->seg_num);
osmv_rmpp_recv_ctx_t *p_recv_ctx = osmv_txn_get_rmpp_recv_ctx(p_txn);
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
uint64_t tid = osmv_txn_get_tid(p_txn);
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
if (seg_num != p_recv_ctx->expected_seg) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"TID 0x%" PRIx64 ": can't accept this segment (%d) - "
"this is a Go-Back-N implementation\n", tid, seg_num);
return IB_INSUFFICIENT_RESOURCES;
}
/* Store the packet's copy in the reassembly list.
* Promote the expected segment counter.
*/
ret = osmv_rmpp_recv_ctx_store_mad_seg(p_recv_ctx, (uint8_t *) p_mad);
if (IB_SUCCESS != ret) {
return ret;
}
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"TID 0x%" PRIx64 ": segment %d accepted\n", tid, seg_num);
p_recv_ctx->expected_seg = seg_num + 1;
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return IB_SUCCESS;
}
void
osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
{
osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind;
osm_vendor_t *p_vend = p_bind->p_vend;
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vw);
CL_ASSERT(p_vw->p_elem);
CL_ASSERT(p_vw->h_bind == h_bind);
if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_put: "
"Retiring MAD %p.\n", ib_get_mad_buf(p_vw->p_elem));
}
status = ib_put_mad(p_vw->p_elem);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_put: ERR 3B26: "
"Unable to retire MAD (%s).\n", ib_get_err_str(status));
}
OSM_LOG_EXIT(p_vend->p_log);
}
/**********************************************************************
Send a MAD through.
What is unclear to me is the need for the setting of all the MAD Wrapper
fields. Seems like the OSM uses these values during it's processing...
**********************************************************************/
ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,
IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
{
osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
osm_vendor_t *const p_vend = p_bind->p_vend;
osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
/*
* If a response is expected to this MAD, then preallocate
* a mad wrapper to contain the wire MAD received in the
* response. Allocating a wrapper here allows for easier
* failure paths than after we already received the wire mad.
*/
if (resp_expected == TRUE) {
/* we track it in the vendor wrapper */
p_vw->p_resp_madw =
osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
if (p_vw->p_resp_madw == NULL) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 5024: "
"Unable to allocate MAD wrapper.\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/* put some minimal info on that wrapper */
((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
/* we also want to track it in the TID based map */
status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *)
p_bind, p_madw);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_send: ERR 5025: "
"Error inserting request madw by TID (%d).\n",
status);
}
} else
p_vw->p_resp_madw = NULL;
/* do the actual send */
/* HACK: to be replaced by call to RMPP Segmentation */
status = osm_ts_send_mad(p_bind, p_madw);
/* we do not get an asycn callback so call it ourselves */
/* this will handle all cleanup if neccessary */
__osm_ts_send_callback(p_bind, !resp_expected, p_madw, status);
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
/**********************************************************************
* Poller thread:
* Always receive 256byte mads from the devcie file
**********************************************************************/
void __osm_vendor_ts_poller(IN void *p_ptr)
{
int ts_ret_code;
struct ib_mad mad;
osm_mad_addr_t mad_addr;
osm_ts_bind_info_t *const p_bind = (osm_ts_bind_info_t *) p_ptr;
OSM_LOG_ENTER(p_bind->p_vend->p_log);
/* we set the type of cancelation for this thread */
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
while (1) {
/* we read one mad at a time and pass it to the read callback function */
ts_ret_code = read(p_bind->ul_dev_fd, &mad, sizeof(mad));
if (ts_ret_code != sizeof(mad)) {
osm_log(p_bind->p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_ts_poller: ERR 5003: "
"error with read, bytes = %d, errno = %d\n",
ts_ret_code, errno);
} else {
osm_log(p_bind->p_vend->p_log, OSM_LOG_DEBUG,
"__osm_vendor_ts_poller: "
"MAD QPN:%d SLID:0x%04x class:0x%02x "
"__osm_vendor_ts_poller:0x%02x attr:0x%04x status:0x%04x "
"__osm_vendor_ts_poller:0x%016" PRIx64 "\n",
cl_ntoh32(mad.dqpn),
cl_ntoh16(mad.slid),
mad.mgmt_class,
mad.r_method,
cl_ntoh16(mad.attribute_id),
cl_ntoh16(mad.status),
cl_ntoh64(mad.transaction_id));
/* first arrange an address */
__osm_ts_conv_mad_rcv_desc_to_osm_addr(p_bind->p_vend,
&mad,
(((ib_mad_t *) &
mad)->
mgmt_class ==
IB_MCLASS_SUBN_LID)
||
(((ib_mad_t *) &
mad)->
mgmt_class ==
IB_MCLASS_SUBN_DIR),
&mad_addr);
/* call the receiver callback */
/* HACK: this should be replaced with a call to the RMPP Assembly ... */
__osm_ts_rcv_callback(p_bind, &mad_addr, 256, &mad);
}
}
OSM_LOG_EXIT(p_bind->p_vend->p_log);
}
static void
__osmv_dispatch_simple_mad(IN osm_bind_handle_t h_bind,
IN const ib_mad_t * p_mad,
IN osmv_txn_ctx_t * p_txn,
IN const osm_mad_addr_t * p_mad_addr)
{
osm_madw_t *p_madw;
ib_mad_t *p_mad_buf;
osm_madw_t *p_req_madw = NULL;
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
/* Build the MAD wrapper to be returned to the user.
* The actual storage for the MAD is allocated there.
*/
p_madw =
osm_mad_pool_get(p_bo->p_osm_pool, h_bind, MAD_BLOCK_SIZE,
p_mad_addr);
if (NULL == p_madw) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"__osmv_dispatch_simple_mad: ERR 6501: "
"Out Of Memory - could not allocate a buffer of size %d\n",
MAD_BLOCK_SIZE);
goto dispatch_simple_mad_done;
}
p_mad_buf = osm_madw_get_mad_ptr(p_madw);
/* Copy the payload to the MAD buffer */
memcpy((void *)p_mad_buf, (void *)p_mad, MAD_BLOCK_SIZE);
if (NULL != p_txn) {
/* This is a RESPONSE MAD. Pair it with the REQUEST MAD, pass upstream */
p_req_madw = p_txn->p_madw;
CL_ASSERT(NULL != p_req_madw);
p_mad_buf->trans_id = cl_hton64(osmv_txn_get_tid(p_txn));
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"Restoring the original TID to 0x%" PRIx64 "\n",
cl_ntoh64(p_mad_buf->trans_id));
/* Reply matched, transaction complete */
osmv_txn_done(h_bind, osmv_txn_get_key(p_txn), FALSE);
} else {
/* This is a REQUEST MAD. Don't create a context, pass upstream */
}
/* Do the job ! */
p_bo->recv_cb(p_madw, p_bo->cb_context, p_req_madw);
dispatch_simple_mad_done:
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
}
static ib_api_status_t
__osm_vendor_open_ca(IN osm_vendor_t * const p_vend,
IN const ib_net64_t port_guid)
{
ib_net64_t ca_guid;
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
ca_guid = osm_vendor_get_ca_guid(p_vend, port_guid);
if (ca_guid == 0) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_open_ca: ERR 3B31: "
"Bad port GUID value 0x%" PRIx64 ".\n",
cl_ntoh64(port_guid));
status = IB_ERROR;
goto Exit;
}
osm_log(p_vend->p_log, OSM_LOG_VERBOSE,
"__osm_vendor_open_ca: "
"Opening HCA 0x%" PRIx64 ".\n", cl_ntoh64(ca_guid));
status = ib_open_ca(p_vend->h_al,
ca_guid,
__osm_al_ca_err_callback, p_vend, &p_vend->h_ca);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_open_ca: ERR 3B15: "
"Unable to open CA (%s).\n", ib_get_err_str(status));
goto Exit;
}
CL_ASSERT(p_vend->h_ca);
status = ib_alloc_pd(p_vend->h_ca, IB_PDT_ALIAS, p_vend, &p_vend->h_pd);
if (status != IB_SUCCESS) {
ib_close_ca(p_vend->h_ca, __osm_al_ca_destroy_callback);
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_open_ca: ERR 3B16: "
"Unable to allocate protection domain (%s).\n",
ib_get_err_str(status));
goto Exit;
}
CL_ASSERT(p_vend->h_pd);
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
void osmv_txn_lock(IN osm_bind_handle_t h_bind)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"--> Acquiring lock %p on bind handle %p\n", &p_bo->lock, p_bo);
cl_spinlock_acquire(&p_bo->lock);
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"--> Acquired lock %p on bind handle %p\n", &p_bo->lock, p_bo);
}
static ib_api_status_t
__osm_ca_info_init(IN osm_vendor_t * const p_vend,
IN osm_ca_info_t * const p_ca_info,
IN const ib_net64_t ca_guid)
{
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
p_ca_info->guid = ca_guid;
if (osm_log_is_active(p_vend->p_log, OSM_LOG_VERBOSE)) {
osm_log(p_vend->p_log, OSM_LOG_VERBOSE,
"__osm_ca_info_init: "
"Querying CA 0x%" PRIx64 ".\n", cl_ntoh64(ca_guid));
}
status = ib_query_ca_by_guid(p_vend->h_al, ca_guid, NULL,
&p_ca_info->attr_size);
if ((status != IB_INSUFFICIENT_MEMORY) && (status != IB_SUCCESS)) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3B05: "
"Unexpected status getting CA attributes (%s).\n",
ib_get_err_str(status));
goto Exit;
}
CL_ASSERT(p_ca_info->attr_size);
p_ca_info->p_attr = malloc(p_ca_info->attr_size);
if (p_ca_info->p_attr == NULL) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3B06: "
"Unable to allocate attribute storage.\n");
goto Exit;
}
status = ib_query_ca_by_guid(p_vend->h_al, ca_guid, p_ca_info->p_attr,
&p_ca_info->attr_size);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3B07: "
"Unexpected status getting CA attributes (%s).\n",
ib_get_err_str(status));
goto Exit;
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
ib_api_status_t
osm_transaction_mgr_get_madw_for_tid(IN osm_vendor_t * const p_vend,
IN ib_mad_t * const p_mad,
OUT osm_madw_t ** req_madw_p)
{
osm_transaction_mgr_t *trans_mgr_p;
osm_madw_req_t *osm_madw_req_p;
cl_map_item_t *p_map_item;
uint64_t key;
OSM_LOG_ENTER(p_vend->p_log);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
*req_madw_p = NULL;
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_get_madw_for_tid: "
"Looking for TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
key = (uint64_t) p_mad->trans_id;
cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
p_map_item = cl_qmap_get(trans_mgr_p->madw_by_tid_map_p, key);
if (p_map_item != cl_qmap_end(trans_mgr_p->madw_by_tid_map_p)) {
/* we found such an item. */
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_map_item, osm_madw_req_t, map_item);
/* Since the Transaction was looked up and provided for */
/* processing we retire it */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* remove the item from the qmap */
cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
&(osm_madw_req_p->map_item));
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_get_madw_for_tid: "
"Removed TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
*req_madw_p = osm_madw_req_p->p_madw;
}
cl_spinlock_release(&(trans_mgr_p->transaction_mgr_lock));
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_get_madw_for_tid: "
"Got MADW:%p.\n", *req_madw_p);
OSM_LOG_EXIT(p_vend->p_log);
return (IB_SUCCESS);
}
static ib_api_status_t
__osm_vendor_get_ca_guids(IN osm_vendor_t * const p_vend,
IN ib_net64_t ** const p_guids,
IN uintn_t * const p_num_guids)
{
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_guids);
CL_ASSERT(p_num_guids);
status = ib_get_ca_guids(p_vend->h_al, NULL, p_num_guids);
if ((status != IB_INSUFFICIENT_MEMORY) && (status != IB_SUCCESS)) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_get_ca_guids: ERR 3B08: "
"Unexpected status getting CA GUID array (%s).\n",
ib_get_err_str(status));
goto Exit;
}
if (*p_num_guids == 0) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_get_ca_guids: ERR 3B09: "
"No available channel adapters.\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
*p_guids = malloc(*p_num_guids * sizeof(**p_guids));
if (*p_guids == NULL) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_vendor_get_ca_guids: ERR 3B10: "
"Unable to allocate CA GUID array.\n");
goto Exit;
}
status = ib_get_ca_guids(p_vend->h_al, *p_guids, p_num_guids);
CL_ASSERT(*p_num_guids);
if (osm_log_is_active(p_vend->p_log, OSM_LOG_VERBOSE)) {
osm_log(p_vend->p_log, OSM_LOG_VERBOSE,
"__osm_vendor_get_ca_guids: "
"Detected %u local channel adapters.\n", *p_num_guids);
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
ib_api_status_t
osm_transaction_mgr_erase_madw(IN osm_vendor_t * const p_vend,
IN ib_mad_t * p_mad)
{
osm_transaction_mgr_t *trans_mgr_p;
osm_madw_req_t *osm_madw_req_p;
uint64_t key;
cl_map_item_t *p_map_item;
OSM_LOG_ENTER(p_vend->p_log);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
key = (uint64_t) p_mad->trans_id;
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_erase_madw: "
"Removing TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
cl_spinlock_acquire(&trans_mgr_p->transaction_mgr_lock);
p_map_item = cl_qmap_get(trans_mgr_p->madw_by_tid_map_p, key);
if (p_map_item != cl_qmap_end(trans_mgr_p->madw_by_tid_map_p)) {
/* we found such an item. */
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_map_item, osm_madw_req_t, map_item);
/* remove the item from the qlist */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* remove the item from the qmap */
cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
&(osm_madw_req_p->map_item));
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_erase_madw: "
"Removed TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
/* free the item */
free(osm_madw_req_p);
} else {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_erase_madw: "
"osm_transaction_mgr_erase_madw:<0x%" PRIx64
"> NOT FOUND.\n", p_mad->trans_id);
}
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
OSM_LOG_EXIT(p_vend->p_log);
return (IB_SUCCESS);
}
ib_api_status_t
osmv_txn_init(IN osm_bind_handle_t h_bind,
IN uint64_t tid, IN uint64_t key, OUT osmv_txn_ctx_t ** pp_txn)
{
ib_api_status_t st;
osmv_txn_ctx_t *p_txn;
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
CL_ASSERT(NULL != h_bind && NULL != pp_txn);
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"Starting transaction 0x%016" PRIx64
" (key=0x%016" PRIx64 ")\n", tid, key);
p_txn = malloc(sizeof(osmv_txn_ctx_t));
if (!p_txn) {
return IB_INSUFFICIENT_MEMORY;
}
memset(p_txn, 0, sizeof(osmv_txn_ctx_t));
p_txn->p_log = p_bo->txn_mgr.p_log;
p_txn->tid = tid;
p_txn->key = key;
p_txn->p_madw = NULL;
p_txn->rmpp_txfr.rmpp_state = OSMV_TXN_RMPP_NONE;
/* insert into transaction manager DB */
st = __osmv_txnmgr_insert_txn(&p_bo->txn_mgr, p_txn, key);
if (IB_SUCCESS != st) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"osmv_txn_init: ERR 6703: "
"Failed to insert to transaction 0x%016" PRIx64
" (key=0x%016" PRIx64 ") to manager DB\n",
tid, key);
goto insert_txn_failed;
}
*pp_txn = p_txn;
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return IB_SUCCESS;
insert_txn_failed:
free(p_txn);
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return st;
}
static void __osm_vendor_internal_unbind(osm_bind_handle_t h_bind)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_log_t *p_log = p_bo->p_vendor->p_log;
OSM_LOG_ENTER(p_log);
/* "notifying" all that from now on no new sends can be done */
p_bo->txn_mgr.p_event_wheel->closing = TRUE;
osmv_txn_lock(p_bo);
/*
the is_closing is set under lock we we know we only need to
check for it after obtaining the lock
*/
p_bo->is_closing = TRUE;
/* notifying all sleeping rmpp sends to exit */
osmv_txn_abort_rmpp_txns(h_bind);
/* unlock the bo to allow for any residual mads to be dispatched */
osmv_txn_unlock(p_bo);
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_vendor_internal_unbind: destroying transport mgr.. \n");
/* wait for the receiver thread to exit */
osmv_transport_done(h_bind);
/* lock to avoid any collissions while we cleanup the structs */
osmv_txn_lock(p_bo);
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_vendor_internal_unbind: destroying txn mgr.. \n");
osmv_txnmgr_done(h_bind);
osm_log(p_log, OSM_LOG_DEBUG,
"__osm_vendor_internal_unbind: destroying bind lock.. \n");
osmv_txn_unlock(p_bo);
/*
we intentionally let the p_bo and its lock leak -
as we did not implement a way to track active bind handles provided to
the client - and the client might use them
cl_spinlock_destroy(&p_bo->lock);
free(p_bo);
*/
OSM_LOG_EXIT(p_log);
}
/**********************************************************************
* Create and Initialize osm_vendor_t Object
**********************************************************************/
osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
IN const uint32_t timeout)
{
ib_api_status_t status;
osm_vendor_t *p_vend;
OSM_LOG_ENTER(p_log);
CL_ASSERT(p_log);
p_vend = malloc(sizeof(*p_vend));
if (p_vend != NULL) {
memset(p_vend, 0, sizeof(*p_vend));
status = osm_vendor_init(p_vend, p_log, timeout);
if (status != IB_SUCCESS) {
osm_vendor_delete(&p_vend);
}
} else {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_new: ERR 5007: "
"Fail to allocate vendor object.\n");
}
OSM_LOG_EXIT(p_log);
return (p_vend);
}
void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_vendor_t const *p_vend = p_bo->p_vendor;
int ret;
ibms_cap_msg_t cap_msg;
OSM_LOG_ENTER(p_vend->p_log);
cap_msg.mask = IB_PORT_CAP_IS_SM;
if (is_sm_val)
cap_msg.capabilities = IB_PORT_CAP_IS_SM;
else
cap_msg.capabilities = 0;
ret = ibms_set_cap(((osmv_ibms_transport_mgr_t *) (p_bo->
p_transp_mgr))->
conHdl, &cap_msg);
if (ret) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_set_sm: ERR 5312: "
"Unable set 'IS_SM' bit to:%u in port attributes.\n",
is_sm_val);
}
OSM_LOG_EXIT(p_vend->p_log);
}
ib_api_status_t
osm_vendor_init(IN osm_vendor_t * const p_vend,
IN osm_log_t * const p_log, IN const uint32_t timeout)
{
ib_api_status_t status;
OSM_LOG_ENTER(p_log);
p_vend->p_log = p_log;
/*
Open our instance of AL.
*/
status = ib_open_al(&p_vend->h_al);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_init: ERR 3B03: "
"Error opening AL (%s).\n", ib_get_err_str(status));
goto Exit;
}
p_vend->timeout = timeout;
Exit:
OSM_LOG_EXIT(p_log);
return (status);
}
void osmv_txn_unlock(IN osm_bind_handle_t h_bind)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
cl_spinlock_t *p_lock = &p_bo->lock;
osm_log_t *p_log = p_bo->p_vendor->p_log;
osm_log(p_log, OSM_LOG_DEBUG,
"<-- Releasing lock %p on bind handle %p\n", p_lock, p_bo);
cl_spinlock_release(&p_bo->lock);
/* We'll use the saved ptrs, since now the p_bo can be destroyed already */
osm_log(p_log, OSM_LOG_DEBUG,
"<-- Released lock %p on bind handle %p\n", p_lock, p_bo);
}
ib_api_status_t
__osmv_txnmgr_remove_txn(IN osmv_txn_mgr_t * p_tx_mgr,
IN uint64_t key, OUT osmv_txn_ctx_t ** pp_txn)
{
cl_map_obj_t *p_obj;
cl_map_item_t *p_item;
OSM_LOG_ENTER(p_tx_mgr->p_log);
CL_ASSERT(p_tx_mgr);
CL_ASSERT(pp_txn);
p_item = cl_qmap_remove(p_tx_mgr->p_txn_map, key);
if (p_item == cl_qmap_end(p_tx_mgr->p_txn_map)) {
osm_log(p_tx_mgr->p_log, OSM_LOG_ERROR,
"__osmv_txnmgr_remove_txn: ERR 6701: "
"Could not remove the transaction 0x%llX - "
"something is really wrong!\n", key);
OSM_LOG_EXIT(p_tx_mgr->p_log);
return IB_NOT_FOUND;
}
p_obj = PARENT_STRUCT(p_item, cl_map_obj_t, item);
*pp_txn = cl_qmap_obj(p_obj);
free(p_obj);
OSM_LOG_EXIT(p_tx_mgr->p_log);
return IB_SUCCESS;
}
ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
IN const uint32_t size,
IN osm_vend_wrap_t * const p_vend_wrap)
{
osm_vendor_t *p_vend;
ib_mad_t *p_mad;
OSM_LOG_ENTER(h_bind->p_vend->p_log);
UNUSED_PARAM(p_vend_wrap);
p_vend = h_bind->p_vend;
/*
Simply malloc the MAD off the heap.
*/
p_mad = (ib_mad_t *) malloc(size);
osm_log(p_vend->p_log, OSM_LOG_VERBOSE,
"osm_vendor_get: " "MAD %p.\n", p_mad);
if (p_mad)
memset(p_mad, 0, size);
OSM_LOG_EXIT(p_vend->p_log);
return (p_mad);
}
void
osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_vendor_t const *p_vend = p_bo->p_vendor;
if (p_bo->is_closing != TRUE) {
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vw);
CL_ASSERT(p_vw->p_mad);
if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_put: " "Retiring MAD %p.\n",
p_vw->p_mad);
}
free(p_vw->p_mad);
p_vw->p_mad = NULL;
OSM_LOG_EXIT(p_vend->p_log);
}
}
osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
IN const uint32_t timeout)
{
ib_api_status_t status;
osm_vendor_t *p_vend;
OSM_LOG_ENTER(p_log);
p_vend = malloc(sizeof(*p_vend));
if (p_vend == NULL) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_new: ERR 3B04: "
"Unable to allocate vendor object.\n");
goto Exit;
}
memset(p_vend, 0, sizeof(*p_vend));
status = osm_vendor_init(p_vend, p_log, timeout);
if (status != IB_SUCCESS) {
free(p_vend);
p_vend = NULL;
}
Exit:
OSM_LOG_EXIT(p_log);
return (p_vend);
}
/**********************************************************************
* Return a MAD by providing it's wrapper object.
**********************************************************************/
void
osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
{
osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
osm_vendor_t *p_vend = p_bind->p_vend;
osm_madw_t *p_madw;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vw);
CL_ASSERT(p_vw->p_mad_buf);
if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_put: " "Retiring MAD %p.\n",
p_vw->p_mad_buf);
}
/*
* We moved the removal of the transaction to immediatly after
* it was looked up.
*/
/* free the mad but the wrapper is part of the madw object */
free(p_vw->p_mad_buf);
p_vw->p_mad_buf = NULL;
p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
p_madw->p_mad = NULL;
OSM_LOG_EXIT(p_vend->p_log);
}
void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_vendor_t const *p_vend = p_bo->p_vendor;
int ts_ioctl_ret;
int device_fd =
((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd;
struct ib_set_port_info_ioctl set_port_data;
OSM_LOG_ENTER(p_vend->p_log);
memset(&set_port_data, 0, sizeof(set_port_data));
set_port_data.port = p_bo->port_num;
set_port_data.port_info.valid_fields = IB_PORT_IS_SM;
set_port_data.port_info.is_sm = is_sm_val;
ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data);
if (ts_ioctl_ret < 0) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_set_sm: ERR 6805: "
"Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
is_sm_val, ts_ioctl_ret);
}
OSM_LOG_EXIT(p_vend->p_log);
}
void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
{
osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind;
osm_vendor_t *p_vend = p_bind->p_vend;
ib_api_status_t status;
ib_port_attr_mod_t attr_mod;
OSM_LOG_ENTER(p_vend->p_log);
memset(&attr_mod, 0, sizeof(attr_mod));
attr_mod.cap.sm = is_sm_val;
status = ib_modify_ca(p_vend->h_ca, p_bind->port_num,
IB_CA_MOD_IS_SM, &attr_mod);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_set_sm: ERR 3B34: "
"Unable set 'IS_SM' bit to:%u in port attributes (%s).\n",
is_sm_val, ib_get_err_str(status));
}
OSM_LOG_EXIT(p_vend->p_log);
}
ib_api_status_t
osmv_rmpp_send_ack(IN osm_bind_handle_t h_bind,
IN const ib_mad_t * p_req_mad,
IN uint32_t seg_num,
IN uint32_t nwl, IN const osm_mad_addr_t * p_mad_addr)
{
uint8_t resp_mad[MAD_BLOCK_SIZE];
ib_rmpp_mad_t *p_resp_mad = (ib_rmpp_mad_t *) resp_mad;
#ifdef OSMV_RANDOM_DROP
if (TRUE == osmv_random_drop()) {
osm_log(((osmv_bind_obj_t *) h_bind)->p_vendor->p_log,
OSM_LOG_DEBUG,
"Error injection - dropping the RMPP ACK\n");
return IB_SUCCESS;
}
#endif
memcpy(p_resp_mad, p_req_mad, MAD_BLOCK_SIZE);
p_resp_mad->common_hdr.method = osmv_invert_method(p_req_mad->method);
p_resp_mad->rmpp_type = IB_RMPP_TYPE_ACK;
p_resp_mad->seg_num = cl_hton32(seg_num);
p_resp_mad->paylen_newwin = cl_hton32(nwl);
p_resp_mad->rmpp_flags = IB_RMPP_FLAG_ACTIVE;
return osmv_transport_mad_send(h_bind, p_resp_mad, p_mad_addr);
}
void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
{
osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
osm_vendor_t *p_vend = p_bind->p_vend;
VAPI_ret_t status;
VAPI_hca_attr_t attr_mod;
VAPI_hca_attr_mask_t attr_mask;
OSM_LOG_ENTER(p_vend->p_log);
memset(&attr_mod, 0, sizeof(attr_mod));
memset(&attr_mask, 0, sizeof(attr_mask));
attr_mod.is_sm = is_sm_val;
attr_mask = HCA_ATTR_IS_SM;
status =
VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
&attr_mask);
if (status != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_set_sm: ERR 5027: "
"Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
is_sm_val, status);
}
OSM_LOG_EXIT(p_vend->p_log);
}
/**********************************************************************
Actually Send a MAD
MADs are buffers of type: struct ib_mad - so they are limited by size.
This is for internal use by osm_vendor_send and the transaction mgr
retry too.
**********************************************************************/
ib_api_status_t
osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
{
osm_vendor_t *const p_vend = p_bind->p_vend;
osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
struct ib_mad ts_mad;
int ret;
ib_api_status_t status;
OSM_LOG_ENTER(p_vend->p_log);
/*
* Copy the MAD over to the sent mad
*/
memcpy(&ts_mad, p_mad, 256);
/*
* For all sends other than directed route SM MADs,
* acquire an address vector for the destination.
*/
if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
__osm_ts_conv_osm_addr_to_ts_addr(p_mad_addr,
p_mad->mgmt_class ==
IB_MCLASS_SUBN_LID, &ts_mad);
} else {
/* is a directed route - we need to construct a permissive address */
/* we do not need port number since it is part of the mad_hndl */
ts_mad.dlid = IB_LID_PERMISSIVE;
ts_mad.slid = IB_LID_PERMISSIVE;
}
if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
(p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
ts_mad.sqpn = 0;
ts_mad.dqpn = 0;
} else {
ts_mad.sqpn = 1;
ts_mad.dqpn = 1;
}
ts_mad.port = p_bind->port_num;
/* send it */
ret = write(p_bind->ul_dev_fd, &ts_mad, sizeof(ts_mad));
if (ret != sizeof(ts_mad)) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_ts_send_mad: ERR 5026: "
"Error sending mad (%d).\n", ret);
status = IB_ERROR;
goto Exit;
}
status = IB_SUCCESS;
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
ib_api_status_t
osmv_simple_send_madw(IN osm_bind_handle_t h_bind,
IN osm_madw_t * const p_madw,
IN osmv_txn_ctx_t * p_txn, IN boolean_t is_retry)
{
ib_api_status_t ret;
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
osm_mad_addr_t *p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
uint8_t mad_buf[MAD_BLOCK_SIZE];
ib_mad_t *p_mad = (ib_mad_t *) mad_buf;
uint64_t key = 0;
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
CL_ASSERT(p_madw->mad_size <= MAD_BLOCK_SIZE);
memset(p_mad, 0, MAD_BLOCK_SIZE);
memcpy(p_mad, osm_madw_get_mad_ptr(p_madw), p_madw->mad_size);
if (NULL != p_txn) {
/* Push a fake txn id to the MAD */
key = osmv_txn_get_key(p_txn);
p_mad->trans_id = cl_hton64(key);
}
/*
Add call for packet drop randomizer.
This is a testing feature. If run_randomizer flag is set to TRUE,
the randomizer will be called, and randomally will drop
a packet. This is used for simulating unstable fabric.
*/
if (p_bo->p_vendor->run_randomizer == TRUE) {
/* Try the randomizer */
if (osm_pkt_randomizer_mad_drop(p_bo->p_vendor->p_log,
p_bo->p_vendor->
p_pkt_randomizer,
p_mad) == TRUE) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"The MAD will not be sent. \n");
ret = IB_SUCCESS;
} else {
ret =
osmv_transport_mad_send(h_bind, p_mad, p_mad_addr);
}
} else {
ret = osmv_transport_mad_send(h_bind, p_mad, p_mad_addr);
}
if ((IB_SUCCESS == ret) && (NULL != p_txn) && (!is_retry)) {
/* Set the timeout for receiving the response MAD */
ret = osmv_txn_set_timeout_ev(h_bind, key,
p_bo->p_vendor->resp_timeout);
}
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return ret;
}
void osm_report_sm_state(osm_sm_t * sm)
{
char buf[64];
const char *state_str = osm_get_sm_mgr_state_str(sm->p_subn->sm_state);
osm_log(sm->p_log, OSM_LOG_SYS, "Entering %s state\n", state_str);
snprintf(buf, sizeof(buf), "ENTERING SM %s STATE", state_str);
OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, buf);
}
