/********************************************************************** Send a MAD through. What is unclear to me is the need for the setting of all the MAD Wrapper fields. Seems like the OSM uses these values during it's processing... **********************************************************************/ ib_api_status_t osm_vendor_send(IN osm_bind_handle_t h_bind, IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) { osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; osm_vendor_t *const p_vend = p_bind->p_vend; osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); ib_api_status_t status; OSM_LOG_ENTER(p_vend->p_log); /* * If a response is expected to this MAD, then preallocate * a mad wrapper to contain the wire MAD received in the * response. Allocating a wrapper here allows for easier * failure paths than after we already received the wire mad. */ if (resp_expected == TRUE) { /* we track it in the vendor wrapper */ p_vw->p_resp_madw = osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); if (p_vw->p_resp_madw == NULL) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_send: ERR 5024: " "Unable to allocate MAD wrapper.\n"); status = IB_INSUFFICIENT_RESOURCES; goto Exit; } /* put some minimal info on that wrapper */ ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind; /* we also want to track it in the TID based map */ status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *) p_bind, p_madw); if (status != IB_SUCCESS) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_send: ERR 5025: " "Error inserting request madw by TID (%d).\n", status); } } else p_vw->p_resp_madw = NULL; /* do the actual send */ /* HACK: to be replaced by call to RMPP Segmentation */ status = osm_ts_send_mad(p_bind, p_madw); /* we do not get an asycn callback so call it ourselves */ /* this will handle all cleanup if neccessary */ __osm_ts_send_callback(p_bind, !resp_expected, p_madw, status); Exit: OSM_LOG_EXIT(p_vend->p_log); return (status); }
/********************************************************************** * TS Send callback : invoked after each send * **********************************************************************/ void __osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p, IN boolean_t is_resp, IN osm_madw_t * madw_p, IN IB_comp_status_t status) { osm_log_t *const p_log = bind_info_p->p_vend->p_log; osm_vend_wrap_t *p_vw; OSM_LOG_ENTER(p_log); osm_log(p_log, OSM_LOG_DEBUG, "__osm_ts_send_callback: INFO 1008: " "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp); /* we need to handle requests and responses differently */ if (is_resp) { if (status != IB_COMP_SUCCESS) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_send_callback: ERR 5012: " "Error Sending Response MADW:%p.\n", madw_p); } else { osm_log(p_log, OSM_LOG_DEBUG, "__osm_ts_send_callback: DBG 1008: " "Completed Sending Response MADW:%p.\n", madw_p); } /* if we are a response - we need to clean it up */ osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p); } else { /* this call back is invoked on completion of send - error or not */ if (status != IB_COMP_SUCCESS) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_send_callback: ERR 5013: " "Received an Error from IB_MGT Send (%d).\n", status); p_vw = osm_madw_get_vend_ptr(madw_p); CL_ASSERT(p_vw); /* Return any wrappers to the pool that may have been pre-emptively allocated to handle a receive. */ if (p_vw->p_resp_madw) { osm_mad_pool_put(bind_info_p->p_osm_pool, p_vw->p_resp_madw); p_vw->p_resp_madw = NULL; } /* invoke the CB */ (*(osm_vend_mad_send_err_callback_t) bind_info_p-> send_err_callback) (bind_info_p->client_context, madw_p); } else { /* successful request send - do nothing - the response will need the out mad */ osm_log(p_log, OSM_LOG_DEBUG, "__osm_ts_send_callback: DBG 1008: " "Completed Sending Request MADW:%p.\n", madw_p); } } OSM_LOG_EXIT(p_log); }
/********************************************************************** * TS RCV Thread callback * HACK: - we need to make this support arbitrary size mads. **********************************************************************/ void __osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind, IN osm_mad_addr_t * p_mad_addr, IN uint32_t mad_size, IN void *p_mad) { ib_api_status_t status; osm_madw_t *p_req_madw = NULL; osm_madw_t *p_madw; osm_vend_wrap_t *p_new_vw; ib_mad_t *p_mad_buf; osm_log_t *const p_log = p_bind->p_vend->p_log; OSM_LOG_ENTER(p_log); /* if it is a response MAD we mustbe able to get the request */ if (ib_mad_is_response((ib_mad_t *) p_mad)) { /* can we find a matching madw by this payload TID */ status = osm_transaction_mgr_get_madw_for_tid(p_bind->p_vend, (ib_mad_t *) p_mad, &p_req_madw); if (status != IB_SUCCESS) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_rcv_callback: ERR 5008: " "Error obtaining request madw by TID (%d).\n", status); p_req_madw = NULL; } if (p_req_madw == NULL) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_rcv_callback: ERR 5009: " "Fail to obtain request madw for receined MAD. Aborting CB.\n"); goto Exit; } } /* do we have a request ??? */ if (p_req_madw == NULL) { /* if not - get new osm_madw and arrange it. */ /* create the new madw in the pool */ p_madw = osm_mad_pool_get(p_bind->p_osm_pool, (osm_bind_handle_t) p_bind, mad_size, p_mad_addr); if (p_madw == NULL) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_rcv_callback: ERR 5010: " "Error request for a new madw.\n"); goto Exit; } /* HACK: we cust to avoid the const ??? */ p_mad_buf = (void *)p_madw->p_mad; } else { /* we have the madw defined during the send and stored in the vend_wrap */ /* we need to make sure the wrapper is correctly init there */ CL_ASSERT(p_req_madw->vend_wrap.p_resp_madw != 0); p_madw = p_req_madw->vend_wrap.p_resp_madw; CL_ASSERT(p_madw->h_bind); p_mad_buf = osm_vendor_get(p_madw->h_bind, mad_size, &p_madw->vend_wrap); if (p_mad_buf == NULL) { osm_log(p_log, OSM_LOG_ERROR, "__osm_ts_rcv_callback: ERR 5011: " "Unable to acquire wire MAD.\n"); goto Exit; } /* Finally, attach the wire MAD to this wrapper. */ osm_madw_set_mad(p_madw, p_mad_buf); } /* init some fields of the vendor wrapper */ p_new_vw = osm_madw_get_vend_ptr(p_madw); p_new_vw->h_bind = p_bind; p_new_vw->size = mad_size; p_new_vw->p_resp_madw = NULL; p_new_vw->p_mad_buf = p_mad_buf; memcpy(p_new_vw->p_mad_buf, p_mad, mad_size); /* attach the buffer to the wrapper */ p_madw->p_mad = p_mad_buf; /* we can also make sure we marked the size and bind on the returned madw */ p_madw->h_bind = p_new_vw->h_bind; /* call the CB */ (*(osm_vend_mad_recv_callback_t) p_bind->rcv_callback) (p_madw, p_bind->client_context, p_req_madw); Exit: OSM_LOG_EXIT(p_log); }
void __mad_recv_processor(IN void *context) { mad_bind_info_t *p_mad_bind_info = (mad_bind_info_t *) context; umadt_obj_t *p_umadt_obj; osm_madw_t *p_osm_madw = NULL; osm_vend_wrap_t *p_vend_wrap = NULL; osm_mad_addr_t osm_mad_addr = { 0 }; cl_list_item_t *p_list_item; void *transaction_context; FSTATUS Status; MadtStruct *pRecvMad = NULL; MadWorkCompletion *pRecvCmp = NULL; CL_ASSERT(context); p_mad_bind_info = (mad_bind_info_t *) context; p_umadt_obj = p_mad_bind_info->p_umadt_obj; /* PollFor a completion */ /* if FNOTFOND, then wait for a completion then again poll and return the MAD */ while (1) { Status = p_umadt_obj->uMadtInterface. uMadtPollForRecvCompletion(p_mad_bind_info->umadt_handle, &pRecvMad, &pRecvCmp); if (Status != FSUCCESS) { if (Status == FNOT_FOUND) { /* Wait for a completion */ Status = p_umadt_obj->uMadtInterface.uMadtWaitForAnyCompletion(p_mad_bind_info->umadt_handle, RECV_COMPLETION, 0x5000); /* 5 sec timeout */ if (Status == FTIMEOUT) { continue; } CL_ASSERT(Status == FSUCCESS); Status = p_umadt_obj->uMadtInterface. uMadtPollForRecvCompletion(p_mad_bind_info-> umadt_handle, &pRecvMad, &pRecvCmp); if (Status != FSUCCESS) { printf (" mad_recv_worker: Error in PollForRecv returning <%x>\n", Status); CL_ASSERT(0); } } else { printf ("uMadtPollForRecvCompletion Status=<%x>\n", Status); CL_ASSERT(0); } } CL_ASSERT(pRecvMad); CL_ASSERT(pRecvCmp); if (((ib_sa_mad_t_vM3 *) (&pRecvMad->IBMad))->frag_flag & 0x20) { /* Ignore the ACK packet */ Status = p_umadt_obj->uMadtInterface. uMadtReleaseRecvMad(p_mad_bind_info->umadt_handle, pRecvMad); continue; } /* */ /* Extract the return address to pass it on to the client */ /* */ osm_mad_addr.dest_lid = pRecvCmp->AddressInfo.DestLid; osm_mad_addr.path_bits = pRecvCmp->AddressInfo.PathBits; osm_mad_addr.static_rate = pRecvCmp->AddressInfo.StaticRate; if (p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_LID || p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_DIR) { osm_mad_addr.addr_type.smi.source_lid = pRecvCmp->AddressInfo.AddrType.Smi.SourceLid; /* osm_mad_addr.addr_type.smi.port_num = pRecvCmp->AddressInfo.AddrType.Smi.PortNumber; */ } else { osm_mad_addr.addr_type.gsi.remote_qp = pRecvCmp->AddressInfo.AddrType.Gsi.RemoteQpNumber; osm_mad_addr.addr_type.gsi.remote_qkey = pRecvCmp->AddressInfo.AddrType.Gsi.RemoteQkey; osm_mad_addr.addr_type.gsi.pkey_ix = 0; osm_mad_addr.addr_type.gsi.service_level = pRecvCmp->AddressInfo.AddrType.Gsi.ServiceLevel; osm_mad_addr.addr_type.gsi.global_route = pRecvCmp->AddressInfo.AddrType.Gsi.GlobalRoute; /* osm_mad_addr.addr_type.gsi.grh_info = pRecvCmp->AddressInfo.AddrType.Gsi.GRHInfo; */ } p_osm_madw = osm_mad_pool_get_wrapper(p_mad_bind_info->p_mad_pool, p_mad_bind_info, MAD_BLOCK_SIZE, (ib_mad_t *) & pRecvMad->IBMad, &osm_mad_addr); CL_ASSERT(p_osm_madw); p_vend_wrap = osm_madw_get_vend_ptr(p_osm_madw); CL_ASSERT(p_vend_wrap); p_vend_wrap->p_madt_struct = pRecvMad; p_vend_wrap->direction = RECEIVE; osm_log(p_mad_bind_info->p_umadt_obj->p_log, OSM_LOG_DEBUG, "__mad_recv_processor: " "Received data p_osm_madw[0x%p].\n", p_osm_madw); /* */ /* Do TID Processing. */ /* */ /* If R bit is set swap the TID */ cl_spinlock_acquire(&p_mad_bind_info->trans_ctxt_lock); p_list_item = cl_qlist_find_from_head(&p_mad_bind_info->trans_ctxt_list, __match_tid_context, &p_osm_madw->p_mad->trans_id); if (p_list_item == cl_qlist_end(&p_mad_bind_info->trans_ctxt_list)) { transaction_context = NULL; } else { transaction_context = ((trans_context_t *) p_list_item)->context; cl_qlist_remove_item(&p_mad_bind_info->trans_ctxt_list, p_list_item); free(p_list_item); } cl_spinlock_release(&p_mad_bind_info->trans_ctxt_lock); ((ib_mad_t *) p_osm_madw->p_mad)->trans_id = cl_ntoh64(p_osm_madw->p_mad->trans_id >> 24); osm_log(p_mad_bind_info->p_umadt_obj->p_log, OSM_LOG_DEBUG, "__mad_recv_processor: " "Received data p_osm_madw [0x%p]" "\n\t\t\t\tTID[0x%" PRIx64 ", context[%p]. \n", p_osm_madw, ((ib_mad_t *) p_osm_madw->p_mad)->trans_id, transaction_context); (*(p_mad_bind_info->mad_recv_callback)) (p_osm_madw, p_mad_bind_info-> client_context, transaction_context); } }
static void __osm_al_rcv_callback(IN void *mad_svc_context, IN ib_mad_element_t * p_elem) { osm_al_bind_info_t *const p_bind = (osm_al_bind_info_t *) mad_svc_context; osm_vendor_t *const p_vend = p_bind->p_vend; osm_madw_t *p_old_madw; osm_madw_t *p_new_madw; osm_vend_wrap_t *p_old_vw; osm_vend_wrap_t *p_new_vw; ib_mad_t *p_new_mad; osm_mad_addr_t mad_addr; OSM_LOG_ENTER(p_vend->p_log); CL_ASSERT(p_elem->context1 == NULL); CL_ASSERT(p_elem->context2 == NULL); p_new_mad = ib_get_mad_buf(p_elem); /* In preperation for initializing the new mad wrapper, Initialize the mad_addr structure for the received wire MAD. */ mad_addr.dest_lid = p_elem->remote_lid; mad_addr.path_bits = p_elem->path_bits; /* TO DO - figure out which #define to use for the 2.5 Gb rate... */ mad_addr.static_rate = 0; if (p_new_mad->mgmt_class == IB_MCLASS_SUBN_LID || p_new_mad->mgmt_class == IB_MCLASS_SUBN_DIR) { mad_addr.addr_type.smi.source_lid = p_elem->remote_lid; } else { mad_addr.addr_type.gsi.remote_qp = p_elem->remote_qp; mad_addr.addr_type.gsi.remote_qkey = p_elem->remote_qkey; mad_addr.addr_type.gsi.pkey_ix = p_elem->pkey_index; mad_addr.addr_type.gsi.service_level = p_elem->remote_sl; mad_addr.addr_type.gsi.global_route = FALSE; } /* If this MAD is a response to a previous request, then grab our pre-allocated MAD wrapper. Otherwise, allocate a new MAD wrapper. */ if (ib_mad_is_response(p_new_mad)) { CL_ASSERT(p_elem->send_context1 != NULL); CL_ASSERT(p_elem->send_context2 == NULL); p_old_madw = (osm_madw_t *) p_elem->send_context1; p_old_vw = osm_madw_get_vend_ptr(p_old_madw); p_new_madw = p_old_vw->p_resp_madw; CL_ASSERT(p_new_madw); osm_madw_init(p_new_madw, p_bind, p_elem->size, &mad_addr); osm_madw_set_mad(p_new_madw, p_new_mad); } else { CL_ASSERT(p_elem->send_context1 == NULL); CL_ASSERT(p_elem->send_context2 == NULL); p_new_madw = osm_mad_pool_get_wrapper(p_bind->p_osm_pool, p_bind, p_elem->size, p_new_mad, &mad_addr); } CL_ASSERT(p_new_madw); p_new_vw = osm_madw_get_vend_ptr(p_new_madw); p_new_vw->h_bind = p_bind; p_new_vw->size = p_elem->size; p_new_vw->p_elem = p_elem; p_new_vw->h_av = 0; p_new_vw->p_resp_madw = NULL; osm_log(p_vend->p_log, OSM_LOG_DEBUG, "__osm_al_rcv_callback: " "Calling receive callback function %p.\n", p_bind->rcv_callback); p_bind->rcv_callback(p_new_madw, p_bind->client_context, p_elem->send_context1); OSM_LOG_EXIT(p_vend->p_log); }
static void __osm_al_send_callback(IN void *mad_svc_context, IN ib_mad_element_t * p_elem) { osm_al_bind_info_t *const p_bind = (osm_al_bind_info_t *) mad_svc_context; osm_vendor_t *const p_vend = p_bind->p_vend; osm_madw_t *const p_madw = (osm_madw_t *) p_elem->context1; osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); ib_mad_t *p_mad; OSM_LOG_ENTER(p_vend->p_log); CL_ASSERT(p_vw); CL_ASSERT(p_vw->h_av); /* Destroy the address vector as necessary. */ if (p_vw->h_av != p_bind->h_dr_av) { if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "__osm_al_send_callback: " "Destroying av handle %p.\n", p_vw->h_av); } ib_destroy_av(p_vw->h_av); } p_mad = ib_get_mad_buf(p_elem); if (p_elem->resp_expected) { /* If the send was unsuccessful, notify the user for MADs that were expecting a response. A NULL mad wrapper parameter is the user's clue that the transaction turned sour. Otherwise, do nothing for successful sends when a reponse is expected. The mad will be returned to the pool later. */ p_madw->status = __osm_al_convert_wcs(p_elem->status); if (p_elem->status != IB_WCS_SUCCESS) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "__osm_al_send_callback: " "MAD completed with work queue error: %s.\n", ib_get_wc_status_str(p_elem->status)); /* Return any wrappers to the pool that may have been pre-emptively allocated to handle a receive. */ if (p_vw->p_resp_madw) { osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); p_vw->p_resp_madw = NULL; } p_bind->send_err_callback(p_bind->client_context, p_madw); } } else { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "__osm_al_send_callback: " "Returning MAD to pool, TID = 0x%" PRIx64 ".\n", cl_ntoh64(p_mad->trans_id)); osm_mad_pool_put(p_bind->p_osm_pool, p_madw); goto Exit; } Exit: OSM_LOG_EXIT(p_vend->p_log); }
ib_api_status_t osm_vendor_send(IN osm_bind_handle_t h_bind, IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) { osm_al_bind_info_t *const p_bind = h_bind; osm_vendor_t *const p_vend = p_bind->p_vend; osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); ib_api_status_t status; ib_mad_element_t *p_elem; ib_av_attr_t av; OSM_LOG_ENTER(p_vend->p_log); CL_ASSERT(p_vw->h_bind == h_bind); CL_ASSERT(p_vw->p_elem); p_elem = p_vw->p_elem; /* If a response is expected to this MAD, then preallocate a mad wrapper to contain the wire MAD received in the response. Allocating a wrapper here allows for easier failure paths than after we already received the wire mad. */ if (resp_expected) { p_vw->p_resp_madw = osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); if (p_vw->p_resp_madw == NULL) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_send: ERR 3B27: " "Unable to allocate MAD wrapper.\n"); status = IB_INSUFFICIENT_RESOURCES; goto Exit; } } else p_vw->p_resp_madw = NULL; /* For all sends other than directed route SM MADs, acquire an address vector for the destination. */ if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { memset(&av, 0, sizeof(av)); av.port_num = p_bind->port_num; av.dlid = p_mad_addr->dest_lid; av.static_rate = p_mad_addr->static_rate; av.path_bits = p_mad_addr->path_bits; if ((p_mad->mgmt_class != IB_MCLASS_SUBN_LID) && (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR)) { av.sl = p_mad_addr->addr_type.gsi.service_level; if (p_mad_addr->addr_type.gsi.global_route) { av.grh_valid = TRUE; /* ANIL */ /* av.grh = p_mad_addr->addr_type.gsi.grh_info; */ } } if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "osm_vendor_send: " "av.port_num 0x%X, " "av.dlid 0x%X, " "av.static_rate %d, " "av.path_bits %d.\n", av.port_num, cl_ntoh16(av.dlid), av.static_rate, av.path_bits); } status = ib_create_av(p_vend->h_pd, &av, &p_vw->h_av); if (status != IB_SUCCESS) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_send: ERR 3B28: " "Unable to create address vector (%s).\n", ib_get_err_str(status)); if (p_vw->p_resp_madw) osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); goto Exit; } if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "osm_vendor_send: " "Allocating av handle %p.\n", p_vw->h_av); } } else { p_vw->h_av = p_bind->h_dr_av; } p_elem->h_av = p_vw->h_av; p_elem->context1 = p_madw; p_elem->context2 = NULL; p_elem->immediate_data = 0; p_elem->p_grh = NULL; p_elem->resp_expected = resp_expected; p_elem->retry_cnt = OSM_DEFAULT_RETRY_COUNT; p_elem->send_opt = IB_SEND_OPT_SIGNALED; p_elem->timeout_ms = p_vend->timeout; /* Completion information. */ p_elem->status = 0; /* Not trusting AL */ if ((p_mad->mgmt_class == IB_MCLASS_SUBN_LID) || (p_mad->mgmt_class == IB_MCLASS_SUBN_DIR)) { p_elem->remote_qp = 0; p_elem->remote_qkey = 0; } else { p_elem->remote_qp = p_mad_addr->addr_type.gsi.remote_qp; p_elem->remote_qkey = p_mad_addr->addr_type.gsi.remote_qkey; osm_log(p_vend->p_log, OSM_LOG_DEBUG, "osm_vendor_send: " "remote qp = 0x%X, remote qkey = 0x%X.\n", cl_ntoh32(p_elem->remote_qp), cl_ntoh32(p_elem->remote_qkey)); } status = ib_send_mad(p_bind->h_svc, p_elem, NULL); if (status != IB_SUCCESS) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_send: ERR 3B29: " "Send failed (%s).\n", ib_get_err_str(status)); if (p_vw->p_resp_madw) osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); goto Exit; } Exit: OSM_LOG_EXIT(p_vend->p_log); return (status); }