int wait_for_pending_transactions(osm_stats_t * stats)
{
#ifdef HAVE_LIBPTHREAD
pthread_mutex_lock(&stats->mutex);
while (stats->qp0_mads_outstanding && !osm_exit_flag)
pthread_cond_wait(&stats->cond, &stats->mutex);
pthread_mutex_unlock(&stats->mutex);
#else
while (1) {
unsigned count = stats->qp0_mads_outstanding;
if (!count || osm_exit_flag)
break;
cl_event_wait_on(&stats->event, EVENT_NO_TIMEOUT, TRUE);
}
#endif
return osm_exit_flag;
}
/* Send a MAD out on the GSI interface */
static ib_api_status_t
__osmv_send_sa_req(IN osmv_sa_bind_info_t * p_bind,
IN const osmv_sa_mad_data_t * const p_sa_mad_data,
IN const osmv_query_req_t * const p_query_req)
{
ib_api_status_t status;
ib_mad_t *p_mad_hdr;
ib_sa_mad_t *p_sa_mad;
osm_madw_t *p_madw;
osm_log_t *p_log = p_bind->p_log;
static atomic32_t trans_id;
boolean_t sync;
osmv_query_req_t *p_query_req_copy;
OSM_LOG_ENTER(p_log);
/*
since the sm_lid might change we obtain it every send
(actually it is cached in the bind object and refreshed
every 30sec by this proc)
*/
if (time(NULL) > p_bind->last_lids_update_sec + 30) {
status = update_umad_port(p_bind->p_vendor);
if (status != IB_SUCCESS) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5509: "
"Failed to obtain the SM lid\n");
goto Exit;
}
p_bind->last_lids_update_sec = time(NULL);
}
/* Get a MAD wrapper for the send */
p_madw = osm_mad_pool_get(p_bind->p_mad_pool,
p_bind->h_bind, MAD_BLOCK_SIZE, NULL);
if (p_madw == NULL) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5510: "
"Unable to acquire MAD\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/* Initialize the Sent MAD: */
/* Initialize the MAD buffer for the send operation. */
p_mad_hdr = osm_madw_get_mad_ptr(p_madw);
p_sa_mad = osm_madw_get_sa_mad_ptr(p_madw);
/* Get a new transaction Id */
cl_atomic_inc(&trans_id);
/* Cleanup the MAD from any residue */
memset(p_sa_mad, 0, MAD_BLOCK_SIZE);
/* Initialize the standard MAD header. */
ib_mad_init_new(p_mad_hdr, /* mad pointer */
IB_MCLASS_SUBN_ADM, /* class */
(uint8_t) 2, /* version */
p_sa_mad_data->method, /* method */
cl_hton64((uint64_t) trans_id), /* tid */
p_sa_mad_data->attr_id, /* attr id */
p_sa_mad_data->attr_mod /* attr mod */);
/* Set the query information. */
p_sa_mad->sm_key = p_query_req->sm_key;
p_sa_mad->attr_offset = 0;
p_sa_mad->comp_mask = p_sa_mad_data->comp_mask;
#ifdef DUAL_SIDED_RMPP
if (p_sa_mad->method == IB_MAD_METHOD_GETMULTI)
p_sa_mad->rmpp_flags = IB_RMPP_FLAG_ACTIVE;
#endif
if (p_sa_mad->comp_mask) {
memcpy(p_sa_mad->data, p_sa_mad_data->p_attr,
ib_get_attr_size(p_sa_mad_data->attr_offset));
}
/*
Provide the address to send to
*/
p_madw->mad_addr.dest_lid =
cl_hton16(p_bind->p_vendor->umad_port.sm_lid);
p_madw->mad_addr.addr_type.smi.source_lid =
cl_hton16(p_bind->p_vendor->umad_port.base_lid);
p_madw->mad_addr.addr_type.gsi.remote_qp = CL_HTON32(1);
p_madw->resp_expected = TRUE;
p_madw->fail_msg = CL_DISP_MSGID_NONE;
/*
Provide MAD context such that the call back will know what to do.
We have to keep the entire request structure so we know the CB.
Since we can not rely on the client to keep it around until
the response - we duplicate it and will later dispose it (in CB).
To store on the MADW we cast it into what opensm has:
p_madw->context.ni_context.node_guid
*/
p_query_req_copy = malloc(sizeof(*p_query_req_copy));
*p_query_req_copy = *p_query_req;
p_madw->context.ni_context.node_guid =
(ib_net64_t) (long)p_query_req_copy;
/* we can support async as well as sync calls */
sync = ((p_query_req->flags & OSM_SA_FLAGS_SYNC) == OSM_SA_FLAGS_SYNC);
/* send the mad asynchronously */
status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
p_madw, p_madw->resp_expected);
/* if synchronous - wait on the event */
if (sync) {
OSM_LOG(p_log, OSM_LOG_DEBUG, "Waiting for async event\n");
cl_event_wait_on(&p_bind->sync_event, EVENT_NO_TIMEOUT, FALSE);
cl_event_reset(&p_bind->sync_event);
status = p_madw->status;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
ib_api_status_t
osmv_rmpp_send_madw(IN osm_bind_handle_t h_bind,
IN osm_madw_t * const p_madw,
IN osmv_txn_ctx_t * p_txn, IN boolean_t is_rmpp_ds)
{
ib_api_status_t ret = IB_SUCCESS;
uint32_t i, total_segs;
osmv_rmpp_send_ctx_t *p_send_ctx = osmv_txn_get_rmpp_send_ctx(p_txn);
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
total_segs = osmv_rmpp_send_ctx_get_num_segs(p_send_ctx);
CL_ASSERT(total_segs >= 1);
/* In the double-sided transfer, wait for ACK 0 */
for (;;) {
if (p_send_ctx->window_first > total_segs) {
/* Every segment is acknowledged */
break;
}
/* Send the next burst. */
for (i = p_send_ctx->window_first; i <= p_send_ctx->window_last;
i++) {
/* Send a segment and setup a timeout timer */
ret = __osmv_rmpp_send_segment(h_bind, p_txn, i);
if (IB_SUCCESS != ret) {
goto send_done;
}
}
/* Set the Response Timeout for the ACK on the last DATA segment */
ret = osmv_txn_set_timeout_ev(h_bind, osmv_txn_get_key(p_txn),
p_bo->p_vendor->resp_timeout);
if (IB_SUCCESS != ret) {
goto send_done;
}
/* Going to sleep. Let the others access the transaction DB */
osmv_txn_unlock(p_bo);
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"RMPP Sender thread (madw=%p) going to sleep ...\n",
p_madw);
/* Await the next event to happen */
cl_event_wait_on(&p_send_ctx->event,
EVENT_NO_TIMEOUT, TRUE /* interruptible */ );
/* Got a signal from the MAD dispatcher/timeout handler */
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"RMPP Sender thread (madw=%p) waking up on a signal ...\n",
p_madw);
/* Let's see what changed... Make this atomic - re-acquire the lock. */
osmv_txn_lock(p_bo);
if (TRUE == p_bo->is_closing) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"osmv_rmpp_send_madw: ERR 6601: "
"The bind handle %p is being closed. "
"Stopping the RMPP Send of MADW %p\n",
h_bind, p_madw);
ret = IB_TIMEOUT;
return IB_INTERRUPTED;
}
/* STOP? ABORT? TIMEOUT? */
if (IB_SUCCESS != p_send_ctx->status) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"osmv_rmpp_send_madw: ERR 6602: "
"An error (%s) happened during the RMPP send of %p. Bailing out.\n",
ib_get_err_str(p_send_ctx->status), p_madw);
ret = p_send_ctx->status;
goto send_done;
}
}
if (TRUE == is_rmpp_ds) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"Double-sided RMPP - switching to be the receiver.\n");
ret = osmv_txn_init_rmpp_receiver(h_bind, p_txn, FALSE
/*Send was initiated by me */
);
if (IB_SUCCESS == ret) {
/* Send ACK on the 0 segment */
ret = __osmv_rmpp_send_segment(h_bind, p_txn, 0);
}
}
send_done:
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return ret;
}
/* Send a MAD out on the GSI interface */
ib_api_status_t
__osmv_send_sa_req(IN osmv_sa_bind_info_t * p_bind,
IN const osmv_sa_mad_data_t * const p_sa_mad_data,
IN const osmv_query_req_t * const p_query_req)
{
ib_api_status_t status;
ib_mad_t *p_mad_hdr;
ib_sa_mad_t *p_sa_mad;
osm_madw_t *p_madw;
osm_log_t *p_log = p_bind->p_log;
static atomic32_t trans_id;
boolean_t sync;
osmv_query_req_t *p_query_req_copy;
OSM_LOG_ENTER(p_log);
/*
since the sm_lid might change we obtain it every send
(actually it is cached in the bind object and refreshed
every 30sec by this proc )
*/
status =
__osmv_get_lid_and_sm_lid_by_port_guid(p_bind->p_vendor,
p_bind->port_guid,
&p_bind->
last_lids_update_sec,
&p_bind->lid,
&p_bind->sm_lid);
if (status != IB_SUCCESS) {
osm_log(p_log, OSM_LOG_ERROR,
"__osmv_send_sa_req: ERR 0509: "
"Fail to obtain the sm lid.\n");
goto Exit;
}
/* Get a MAD wrapper for the send */
p_madw = osm_mad_pool_get(p_bind->p_mad_pool,
p_bind->h_bind, MAD_BLOCK_SIZE, NULL);
if (p_madw == NULL) {
osm_log(p_log, OSM_LOG_ERROR,
"__osmv_send_sa_req: ERR 0510: "
"Unable to acquire MAD.\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/* Initialize the Sent MAD: */
/* Initialize the MAD buffer for the send operation. */
p_mad_hdr = osm_madw_get_mad_ptr(p_madw);
p_sa_mad = osm_madw_get_sa_mad_ptr(p_madw);
/* Get a new transaction Id */
cl_atomic_inc(&trans_id);
/* Cleanup the MAD from any residue */
memset(p_sa_mad, 0, MAD_BLOCK_SIZE);
/* Initialize the standard MAD header. */
ib_mad_init_new(p_mad_hdr, /* mad pointer */
IB_MCLASS_SUBN_ADM, /* class */
(uint8_t) 2, /* version */
p_sa_mad_data->method, /* method */
cl_hton64((uint64_t) trans_id), /* tid */
p_sa_mad_data->attr_id, /* attr id */
p_sa_mad_data->attr_mod /* attr mod */
);
/* Set the query information. */
p_sa_mad->sm_key = p_query_req->sm_key;
p_sa_mad->attr_offset = 0;
p_sa_mad->comp_mask = p_sa_mad_data->comp_mask;
if (p_sa_mad->comp_mask) {
memcpy(p_sa_mad->data, p_sa_mad_data->p_attr,
ib_get_attr_size(p_sa_mad_data->attr_offset));
}
/*
Provide the address to send to
*/
/* Patch to handle IBAL - host order , where it should take destination lid in network order */
#ifdef OSM_VENDOR_INTF_AL
p_madw->mad_addr.dest_lid = p_bind->sm_lid;
#else
p_madw->mad_addr.dest_lid = cl_hton16(p_bind->sm_lid);
#endif
p_madw->mad_addr.addr_type.smi.source_lid = cl_hton16(p_bind->lid);
p_madw->mad_addr.addr_type.gsi.remote_qp = CL_HTON32(1);
p_madw->mad_addr.addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
p_madw->mad_addr.addr_type.gsi.pkey_ix = 0;
p_madw->resp_expected = TRUE;
p_madw->fail_msg = CL_DISP_MSGID_NONE;
/*
Provide MAD context such that the call back will know what to do.
We have to keep the entire request structure so we know the CB.
Since we can not rely on the client to keep it arroud until
the response - we duplicate it and will later dispose it (in CB).
To store on the MADW we cast it into what opensm has:
p_madw->context.arb_context.context1
*/
p_query_req_copy = malloc(sizeof(*p_query_req_copy));
*p_query_req_copy = *p_query_req;
p_madw->context.arb_context.context1 = p_query_req_copy;
/* we can support async as well as sync calls */
sync = ((p_query_req->flags & OSM_SA_FLAGS_SYNC) == OSM_SA_FLAGS_SYNC);
/* send the mad asynchronously */
status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
p_madw, p_madw->resp_expected);
/* if synchronous - wait on the event */
if (sync) {
osm_log(p_log, OSM_LOG_DEBUG,
"__osmv_send_sa_req: " "Waiting for async event.\n");
cl_event_wait_on(&p_bind->sync_event, EVENT_NO_TIMEOUT, FALSE);
cl_event_reset(&p_bind->sync_event);
status = p_madw->status;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
static void vl15_poller(IN void *p_ptr)
{
ib_api_status_t status;
osm_madw_t *p_madw;
osm_vl15_t *p_vl = p_ptr;
cl_qlist_t *p_fifo;
int32_t max_smps = p_vl->max_wire_smps;
int32_t max_smps2 = p_vl->max_wire_smps2;
OSM_LOG_ENTER(p_vl->p_log);
if (p_vl->thread_state == OSM_THREAD_STATE_NONE)
p_vl->thread_state = OSM_THREAD_STATE_RUN;
while (p_vl->thread_state == OSM_THREAD_STATE_RUN) {
/*
Start servicing the FIFOs by pulling off MAD wrappers
and passing them to the transport interface.
There are lots of corner cases here so tread carefully.
The unicast FIFO has priority, since somebody is waiting
for a timely response.
*/
cl_spinlock_acquire(&p_vl->lock);
if (cl_qlist_count(&p_vl->ufifo) != 0)
p_fifo = &p_vl->ufifo;
else
p_fifo = &p_vl->rfifo;
p_madw = (osm_madw_t *) cl_qlist_remove_head(p_fifo);
cl_spinlock_release(&p_vl->lock);
if (p_madw != (osm_madw_t *) cl_qlist_end(p_fifo)) {
OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
"Servicing p_madw = %p\n", p_madw);
if (osm_log_is_active(p_vl->p_log, OSM_LOG_FRAMES))
osm_dump_dr_smp(p_vl->p_log,
osm_madw_get_smp_ptr(p_madw),
OSM_LOG_FRAMES);
vl15_send_mad(p_vl, p_madw);
} else
/*
The VL15 FIFO is empty, so we have nothing left to do.
*/
status = cl_event_wait_on(&p_vl->signal,
EVENT_NO_TIMEOUT, TRUE);
while (p_vl->p_stats->qp0_mads_outstanding_on_wire >= max_smps &&
p_vl->thread_state == OSM_THREAD_STATE_RUN) {
status = cl_event_wait_on(&p_vl->signal,
p_vl->max_smps_timeout,
TRUE);
if (status == CL_TIMEOUT) {
if (max_smps < max_smps2)
max_smps++;
break;
} else if (status != CL_SUCCESS) {
OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
"Event wait failed (%s)\n",
CL_STATUS_MSG(status));
break;
}
max_smps = p_vl->max_wire_smps;
}
}
/*
since we abort immediately when the state != OSM_THREAD_STATE_RUN
we might have some mads on the queues. After the thread exits
the vl15 destroy routine should put these mads back...
*/
OSM_LOG_EXIT(p_vl->p_log);
}