static uint64_t ibping(ib_portid_t * portid, int quiet)
{
char data[IB_VENDOR_RANGE2_DATA_SIZE] = { 0 };
ib_vendor_call_t call;
uint64_t start, rtt;
DEBUG("Ping..");
start = cl_get_time_stamp();
call.method = IB_MAD_METHOD_GET;
call.mgmt_class = IB_VENDOR_OPENIB_PING_CLASS;
call.attrid = 0;
call.mod = 0;
call.oui = IB_OPENIB_OUI;
call.timeout = 0;
memset(&call.rmpp, 0, sizeof call.rmpp);
if (!ib_vendor_call_via(data, portid, &call, srcport))
return ~0ull;
rtt = cl_get_time_stamp() - start;
if (!last_host[0])
memcpy(last_host, data, sizeof last_host);
if (!quiet)
printf("Pong from %s (%s): time %" PRIu64 ".%03" PRIu64 " ms\n",
data, portid2str(portid), rtt / 1000, rtt % 1000);
return rtt;
}
int main()
{
cl_event_wheel_t event_wheel;
/* uint64_t key; */
/* init complib */
complib_init();
/* construct */
cl_event_wheel_construct(&event_wheel);
/* init */
cl_event_wheel_init(&event_wheel);
/* Start Playing */
cl_event_wheel_reg(&event_wheel, 1, /* key */
cl_get_time_stamp() + 3000000, /* 3 sec lifetime */
__test_event_aging, /* cb */
"The first Aging Event");
cl_event_wheel_reg(&event_wheel, 2, /* key */
cl_get_time_stamp() + 3000000, /* 3 sec lifetime */
__test_event_aging, /* cb */
"The Second Aging Event");
cl_event_wheel_reg(&event_wheel, 3, /* key */
cl_get_time_stamp() + 3500000, /* 3 sec lifetime */
__test_event_aging, /* cb */
"The Third Aging Event");
__cl_event_wheel_dump(&event_wheel);
sleep(2);
cl_event_wheel_reg(&event_wheel, 2, /* key */
cl_get_time_stamp() + 8000000, /* 3 sec lifetime */
__test_event_aging, /* cb */
"The Second Aging Event Moved");
__cl_event_wheel_dump(&event_wheel);
sleep(1);
/* remove the third event */
cl_event_wheel_unreg(&event_wheel, 3); /* key */
/* get the number of registrations for the keys */
printf("Event 1 Registered: %u\n",
cl_event_wheel_num_regs(&event_wheel, 1));
printf("Event 2 Registered: %u\n",
cl_event_wheel_num_regs(&event_wheel, 2));
sleep(5);
/* destroy */
cl_event_wheel_destroy(&event_wheel);
complib_exit();
return (0);
}
/*
* NAME
* osmv_txn_set_timeout_ev
*
* DESCRIPTION
*
* SEE ALSO
*
*/
ib_api_status_t
osmv_txn_set_timeout_ev(IN osm_bind_handle_t h_bind,
IN uint64_t key, IN uint64_t msec)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
cl_event_wheel_t *p_event_wheel = p_bo->txn_mgr.p_event_wheel;
cl_status_t status;
status = cl_event_wheel_reg(p_event_wheel, key, cl_get_time_stamp() + 1000 * msec, /* TTL */
__osmv_txn_timeout_cb,
p_bo /* The context */ );
return (ib_api_status_t) status;
}
/********************************************************************
__cl_disp_worker
Description:
This function takes messages off the FIFO and calls Processmsg()
This function executes as passive level.
Inputs:
p_disp - Pointer to Dispatcher object
Outputs:
None
Returns:
None
********************************************************************/
void __cl_disp_worker(IN void *context)
{
cl_disp_msg_t *p_msg;
cl_dispatcher_t *p_disp = (cl_dispatcher_t *) context;
cl_spinlock_acquire(&p_disp->lock);
/* Process the FIFO until we drain it dry. */
while (cl_qlist_count(&p_disp->msg_fifo)) {
/* Pop the message at the head from the FIFO. */
p_msg =
(cl_disp_msg_t *) cl_qlist_remove_head(&p_disp->msg_fifo);
/* we track the tim ethe last message spent in the queue */
p_disp->last_msg_queue_time_us =
cl_get_time_stamp() - p_msg->in_time;
/*
* Release the spinlock while the message is processed.
* The user's callback may reenter the dispatcher
* and cause the lock to be reaquired.
*/
cl_spinlock_release(&p_disp->lock);
p_msg->p_dest_reg->pfn_rcv_callback((void *)p_msg->p_dest_reg->
context,
(void *)p_msg->p_data);
cl_atomic_dec(&p_msg->p_dest_reg->ref_cnt);
/* The client has seen the data. Notify the sender as appropriate. */
if (p_msg->pfn_xmt_callback) {
p_msg->pfn_xmt_callback((void *)p_msg->context,
(void *)p_msg->p_data);
cl_atomic_dec(&p_msg->p_src_reg->ref_cnt);
}
/* Grab the lock for the next iteration through the list. */
cl_spinlock_acquire(&p_disp->lock);
/* Return this message to the pool. */
cl_qpool_put(&p_disp->msg_pool, (cl_pool_item_t *) p_msg);
}
cl_spinlock_release(&p_disp->lock);
}
void report(int sig)
{
total_time = cl_get_time_stamp() - start;
DEBUG("out due signal %d", sig);
printf("\n--- %s (%s) ibping statistics ---\n", last_host,
portid2str(&portid));
printf("%" PRIu64 " packets transmitted, %" PRIu64 " received, %" PRIu64
"%% packet loss, time %" PRIu64 " ms\n", ntrans, replied,
(lost != 0) ? lost * 100 / ntrans : 0, total_time / 1000);
printf("rtt min/avg/max = %" PRIu64 ".%03" PRIu64 "/%" PRIu64 ".%03"
PRIu64 "/%" PRIu64 ".%03" PRIu64 " ms\n",
minrtt == ~0ull ? 0 : minrtt / 1000,
minrtt == ~0ull ? 0 : minrtt % 1000,
replied ? total_rtt / replied / 1000 : 0,
replied ? (total_rtt / replied) % 1000 : 0, maxrtt / 1000,
maxrtt % 1000);
exit(0);
}
/* //////////////////////////////////////////////////////////////////////// */
ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,
IN osm_vend_wrap_t * const p_vend_wrap,
IN osm_mad_addr_t * const p_mad_addr,
IN ib_mad_t * const p_mad,
IN void *transaction_context, IN boolean_t const resp_expected)
{
FSTATUS Status;
MadAddrStruct destAddr = { 0 };
mad_bind_info_t *p_mad_bind_info;
trans_context_t *p_trans_context;
umadt_obj_t *p_umadt_obj = NULL;
uint32_t mad_count = 0;
MadtStruct *p_madt_struct = NULL;
uint32_t i;
uint32_t num_mads = 0;
uint32_t seg_num = 0;
uint8_t *p_frag_data = NULL;
ib_sa_mad_t_vM3 *p_sa_mad = NULL;
CL_ASSERT(h_bind);
p_mad_bind_info = (mad_bind_info_t *) h_bind;
p_umadt_obj = p_mad_bind_info->p_umadt_obj;
/* sanity check */
CL_ASSERT(p_umadt_obj);
CL_ASSERT(p_umadt_obj->init_done);
CL_ASSERT(__valid_mad_handle(p_mad_bind_info));
CL_ASSERT(p_vend_wrap);
CL_ASSERT(p_mad_addr);
CL_ASSERT(p_mad);
/* CL_ASSERT( (ib_mad_t*)&p_vend_wrap->p_madt_struct->IBMad == p_mad ); */
/* */
/* based on the class, fill out the address info */
/* */
destAddr.DestLid = p_mad_addr->dest_lid;
destAddr.PathBits = p_mad_addr->path_bits;
destAddr.StaticRate = p_mad_addr->static_rate;
if (p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_LID ||
p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_DIR) {
CL_ASSERT(p_mad_addr->addr_type.smi.source_lid);
destAddr.AddrType.Smi.SourceLid =
p_mad_addr->addr_type.smi.source_lid;
} else {
destAddr.AddrType.Gsi.RemoteQpNumber =
p_mad_addr->addr_type.gsi.remote_qp;
destAddr.AddrType.Gsi.RemoteQkey =
p_mad_addr->addr_type.gsi.remote_qkey;
destAddr.AddrType.Gsi.PKey = OSM_DEFAULT_PKEY;
destAddr.AddrType.Gsi.ServiceLevel =
p_mad_addr->addr_type.gsi.service_level;
destAddr.AddrType.Gsi.GlobalRoute =
p_mad_addr->addr_type.gsi.global_route;
/* destAddr.AddrType.Gsi.GRHInfo = p_mad_addr->addr_type.gsi.grh_info; */
}
p_mad->trans_id = cl_ntoh64(p_mad->trans_id) << 24;
/* */
/* Create a transaction context for this send and save the TID and client context. */
/* */
if (resp_expected) {
p_trans_context = malloc(sizeof(trans_context_t));
CL_ASSERT(p_trans_context);
memset(p_trans_context, 0, sizeof(trans_context_t));
p_trans_context->trans_id = p_mad->trans_id;
p_trans_context->context = transaction_context;
p_trans_context->sent_time = cl_get_time_stamp();
cl_spinlock_acquire(&p_mad_bind_info->trans_ctxt_lock);
cl_qlist_insert_tail(&p_mad_bind_info->trans_ctxt_list,
&p_trans_context->list_item);
cl_spinlock_release(&p_mad_bind_info->trans_ctxt_lock);
}
if (p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_LID ||
p_mad_bind_info->umadt_reg_class.ClassId == IB_MCLASS_SUBN_DIR) {
/* Get one mad from uMadt */
mad_count = 1;
Status =
p_umadt_obj->uMadtInterface.
uMadtGetSendMad(p_mad_bind_info->umadt_handle, &mad_count,
&p_madt_struct);
if (Status != FSUCCESS || p_madt_struct == NULL) {
return IB_ERROR;
}
/* No Segmentation required */
memcpy(&p_madt_struct->IBMad, p_mad, MAD_BLOCK_SIZE);
/* Post the MAD */
Status =
p_umadt_obj->uMadtInterface.uMadtPostSend(p_mad_bind_info->
umadt_handle,
p_madt_struct,
&destAddr);
if (Status != FSUCCESS) {
printf("uMadtPostSendMad: Status = <%d>\n", Status);
return IB_ERROR;
}
/* Release send MAD */
Status =
p_umadt_obj->uMadtInterface.
uMadtReleaseSendMad(p_mad_bind_info->umadt_handle,
p_madt_struct);
if (Status != FSUCCESS) {
printf("uMadtReleaseSendMad: Status = <%d>\n", Status);
return IB_ERROR;
}
} else {
/* */
/* Segment the MAD, get the required send mads from uMadt and post the MADs. */
/* */
uint32_t payload_len;
payload_len =
cl_ntoh32(((ib_sa_mad_t_vM3 *) p_mad)->payload_len);
num_mads = payload_len / IB_SA_DATA_SIZE;
if (payload_len % IB_SA_DATA_SIZE != 0) {
num_mads++; /* Get one additional mad for the remainder */
}
for (i = 0; i < num_mads; i++) {
/* Get one mad from uMadt */
mad_count = 1;
Status =
p_umadt_obj->uMadtInterface.
uMadtGetSendMad(p_mad_bind_info->umadt_handle,
&mad_count, &p_madt_struct);
if (Status != FSUCCESS || p_madt_struct == NULL) {
return IB_ERROR;
}
/* Copy client MAD into uMadt's MAD. */
if (i == 0) { /* First Packet */
/* Since this is the first MAD, copy the entire MAD_SIZE */
memcpy(&p_madt_struct->IBMad, p_mad,
MAD_BLOCK_SIZE);
p_frag_data =
(uint8_t *) p_mad + MAD_BLOCK_SIZE;
p_sa_mad =
(ib_sa_mad_t_vM3 *) & p_madt_struct->IBMad;
if (num_mads == 1) { /* Only one Packet */
p_sa_mad->seg_num = 0;
p_sa_mad->frag_flag = 5; /* Set bit 0 for first pkt and b4 for last pkt */
/* the payload length gets copied with the mad header above */
} else { /* More than one packet in this response */
seg_num = 1;
p_sa_mad->seg_num =
cl_ntoh32(seg_num++);
p_sa_mad->frag_flag = 1; /* Set bit 0 for first pkt */
/* the payload length gets copied with the mad header above */
}
} else if (i < num_mads - 1) { /* Not last packet */
/* First copy only the header */
memcpy(&p_madt_struct->IBMad, p_mad,
IB_SA_MAD_HDR_SIZE);
/* Set the relevant fields in the SA_MAD_HEADER */
p_sa_mad =
(ib_sa_mad_t_vM3 *) & p_madt_struct->IBMad;
p_sa_mad->payload_len =
cl_ntoh32(IB_SA_DATA_SIZE);
p_sa_mad->seg_num = cl_ntoh32(seg_num++);
p_sa_mad->frag_flag = 0;
/* Now copy the fragmented data */
memcpy(((uint8_t *) & p_madt_struct->IBMad) +
IB_SA_MAD_HDR_SIZE, p_frag_data,
IB_SA_DATA_SIZE);
p_frag_data = p_frag_data + IB_SA_DATA_SIZE;
} else if (i == num_mads - 1) { /* Last packet */
/* First copy only the header */
memcpy(&p_madt_struct->IBMad, p_mad,
IB_SA_MAD_HDR_SIZE);
/* Set the relevant fields in the SA_MAD_HEADER */
p_sa_mad =
(ib_sa_mad_t_vM3 *) & p_madt_struct->IBMad;
p_sa_mad->seg_num = cl_ntoh32(seg_num++);
p_sa_mad->frag_flag = 4; /* Set Bit 2 for last pkt */
p_sa_mad->payload_len =
cl_ntoh32(cl_ntoh32
(((ib_sa_mad_t_vM3 *) p_mad)->
payload_len) % IB_SA_DATA_SIZE);
/* Now copy the fragmented data */
memcpy((((uint8_t *) & p_madt_struct->IBMad)) +
IB_SA_MAD_HDR_SIZE, p_frag_data,
cl_ntoh32(p_sa_mad->payload_len));
p_frag_data = p_frag_data + IB_SA_DATA_SIZE;
}
/* Post the MAD */
Status =
p_umadt_obj->uMadtInterface.
uMadtPostSend(p_mad_bind_info->umadt_handle,
p_madt_struct, &destAddr);
if (Status != FSUCCESS) {
printf("uMadtPostSendMad: Status = <%d>\n",
Status);
return IB_ERROR;
}
/* Release send MAD */
Status =
p_umadt_obj->uMadtInterface.
uMadtReleaseSendMad(p_mad_bind_info->umadt_handle,
p_madt_struct);
if (Status != FSUCCESS) {
printf("uMadtReleaseSendMad: Status = <%d>\n",
Status);
return IB_ERROR;
}
}
}
return (IB_SUCCESS);
}
void __osm_vendor_timer_callback(IN void *context)
{
uint64_t current_time;
mad_bind_info_t *p_mad_bind_info;
umadt_obj_t *p_umadt_obj;
uint32_t timeout;
cl_list_item_t *p_list_item, *p_next_list_item;
CL_ASSERT(context);
p_mad_bind_info = (mad_bind_info_t *) context;
p_umadt_obj = p_mad_bind_info->p_umadt_obj;
timeout = p_umadt_obj->timeout * 1000;
current_time = cl_get_time_stamp();
cl_spinlock_acquire(&p_mad_bind_info->trans_ctxt_lock);
p_list_item = cl_qlist_head(&p_mad_bind_info->trans_ctxt_list);
while (p_list_item != cl_qlist_end(&p_mad_bind_info->trans_ctxt_list)) {
p_next_list_item = cl_qlist_next(p_list_item);
/* DEFAULT_PKT_TIMEOUT is in milli seconds */
if (current_time - ((trans_context_t *) p_list_item)->sent_time
> timeout) {
/* Add this transaction to the timeout_list */
cl_qlist_remove_item(&p_mad_bind_info->trans_ctxt_list,
p_list_item);
cl_qlist_insert_tail(&p_mad_bind_info->timeout_list,
p_list_item);
}
p_list_item = p_next_list_item;
}
cl_spinlock_release(&p_mad_bind_info->trans_ctxt_lock);
p_list_item = cl_qlist_head(&p_mad_bind_info->timeout_list);
while (p_list_item != cl_qlist_end(&p_mad_bind_info->timeout_list)) {
osm_log(p_mad_bind_info->p_umadt_obj->p_log, OSM_LOG_DEBUG,
"__osm_vendor_timer_callback: "
"Timing out transaction context [0x%p].\n",
((trans_context_t *) p_list_item)->context);
(*(p_mad_bind_info->mad_recv_callback)) (NULL,
p_mad_bind_info->
client_context,
((trans_context_t *)
p_list_item)->
context);
p_next_list_item = cl_qlist_next(p_list_item);
cl_qlist_remove_item(&p_mad_bind_info->timeout_list,
p_list_item);
free(p_list_item);
p_list_item = p_next_list_item;
}
cl_timer_start(&p_mad_bind_info->timeout_timer,
DEFAULT_TIMER_INTERVAL_MSEC);
}
static uint64_t
__osmv_txn_timeout_cb(IN uint64_t key,
IN uint32_t num_regs, IN void *cb_context)
{
osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) cb_context;
uint64_t ret = 0;
osmv_txn_ctx_t *p_txn;
osmv_rmpp_send_ctx_t *p_send_ctx;
osm_madw_t *p_madw = NULL;
ib_mad_t *p_mad;
osm_mad_addr_t *p_mad_addr;
boolean_t invoke_err_cb = FALSE;
OSM_LOG_ENTER(p_bo->p_vendor->p_log);
/* Don't try to acquire a lock on the Bind Object -
* it's taken by the mechanism that drives the timeout based events!
* (Recall the special constructor that the Event Wheel is applied with)
*/
if (p_bo->is_closing) {
goto txn_done;
}
ret = osmv_txn_lookup(p_bo, key, &p_txn);
if (IB_NOT_FOUND == ret) {
/* Prevent a race - the transaction is already destroyed */
goto txn_done;
}
p_madw = p_txn->p_madw;
switch (osmv_txn_get_rmpp_state(p_txn)) {
case OSMV_TXN_RMPP_NONE:
if (num_regs <= OSMV_MAX_RETRANSMIT) {
/* We still did not exceed the limit of retransmissions.
* Set the next timeout's value.
*/
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"__osmv_txn_timeout_cb: "
"The transaction request (tid=0x%llX) timed out %d times. "
"Retrying the send.\n",
osmv_txn_get_tid(p_txn), num_regs);
/* resend this mad */
ret = osmv_simple_send_madw((osm_bind_handle_t *) p_bo,
p_madw, p_txn, TRUE);
if (ret != IB_SUCCESS) {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"__osmv_txn_timeout_cb: "
"Fail to send retry for transaction request (tid=0x%llX).\n",
osmv_txn_get_tid(p_txn));
osmv_txn_done((osm_bind_handle_t) p_bo, key,
TRUE /*in timeout callback */ );
/* This is a requester. Always apply the callback */
invoke_err_cb = TRUE;
} else {
uint64_t next_timeout_ms;
next_timeout_ms =
p_bo->p_vendor->resp_timeout * (num_regs +
1) *
(num_regs + 1);
/* when do we need to timeout again */
ret =
cl_get_time_stamp() +
(uint64_t) (1000 * next_timeout_ms);
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"__osmv_txn_timeout_cb: "
"Retry request timout in : %lu [msec].\n",
next_timeout_ms);
}
} else {
osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
"__osmv_txn_timeout_cb: ERR 6702: "
"The transaction request (tid=0x%llX) timed out (after %d retries). "
"Invoking the error callback.\n",
osmv_txn_get_tid(p_txn), num_regs);
osmv_txn_done((osm_bind_handle_t) p_bo, key,
TRUE /*in timeout callback */ );
/* This is a requester. Always apply the callback */
invoke_err_cb = TRUE;
}
break;
case OSMV_TXN_RMPP_SENDER:
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"RMPP sender (tid=0x%llX) did not receive ACK "
"on every segment in the current send window.\n",
osmv_txn_get_tid(p_txn));
p_send_ctx = osmv_txn_get_rmpp_send_ctx(p_txn);
if (num_regs <= OSMV_MAX_RETRANSMIT) {
/* We still did not exceed the limit of retransmissions.
* Set the next timeout's value.
*/
ret =
cl_get_time_stamp() +
1000 * p_bo->p_vendor->resp_timeout;
} else {
p_send_ctx->status = IB_TIMEOUT;
p_mad = osm_madw_get_mad_ptr(p_madw);
p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
/* Send an ABORT to the other side */
osmv_rmpp_send_nak((osm_bind_handle_t) p_bo, p_mad,
p_mad_addr, IB_RMPP_TYPE_ABORT,
IB_RMPP_STATUS_T2L);
}
/* Wake the RMPP sender thread up */
cl_event_signal(&p_send_ctx->event);
break;
case OSMV_TXN_RMPP_RECEIVER:
osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
"Transaction timeout on an RMPP receiver (tid=0x%llX). "
"Dropping the transaction.\n", osmv_txn_get_tid(p_txn));
osmv_txn_done((osm_bind_handle_t) p_bo, key,
TRUE /*in timeout callback */ );
if (FALSE == osmv_txn_is_rmpp_init_by_peer(p_txn)) {
/* This is a requester, still waiting for the reply. Apply the callback */
invoke_err_cb = TRUE;
}
break;
default:
CL_ASSERT(FALSE);
}
if (TRUE == invoke_err_cb) {
CL_ASSERT(NULL != p_madw);
/* update the status in the p_madw */
p_madw->status = IB_TIMEOUT;
p_bo->send_err_cb(p_bo->cb_context, p_madw);
/* no re-registration */
ret = 0;
}
txn_done:
OSM_LOG_EXIT(p_bo->p_vendor->p_log);
return ret;
}
int main(int argc, char **argv)
{
int mgmt_classes[1] = { IB_SA_CLASS };
int ping_class = IB_VENDOR_OPENIB_PING_CLASS;
uint64_t rtt;
char *err;
const struct ibdiag_opt opts[] = {
{"count", 'c', 1, "<num>", "stop after count packets"},
{"flood", 'f', 0, NULL, "flood destination"},
{"oui", 'o', 1, NULL, "use specified OUI number"},
{"Server", 'S', 0, NULL, "start in server mode"},
{0}
};
char usage_args[] = "<dest lid|guid>";
ibdiag_process_opts(argc, argv, NULL, "DKy", opts, process_opt,
usage_args, NULL);
argc -= optind;
argv += optind;
if (!argc && !server)
ibdiag_show_usage();
srcport = mad_rpc_open_port(ibd_ca, ibd_ca_port, mgmt_classes, 1);
if (!srcport)
IBEXIT("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port);
if (server) {
if (mad_register_server_via(ping_class, 0, 0, oui, srcport) < 0)
IBEXIT("can't serve class %d on this port",
ping_class);
get_host_and_domain(host_and_domain, sizeof host_and_domain);
if ((err = ibping_serv()))
IBEXIT("ibping to %s: %s", portid2str(&portid), err);
exit(0);
}
if (mad_register_client_via(ping_class, 0, srcport) < 0)
IBEXIT("can't register ping class %d on this port",
ping_class);
if (resolve_portid_str(ibd_ca, ibd_ca_port, &portid, argv[0],
ibd_dest_type, ibd_sm_id, srcport) < 0)
IBEXIT("can't resolve destination port %s", argv[0]);
signal(SIGINT, report);
signal(SIGTERM, report);
start = cl_get_time_stamp();
while (count-- > 0) {
ntrans++;
if ((rtt = ibping(&portid, flood)) == ~0ull) {
DEBUG("ibping to %s failed", portid2str(&portid));
lost++;
} else {
if (rtt < minrtt)
minrtt = rtt;
if (rtt > maxrtt)
maxrtt = rtt;
total_rtt += rtt;
replied++;
}
if (!flood)
sleep(1);
}
report(0);
mad_rpc_close_port(srcport);
exit(-1);
}
ib_api_status_t
osm_transaction_mgr_insert_madw(IN osm_bind_handle_t * const p_bind,
IN osm_madw_t * p_madw)
{
#ifdef OSM_VENDOR_INTF_MTL
osm_vendor_t *const p_vend = ((osm_mtl_bind_info_t *) p_bind)->p_vend;
#else
osm_vendor_t *const p_vend = ((osm_ts_bind_info_t *) p_bind)->p_vend;
#endif
osm_transaction_mgr_t *trans_mgr_p;
osm_madw_req_t *osm_madw_req_p;
uint64_t timeout;
uint64_t waking_time;
cl_status_t cl_status;
uint64_t key;
const ib_mad_t *mad_p = p_madw->p_mad;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(mad_p);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
timeout = (uint64_t) (p_vend->timeout) * 1000; /* change the miliseconds value of timeout to microseconds. */
waking_time = timeout + cl_get_time_stamp();
osm_madw_req_p = (osm_madw_req_t *) malloc(sizeof(osm_madw_req_t));
osm_madw_req_p->p_madw = p_madw;
osm_madw_req_p->waking_time = waking_time;
osm_madw_req_p->retry_cnt = OSM_DEFAULT_RETRY_COUNT;
osm_madw_req_p->p_bind = p_bind;
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_insert_madw: "
"Inserting MADW:%p with waking_time: <0x%" PRIx64 "> TID:<0x%"
PRIx64 ">.\n", p_madw, waking_time, p_madw->p_mad->trans_id);
/* Get the lock on the manager */
cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
/* If the list is empty - need to start the timer with timer of timeout (in miliseconds) */
if (cl_is_qlist_empty(trans_mgr_p->madw_reqs_list_p)) {
/* stop the timer if it is running */
cl_timer_stop(&trans_mgr_p->madw_list_timer);
/* start the timer to the timeout (in miliseconds) */
cl_status = cl_timer_start(&trans_mgr_p->madw_list_timer,
p_vend->timeout);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_transaction_mgr_insert_madw : ERROR 1000: "
"Failed to start timer\n");
}
}
/* insert the object to the qlist and the qmap */
cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* get the key */
key = (uint64_t) mad_p->trans_id;
cl_qmap_insert(trans_mgr_p->madw_by_tid_map_p, key,
&(osm_madw_req_p->map_item));
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
OSM_LOG_EXIT(p_vend->p_log);
return (IB_SUCCESS);
}
/* this is the callback function of the timer */
void __osm_transaction_mgr_callback(IN void *context)
{
osm_transaction_mgr_t *trans_mgr_p;
osm_vendor_t *p_vend = (osm_vendor_t *) context;
cl_list_item_t *p_list_item;
cl_list_item_t *p_list_next_item;
osm_madw_req_t *osm_madw_req_p;
uint64_t current_time; /* [usec] */
uint32_t new_timeout; /* [msec] */
cl_status_t cl_status;
ib_mad_t *p_mad;
#ifdef OSM_VENDOR_INTF_MTL
osm_mtl_bind_info_t *p_bind;
#else
osm_ts_bind_info_t *p_bind;
#endif
cl_list_t tmp_madw_p_list; /* this list will include all the madw_p that should be removed. */
cl_list_t retry_madw_p_list; /* this list will include all the madw_p that were retried and need to be removed. */
osm_madw_t *madw_p;
OSM_LOG_ENTER(p_vend->p_log);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
/* initialize the tmp_madw_p_list */
cl_list_construct(&tmp_madw_p_list);
cl_status = cl_list_init(&tmp_madw_p_list, 50);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to create tmp_madw_p_list\n");
}
cl_list_construct(&retry_madw_p_list);
cl_status = cl_list_init(&retry_madw_p_list, 50);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to create retry_madw_p_list\n");
}
current_time = cl_get_time_stamp();
cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
p_list_item = cl_qlist_head(trans_mgr_p->madw_reqs_list_p);
if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* the list is empty - nothing to do */
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Nothing to do\n");
goto Exit;
}
/* non empty list: */
/* get the osm_madw_req_p */
osm_madw_req_p = PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
while (osm_madw_req_p->waking_time <= current_time) {
/* this object was supposed to have gotten a response */
/* we need to decide if we need to retry or done with it. */
if (osm_madw_req_p->retry_cnt > 0) {
/* add to the list of the retrys : */
cl_list_insert_tail(&retry_madw_p_list, osm_madw_req_p);
/* update wakeup time and retry count */
osm_madw_req_p->waking_time =
p_vend->timeout * 1000 + cl_get_time_stamp();
osm_madw_req_p->retry_cnt--;
/* make sure we will get some timer call if not earlier */
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Timer restart:%u\n",
p_vend->timeout);
cl_status =
cl_timer_start(&trans_mgr_p->madw_list_timer,
p_vend->timeout);
/* go to the next object and check if it also needs to be removed - didn't receive response */
/* we need to do it before we move current item to the end of the list */
p_list_next_item = cl_qlist_next(p_list_item);
/* remove from the head */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* insert the object to the qlist and the qmap */
cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
} else {
/* go to the next object and check if it also needs to be removed - didn't receive response */
p_list_next_item = cl_qlist_next(p_list_item);
/* remove from the head */
cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* add it to the tmp_madw_p_list to be removed */
cl_list_insert_tail(&tmp_madw_p_list,
osm_madw_req_p->p_madw);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Found failed transaction madw: %p\n",
osm_madw_req_p->p_madw);
}
/* Advance */
p_list_item = p_list_next_item;
if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* the list is empty - nothing to do */
break;
}
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
}
/* look at the current p_list_item. If it is not the end item - then we need to */
/* re-start the timer */
if (p_list_item != cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
/* get the osm_madw_req_p */
osm_madw_req_p =
PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
/* we have the object that still didn't get response - re-start the timer */
/* start the timer to the timeout (in miliseconds) */
new_timeout =
(osm_madw_req_p->waking_time - cl_get_time_stamp()) / 1000 +
1;
cl_status =
cl_timer_start(&trans_mgr_p->madw_list_timer, new_timeout);
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : Timer restart:%u\n",
new_timeout);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_transaction_mgr_callback : ERROR 1000: "
"Failed to start timer\n");
}
}
/* if not empty - retry on retry list: */
if (!cl_is_list_empty(&retry_madw_p_list)) {
/* remove all elements that were retried: */
osm_madw_req_p =
(osm_madw_req_t
*) (cl_list_remove_head(&retry_madw_p_list));
while (osm_madw_req_p != NULL) {
/* resend: */
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_transaction_mgr_callback : "
"Retry %d of madw %p\n",
OSM_DEFAULT_RETRY_COUNT -
osm_madw_req_p->retry_cnt,
osm_madw_req_p->p_madw);
/* actually send it */
#ifdef OSM_VENDOR_INTF_MTL
osm_mtl_send_mad((osm_mtl_bind_info_t *)
osm_madw_req_p->p_bind,
osm_madw_req_p->p_madw);
#else
ib_api_status_t
osm_ts_send_mad(osm_ts_bind_info_t * p_bind,
osm_madw_t * const p_madw);
osm_ts_send_mad((osm_ts_bind_info_t *) osm_madw_req_p->
p_bind, osm_madw_req_p->p_madw);
#endif
/* next one */
osm_madw_req_p =
(osm_madw_req_t
*) (cl_list_remove_head(&retry_madw_p_list));
}
}
/* if the tmp_madw_p_list has elements - need to call the send_err_callback */
madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
while (madw_p != NULL) {
/* need to remove it from pool */
/* obtain the madw_p stored as the wrid in the send call */
p_mad = osm_madw_get_mad_ptr(madw_p);
p_bind = madw_p->h_bind;
/*
Return any wrappers to the pool that may have been
pre-emptively allocated to handle a receive.
*/
if (madw_p->vend_wrap.p_resp_madw) {
#ifdef OSM_VENDOR_INTF_MTL
osm_mad_pool_put(p_bind->p_osm_pool,
madw_p->vend_wrap.p_resp_madw);
#else
osm_mad_pool_put(p_bind->p_osm_pool,
madw_p->vend_wrap.p_resp_madw);
#endif
madw_p->vend_wrap.p_resp_madw = NULL;
}
/* invoke the CB */
(*(osm_vend_mad_send_err_callback_t)
(p_bind->send_err_callback)) (p_bind->client_context, madw_p);
madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
}
Exit:
OSM_LOG_EXIT(p_vend->p_log);
}
static void __cl_event_wheel_callback(IN void *context)
{
cl_event_wheel_t *p_event_wheel = (cl_event_wheel_t *) context;
cl_list_item_t *p_list_item, *p_prev_event_list_item;
cl_list_item_t *p_list_next_item;
cl_event_wheel_reg_info_t *p_event;
uint64_t current_time;
uint64_t next_aging_time;
uint32_t new_timeout;
cl_status_t cl_status;
/* might be during closing ... */
if (p_event_wheel->closing)
return;
current_time = cl_get_time_stamp();
if (NULL != p_event_wheel->p_external_lock)
/* Take care of the order of acquiring locks to avoid the deadlock!
* The external lock goes first.
*/
cl_spinlock_acquire(p_event_wheel->p_external_lock);
cl_spinlock_acquire(&p_event_wheel->lock);
p_list_item = cl_qlist_head(&p_event_wheel->events_wheel);
if (p_list_item == cl_qlist_end(&p_event_wheel->events_wheel))
/* the list is empty - nothing to do */
goto Exit;
/* we found such an item. get the p_event */
p_event =
PARENT_STRUCT(p_list_item, cl_event_wheel_reg_info_t, list_item);
while (p_event->aging_time <= current_time) {
/* this object has aged - invoke it's callback */
if (p_event->pfn_aged_callback)
next_aging_time =
p_event->pfn_aged_callback(p_event->key,
p_event->num_regs,
p_event->context);
else
next_aging_time = 0;
/* point to the next object in the wheel */
p_list_next_item = cl_qlist_next(p_list_item);
/* We need to retire the event if the next aging time passed */
if (next_aging_time < current_time) {
/* remove it from the map */
cl_qmap_remove_item(&p_event_wheel->events_map,
&(p_event->map_item));
/* pop p_event from the wheel */
cl_qlist_remove_head(&p_event_wheel->events_wheel);
/* delete the event info object - allocated by cl_event_wheel_reg */
free(p_event);
} else {
/* update the required aging time */
p_event->aging_time = next_aging_time;
p_event->num_regs++;
/* do not remove from the map - but remove from the list head and
place in the correct position */
/* pop p_event from the wheel */
cl_qlist_remove_head(&p_event_wheel->events_wheel);
/* find the event that ages just before */
p_prev_event_list_item =
cl_qlist_find_from_tail(&p_event_wheel->
events_wheel,
__event_will_age_before,
&p_event->aging_time);
/* insert just after */
cl_qlist_insert_next(&p_event_wheel->events_wheel,
p_prev_event_list_item,
&p_event->list_item);
/* as we have modified the list - restart from first item: */
p_list_next_item =
cl_qlist_head(&p_event_wheel->events_wheel);
}
/* advance to next event */
p_list_item = p_list_next_item;
if (p_list_item == cl_qlist_end(&p_event_wheel->events_wheel))
/* the list is empty - nothing to do */
break;
/* get the p_event */
p_event =
PARENT_STRUCT(p_list_item, cl_event_wheel_reg_info_t,
list_item);
}
/* We need to restart the timer only if the list is not empty now */
if (p_list_item != cl_qlist_end(&p_event_wheel->events_wheel)) {
/* get the p_event */
p_event =
PARENT_STRUCT(p_list_item, cl_event_wheel_reg_info_t,
list_item);
/* start the timer to the timeout [msec] */
new_timeout =
(uint32_t) ((p_event->aging_time - current_time + 500) / 1000);
CL_DBG("__cl_event_wheel_callback: Restart timer in: "
"%u [msec]\n", new_timeout);
cl_status = cl_timer_start(&p_event_wheel->timer, new_timeout);
if (cl_status != CL_SUCCESS) {
CL_DBG("__cl_event_wheel_callback : ERR 6200: "
"Failed to start timer\n");
}
}
/* release the lock */
Exit:
cl_spinlock_release(&p_event_wheel->lock);
if (NULL != p_event_wheel->p_external_lock)
cl_spinlock_release(p_event_wheel->p_external_lock);
}
cl_status_t cl_event_wheel_reg(IN cl_event_wheel_t * const p_event_wheel,
IN const uint64_t key,
IN const uint64_t aging_time_usec,
IN cl_pfn_event_aged_cb_t pfn_callback,
IN void *const context)
{
cl_event_wheel_reg_info_t *p_event;
uint64_t timeout;
uint32_t to;
cl_status_t cl_status = CL_SUCCESS;
cl_list_item_t *prev_event_list_item;
cl_map_item_t *p_map_item;
/* Get the lock on the manager */
cl_spinlock_acquire(&(p_event_wheel->lock));
cl_event_wheel_dump(p_event_wheel);
/* Make sure such a key does not exists */
p_map_item = cl_qmap_get(&p_event_wheel->events_map, key);
if (p_map_item != cl_qmap_end(&p_event_wheel->events_map)) {
CL_DBG("cl_event_wheel_reg: Already exists key:0x%"
PRIx64 "\n", key);
/* already there - remove it from the list as it is getting a new time */
p_event =
PARENT_STRUCT(p_map_item, cl_event_wheel_reg_info_t,
map_item);
/* remove the item from the qlist */
cl_qlist_remove_item(&p_event_wheel->events_wheel,
&p_event->list_item);
/* and the qmap */
cl_qmap_remove_item(&p_event_wheel->events_map,
&p_event->map_item);
} else {
/* make a new one */
p_event = (cl_event_wheel_reg_info_t *)
malloc(sizeof(cl_event_wheel_reg_info_t));
p_event->num_regs = 0;
}
p_event->key = key;
p_event->aging_time = aging_time_usec;
p_event->pfn_aged_callback = pfn_callback;
p_event->context = context;
p_event->num_regs++;
CL_DBG("cl_event_wheel_reg: Registering event key:0x%" PRIx64
" aging in %u [msec]\n", p_event->key,
(uint32_t) ((p_event->aging_time - cl_get_time_stamp()) / 1000));
/* If the list is empty - need to start the timer */
if (cl_is_qlist_empty(&p_event_wheel->events_wheel)) {
/* Edward Bortnikov 03/29/2003
* ++TBD Consider moving the timer manipulation behind the list manipulation.
*/
/* calculate the new timeout */
timeout =
(p_event->aging_time - cl_get_time_stamp() + 500) / 1000;
/* stop the timer if it is running */
/* Edward Bortnikov 03/29/2003
* Don't call cl_timer_stop() because it spins forever.
* cl_timer_start() will invoke cl_timer_stop() by itself.
*
* The problematic scenario is when __cl_event_wheel_callback()
* is in race condition with this code. It sets timer.in_timer_cb
* to TRUE and then blocks on p_event_wheel->lock. Following this,
* the call to cl_timer_stop() hangs. Following this, the whole system
* enters into a deadlock.
*
* cl_timer_stop(&p_event_wheel->timer);
*/
/* The timeout for the cl_timer_start should be given as uint32_t.
if there is an overflow - warn about it. */
to = (uint32_t) timeout;
if (timeout > (uint32_t) timeout) {
to = 0xffffffff; /* max 32 bit timer */
CL_DBG("cl_event_wheel_reg: timeout requested is "
"too large. Using timeout: %u\n", to);
}
/* start the timer to the timeout [msec] */
cl_status = cl_timer_start(&p_event_wheel->timer, to);
if (cl_status != CL_SUCCESS) {
CL_DBG("cl_event_wheel_reg : ERR 6203: "
"Failed to start timer\n");
goto Exit;
}
}
/* insert the object to the qlist and the qmap */
/* BUT WE MUST INSERT IT IN A SORTED MANNER */
prev_event_list_item =
cl_qlist_find_from_tail(&p_event_wheel->events_wheel,
__event_will_age_before,
&p_event->aging_time);
cl_qlist_insert_next(&p_event_wheel->events_wheel,
prev_event_list_item, &p_event->list_item);
cl_qmap_insert(&p_event_wheel->events_map, key, &(p_event->map_item));
Exit:
cl_spinlock_release(&p_event_wheel->lock);
return cl_status;
}
