cl_pool_item_t *cl_qcpool_get_tail(IN cl_qcpool_t * const p_pool)
{
cl_list_item_t *p_list_item;
CL_ASSERT(p_pool);
CL_ASSERT(p_pool->state == CL_INITIALIZED);
if (cl_is_qlist_empty(&p_pool->free_list)) {
/*
* No object is available.
* Return NULL if the user does not want automatic growth.
*/
if (!p_pool->grow_size)
return (NULL);
/* We ran out of elements. Get more */
cl_qcpool_grow(p_pool, p_pool->grow_size);
/*
* We may not have gotten everything we wanted but we might have
* gotten something.
*/
if (cl_is_qlist_empty(&p_pool->free_list))
return (NULL);
}
p_list_item = cl_qlist_remove_tail(&p_pool->free_list);
/* OK, at this point we have an object */
CL_ASSERT(p_list_item != cl_qlist_end(&p_pool->free_list));
return ((cl_pool_item_t *) p_list_item);
}
void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool)
{
osm_madw_t *p_madw;
OSM_LOG_ENTER(p_vl->p_log);
/*
Signal our threads that we're leaving.
*/
p_vl->thread_state = OSM_THREAD_STATE_EXIT;
/*
Don't trigger unless event has been initialized.
Destroy the thread before we tear down the other objects.
*/
if (p_vl->state != OSM_VL15_STATE_INIT)
cl_event_signal(&p_vl->signal);
cl_thread_destroy(&p_vl->poller);
/*
Return the outstanding messages to the pool
*/
cl_spinlock_acquire(&p_vl->lock);
while (!cl_is_qlist_empty(&p_vl->rfifo)) {
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
osm_mad_pool_put(p_pool, p_madw);
}
while (!cl_is_qlist_empty(&p_vl->ufifo)) {
p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
osm_mad_pool_put(p_pool, p_madw);
}
cl_spinlock_release(&p_vl->lock);
cl_event_destroy(&p_vl->signal);
p_vl->state = OSM_VL15_STATE_INIT;
cl_spinlock_destroy(&p_vl->lock);
OSM_LOG_EXIT(p_vl->p_log);
}
void cl_qcpool_destroy(IN cl_qcpool_t * const p_pool)
{
/* CL_ASSERT that a non-NULL pointer was provided. */
CL_ASSERT(p_pool);
/* CL_ASSERT that we are in a valid state (not uninitialized memory). */
CL_ASSERT(cl_is_state_valid(p_pool->state));
if (p_pool->state == CL_INITIALIZED) {
/*
* Assert if the user hasn't put everything back in the pool
* before destroying it
* if they haven't, then most likely they are still using memory
* that will be freed, and the destructor will not be called!
*/
#ifdef _DEBUG_
/* but we do not want "free" version to assert on this one */
CL_ASSERT(cl_qcpool_count(p_pool) == p_pool->num_objects);
#endif
/* call the user's destructor for each object in the pool */
if (p_pool->pfn_dtor) {
while (!cl_is_qlist_empty(&p_pool->free_list)) {
p_pool->pfn_dtor((cl_pool_item_t *)
cl_qlist_remove_head(&p_pool->
free_list),
(void *)p_pool->context);
}
} else {
cl_qlist_remove_all(&p_pool->free_list);
}
/* Free all allocated memory blocks. */
while (!cl_is_qlist_empty(&p_pool->alloc_list))
free(cl_qlist_remove_head(&p_pool->alloc_list));
if (p_pool->component_sizes) {
free(p_pool->component_sizes);
p_pool->component_sizes = NULL;
}
}
p_pool->state = CL_UNINITIALIZED;
}
cl_status_t
cl_timer_start(IN cl_timer_t * const p_timer, IN const uint32_t time_ms)
{
struct timeval curtime;
cl_list_item_t *p_list_item;
uint32_t delta_time = time_ms;
CL_ASSERT(p_timer);
CL_ASSERT(p_timer->state == CL_INITIALIZED);
pthread_mutex_lock(&gp_timer_prov->mutex);
/* Signal the timer provider thread to wake up. */
pthread_cond_signal(&gp_timer_prov->cond);
/* Remove the timer from the queue if currently queued. */
if (p_timer->timer_state == CL_TIMER_QUEUED)
cl_qlist_remove_item(&gp_timer_prov->queue,
&p_timer->list_item);
/* Get the current time */
#ifndef timerclear
#define timerclear(tvp) (tvp)->tv_sec = (time_t)0, (tvp)->tv_usec = 0L
#endif
timerclear(&curtime);
gettimeofday(&curtime, NULL);
/* do not do 0 wait ! */
/* if (delta_time < 1000.0) {delta_time = 1000;} */
/* Calculate the timeout. */
p_timer->timeout.tv_sec = curtime.tv_sec + (delta_time / 1000);
p_timer->timeout.tv_nsec =
(curtime.tv_usec + ((delta_time % 1000) * 1000)) * 1000;
/* Add the timer to the queue. */
if (cl_is_qlist_empty(&gp_timer_prov->queue)) {
/* The timer list is empty. Add to the head. */
cl_qlist_insert_head(&gp_timer_prov->queue,
&p_timer->list_item);
} else {
/* Find the correct insertion place in the list for the timer. */
p_list_item = cl_qlist_find_from_tail(&gp_timer_prov->queue,
__cl_timer_find, p_timer);
/* Insert the timer. */
cl_qlist_insert_next(&gp_timer_prov->queue, p_list_item,
&p_timer->list_item);
}
/* Set the state. */
p_timer->timer_state = CL_TIMER_QUEUED;
pthread_mutex_unlock(&gp_timer_prov->mutex);
return (CL_SUCCESS);
}
void cl_disp_shutdown(IN cl_dispatcher_t * const p_disp)
{
CL_ASSERT(p_disp);
/* Stop the thread pool. */
cl_thread_pool_destroy(&p_disp->worker_threads);
/* Process all outstanding callbacks. */
__cl_disp_worker(p_disp);
/* Free all registration info. */
while (!cl_is_qlist_empty(&p_disp->reg_list))
free(cl_qlist_remove_head(&p_disp->reg_list));
}
/*
* This is the internal work function executed by the timer's thread.
*/
static void *__cl_timer_prov_cb(IN void *const context)
{
int ret;
cl_timer_t *p_timer;
pthread_mutex_lock(&gp_timer_prov->mutex);
while (!gp_timer_prov->exit) {
if (cl_is_qlist_empty(&gp_timer_prov->queue)) {
/* Wait until we exit or a timer is queued. */
/* cond wait does:
* pthread_cond_wait atomically unlocks the mutex (as per
* pthread_unlock_mutex) and waits for the condition variable
* cond to be signaled. The thread execution is suspended and
* does not consume any CPU time until the condition variable is
* signaled. The mutex must be locked by the calling thread on
* entrance to pthread_cond_wait. Before RETURNING TO THE
* CALLING THREAD, PTHREAD_COND_WAIT RE-ACQUIRES MUTEX (as per
* pthread_lock_mutex).
*/
ret = pthread_cond_wait(&gp_timer_prov->cond,
&gp_timer_prov->mutex);
} else {
/*
* The timer elements are on the queue in expiration order.
* Get the first in the list to determine how long to wait.
*/
p_timer =
(cl_timer_t *) cl_qlist_head(&gp_timer_prov->queue);
ret =
pthread_cond_timedwait(&gp_timer_prov->cond,
&gp_timer_prov->mutex,
&p_timer->timeout);
/*
Sleep again on every event other than timeout and invalid
Note: EINVAL means that we got behind. This can occur when
we are very busy...
*/
if (ret != ETIMEDOUT && ret != EINVAL)
continue;
/*
* The timer expired. Check the state in case it was cancelled
* after it expired but before we got a chance to invoke the
* callback.
*/
if (p_timer->timer_state != CL_TIMER_QUEUED)
continue;
/*
* Mark the timer as running to synchronize with its
* cancelation since we can't hold the mutex during the
* callback.
*/
p_timer->timer_state = CL_TIMER_RUNNING;
/* Remove the item from the timer queue. */
cl_qlist_remove_item(&gp_timer_prov->queue,
&p_timer->list_item);
pthread_mutex_unlock(&gp_timer_prov->mutex);
/* Invoke the callback. */
p_timer->pfn_callback((void *)p_timer->context);
/* Acquire the mutex again. */
pthread_mutex_lock(&gp_timer_prov->mutex);
/*
* Only set the state to idle if the timer has not been accessed
* from the callback
*/
if (p_timer->timer_state == CL_TIMER_RUNNING)
p_timer->timer_state = CL_TIMER_IDLE;
/*
* Signal any thread trying to manipulate the timer
* that expired.
*/
pthread_cond_signal(&p_timer->cond);
}
}
gp_timer_prov->thread = 0;
pthread_mutex_unlock(&gp_timer_prov->mutex);
pthread_exit(NULL);
}
ib_api_status_t
osm_transaction_mgr_insert_madw(IN osm_bind_handle_t * const p_bind,
IN osm_madw_t * p_madw)
{
#ifdef OSM_VENDOR_INTF_MTL
osm_vendor_t *const p_vend = ((osm_mtl_bind_info_t *) p_bind)->p_vend;
#else
osm_vendor_t *const p_vend = ((osm_ts_bind_info_t *) p_bind)->p_vend;
#endif
osm_transaction_mgr_t *trans_mgr_p;
osm_madw_req_t *osm_madw_req_p;
uint64_t timeout;
uint64_t waking_time;
cl_status_t cl_status;
uint64_t key;
const ib_mad_t *mad_p = p_madw->p_mad;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(mad_p);
trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
timeout = (uint64_t) (p_vend->timeout) * 1000; /* change the miliseconds value of timeout to microseconds. */
waking_time = timeout + cl_get_time_stamp();
osm_madw_req_p = (osm_madw_req_t *) malloc(sizeof(osm_madw_req_t));
osm_madw_req_p->p_madw = p_madw;
osm_madw_req_p->waking_time = waking_time;
osm_madw_req_p->retry_cnt = OSM_DEFAULT_RETRY_COUNT;
osm_madw_req_p->p_bind = p_bind;
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_transaction_mgr_insert_madw: "
"Inserting MADW:%p with waking_time: <0x%" PRIx64 "> TID:<0x%"
PRIx64 ">.\n", p_madw, waking_time, p_madw->p_mad->trans_id);
/* Get the lock on the manager */
cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
/* If the list is empty - need to start the timer with timer of timeout (in miliseconds) */
if (cl_is_qlist_empty(trans_mgr_p->madw_reqs_list_p)) {
/* stop the timer if it is running */
cl_timer_stop(&trans_mgr_p->madw_list_timer);
/* start the timer to the timeout (in miliseconds) */
cl_status = cl_timer_start(&trans_mgr_p->madw_list_timer,
p_vend->timeout);
if (cl_status != CL_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_transaction_mgr_insert_madw : ERROR 1000: "
"Failed to start timer\n");
}
}
/* insert the object to the qlist and the qmap */
cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
&(osm_madw_req_p->list_item));
/* get the key */
key = (uint64_t) mad_p->trans_id;
cl_qmap_insert(trans_mgr_p->madw_by_tid_map_p, key,
&(osm_madw_req_p->map_item));
cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
OSM_LOG_EXIT(p_vend->p_log);
return (IB_SUCCESS);
}
/* rank is a SWITCH for BFS purpose */
static int updn_subn_rank(IN updn_t * p_updn)
{
osm_switch_t *p_sw;
osm_physp_t *p_physp, *p_remote_physp;
cl_qlist_t list;
cl_map_item_t *item;
struct updn_node *u, *remote_u;
uint8_t num_ports, port_num;
osm_log_t *p_log = &p_updn->p_osm->log;
unsigned max_rank = 0;
OSM_LOG_ENTER(p_log);
cl_qlist_init(&list);
/* add all roots to the list */
for (item = cl_qmap_head(&p_updn->p_osm->subn.sw_guid_tbl);
item != cl_qmap_end(&p_updn->p_osm->subn.sw_guid_tbl);
item = cl_qmap_next(item)) {
p_sw = (osm_switch_t *)item;
u = p_sw->priv;
if (!u->rank)
cl_qlist_insert_tail(&list, &u->list);
}
/* BFS the list till it's empty */
while (!cl_is_qlist_empty(&list)) {
u = (struct updn_node *)cl_qlist_remove_head(&list);
/* Go over all remote nodes and rank them (if not already visited) */
p_sw = u->sw;
num_ports = p_sw->num_ports;
OSM_LOG(p_log, OSM_LOG_DEBUG,
"Handling switch GUID 0x%" PRIx64 "\n",
cl_ntoh64(osm_node_get_node_guid(p_sw->p_node)));
for (port_num = 1; port_num < num_ports; port_num++) {
ib_net64_t port_guid;
/* Current port fetched in order to get remote side */
p_physp =
osm_node_get_physp_ptr(p_sw->p_node, port_num);
if (!p_physp)
continue;
p_remote_physp = p_physp->p_remote_physp;
/*
make sure that all the following occur on p_remote_physp:
1. The port isn't NULL
2. It is a switch
*/
if (p_remote_physp && p_remote_physp->p_node->sw) {
remote_u = p_remote_physp->p_node->sw->priv;
port_guid = p_remote_physp->port_guid;
if (remote_u->rank > u->rank + 1) {
remote_u->rank = u->rank + 1;
max_rank = remote_u->rank;
cl_qlist_insert_tail(&list,
&remote_u->list);
OSM_LOG(p_log, OSM_LOG_DEBUG,
"Rank of port GUID 0x%" PRIx64
" = %u\n", cl_ntoh64(port_guid),
remote_u->rank);
}
}
}
}
/* Print Summary of ranking */
OSM_LOG(p_log, OSM_LOG_VERBOSE,
"Subnet ranking completed. Max Node Rank = %d\n", max_rank);
OSM_LOG_EXIT(p_log);
return 0;
}
/**********************************************************************
* This function does the bfs of min hop table calculation by guid index
* as a starting point.
**********************************************************************/
static int updn_bfs_by_node(IN osm_log_t * p_log, IN osm_subn_t * p_subn,
IN osm_switch_t * p_sw)
{
uint8_t pn, pn_rem;
cl_qlist_t list;
uint16_t lid;
struct updn_node *u;
updn_switch_dir_t next_dir, current_dir;
OSM_LOG_ENTER(p_log);
lid = osm_node_get_base_lid(p_sw->p_node, 0);
lid = cl_ntoh16(lid);
osm_switch_set_hops(p_sw, lid, 0, 0);
OSM_LOG(p_log, OSM_LOG_DEBUG,
"Starting from switch - port GUID 0x%" PRIx64 " lid %u\n",
cl_ntoh64(p_sw->p_node->node_info.port_guid), lid);
u = p_sw->priv;
u->dir = UP;
/* Update list with the new element */
cl_qlist_init(&list);
cl_qlist_insert_tail(&list, &u->list);
/* BFS the list till no next element */
while (!cl_is_qlist_empty(&list)) {
u = (struct updn_node *)cl_qlist_remove_head(&list);
u->visited = 0; /* cleanup */
current_dir = u->dir;
/* Go over all ports of the switch and find unvisited remote nodes */
for (pn = 1; pn < u->sw->num_ports; pn++) {
osm_node_t *p_remote_node;
struct updn_node *rem_u;
uint8_t current_min_hop, remote_min_hop,
set_hop_return_value;
osm_switch_t *p_remote_sw;
p_remote_node =
osm_node_get_remote_node(u->sw->p_node, pn,
&pn_rem);
/* If no remote node OR remote node is not a SWITCH
continue to next pn */
if (!p_remote_node || !p_remote_node->sw)
continue;
/* Fetch remote guid only after validation of remote node */
p_remote_sw = p_remote_node->sw;
rem_u = p_remote_sw->priv;
/* Decide which direction to mark it (UP/DOWN) */
next_dir = updn_get_dir(u->rank, rem_u->rank,
u->id, rem_u->id);
/* Check if this is a legal step : the only illegal step is going
from DOWN to UP */
if ((current_dir == DOWN) && (next_dir == UP)) {
OSM_LOG(p_log, OSM_LOG_DEBUG,
"Avoiding move from 0x%016" PRIx64
" to 0x%016" PRIx64 "\n",
cl_ntoh64(osm_node_get_node_guid(u->sw->p_node)),
cl_ntoh64(osm_node_get_node_guid(p_remote_node)));
/* Illegal step */
continue;
}
/* Set MinHop value for the current lid */
current_min_hop = osm_switch_get_least_hops(u->sw, lid);
/* Check hop count if better insert into list && update
the remote node Min Hop Table */
remote_min_hop =
osm_switch_get_hop_count(p_remote_sw, lid, pn_rem);
if (current_min_hop + 1 < remote_min_hop) {
set_hop_return_value =
osm_switch_set_hops(p_remote_sw, lid,
pn_rem,
current_min_hop + 1);
if (set_hop_return_value) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR AA01: "
"Invalid value returned from set min hop is: %d\n",
set_hop_return_value);
}
/* Check if remote port has already been visited */
if (!rem_u->visited) {
/* Insert updn_switch item into the list */
rem_u->dir = next_dir;
rem_u->visited = 1;
cl_qlist_insert_tail(&list,
&rem_u->list);
}
}
}
}
OSM_LOG_EXIT(p_log);
return 0;
}
cl_status_t cl_event_wheel_reg(IN cl_event_wheel_t * const p_event_wheel,
IN const uint64_t key,
IN const uint64_t aging_time_usec,
IN cl_pfn_event_aged_cb_t pfn_callback,
IN void *const context)
{
cl_event_wheel_reg_info_t *p_event;
uint64_t timeout;
uint32_t to;
cl_status_t cl_status = CL_SUCCESS;
cl_list_item_t *prev_event_list_item;
cl_map_item_t *p_map_item;
/* Get the lock on the manager */
cl_spinlock_acquire(&(p_event_wheel->lock));
cl_event_wheel_dump(p_event_wheel);
/* Make sure such a key does not exists */
p_map_item = cl_qmap_get(&p_event_wheel->events_map, key);
if (p_map_item != cl_qmap_end(&p_event_wheel->events_map)) {
CL_DBG("cl_event_wheel_reg: Already exists key:0x%"
PRIx64 "\n", key);
/* already there - remove it from the list as it is getting a new time */
p_event =
PARENT_STRUCT(p_map_item, cl_event_wheel_reg_info_t,
map_item);
/* remove the item from the qlist */
cl_qlist_remove_item(&p_event_wheel->events_wheel,
&p_event->list_item);
/* and the qmap */
cl_qmap_remove_item(&p_event_wheel->events_map,
&p_event->map_item);
} else {
/* make a new one */
p_event = (cl_event_wheel_reg_info_t *)
malloc(sizeof(cl_event_wheel_reg_info_t));
p_event->num_regs = 0;
}
p_event->key = key;
p_event->aging_time = aging_time_usec;
p_event->pfn_aged_callback = pfn_callback;
p_event->context = context;
p_event->num_regs++;
CL_DBG("cl_event_wheel_reg: Registering event key:0x%" PRIx64
" aging in %u [msec]\n", p_event->key,
(uint32_t) ((p_event->aging_time - cl_get_time_stamp()) / 1000));
/* If the list is empty - need to start the timer */
if (cl_is_qlist_empty(&p_event_wheel->events_wheel)) {
/* Edward Bortnikov 03/29/2003
* ++TBD Consider moving the timer manipulation behind the list manipulation.
*/
/* calculate the new timeout */
timeout =
(p_event->aging_time - cl_get_time_stamp() + 500) / 1000;
/* stop the timer if it is running */
/* Edward Bortnikov 03/29/2003
* Don't call cl_timer_stop() because it spins forever.
* cl_timer_start() will invoke cl_timer_stop() by itself.
*
* The problematic scenario is when __cl_event_wheel_callback()
* is in race condition with this code. It sets timer.in_timer_cb
* to TRUE and then blocks on p_event_wheel->lock. Following this,
* the call to cl_timer_stop() hangs. Following this, the whole system
* enters into a deadlock.
*
* cl_timer_stop(&p_event_wheel->timer);
*/
/* The timeout for the cl_timer_start should be given as uint32_t.
if there is an overflow - warn about it. */
to = (uint32_t) timeout;
if (timeout > (uint32_t) timeout) {
to = 0xffffffff; /* max 32 bit timer */
CL_DBG("cl_event_wheel_reg: timeout requested is "
"too large. Using timeout: %u\n", to);
}
/* start the timer to the timeout [msec] */
cl_status = cl_timer_start(&p_event_wheel->timer, to);
if (cl_status != CL_SUCCESS) {
CL_DBG("cl_event_wheel_reg : ERR 6203: "
"Failed to start timer\n");
goto Exit;
}
}
/* insert the object to the qlist and the qmap */
/* BUT WE MUST INSERT IT IN A SORTED MANNER */
prev_event_list_item =
cl_qlist_find_from_tail(&p_event_wheel->events_wheel,
__event_will_age_before,
&p_event->aging_time);
cl_qlist_insert_next(&p_event_wheel->events_wheel,
prev_event_list_item, &p_event->list_item);
cl_qmap_insert(&p_event_wheel->events_map, key, &(p_event->map_item));
Exit:
cl_spinlock_release(&p_event_wheel->lock);
return cl_status;
}
