/*
* reserving context inserts it into the hash table if !hashed
* and increments reference count
*/
Status_t
sa_cntxt_reserve( sa_cntxt_t* sa_cntxt )
{
int bucket;
Status_t status;
IB_ENTER( "sa_cntxt_reserve", sa_cntxt, 0, 0, 0 );
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_reserve: Failed to lock SA context rc:", status);
} else {
sa_cntxt->ref ++;
if( sa_cntxt->hashed == 0 ) {
// This context needs to be inserted into the hash table
bucket = sa_cntxt->lid % SA_CNTXT_HASH_TABLE_DEPTH;
sa_cntxt->hashed = 1 ;
vs_time_get( &sa_cntxt->tstamp );
sa_cntxt_insert_head( sa_hash[ bucket ], sa_cntxt );
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_reserve: Failed to unlock SA context rc:", status);
}
}
IB_EXIT( "sa_cntxt_reserve", VSTATUS_OK );
return VSTATUS_OK ;
}
/*
* Age context and do resends for those that timed out
*/
void sa_cntxt_age(void)
{
sa_cntxt_t* sa_cntxt = NULL ;
sa_cntxt_t* tout_cntxt ;
int i;
Status_t status;
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_age: Failed to lock SA context rc:", status);
} else {
vs_time_get( &timeLastAged );
for (i=0; i<SA_CNTXT_HASH_TABLE_DEPTH; i++) {
sa_cntxt = sa_hash[ i ];
while( sa_cntxt ) {
// Iterate before the pointers are destroyed ;
tout_cntxt = sa_cntxt ;
sa_cntxt = sa_cntxt->next ;
// Though list is sorted, it does not hurt to
// do a simple comparison
if( timeLastAged - tout_cntxt->tstamp > tout_cntxt->RespTimeout) { // used to be VTIMER_1S
// Timeout this entry
sa_cntxt_delete_entry( sa_hash[ i ], tout_cntxt );
sa_cntxt_insert_head( sa_hash[ i ], tout_cntxt );
// Touch the entry
tout_cntxt->tstamp = timeLastAged ;
// Call timeout
tout_cntxt->sendFd = fd_sa_w; // use sa writer mai handle for restransmits
if (tout_cntxt->method == SA_CM_GETMULTI && tout_cntxt->reqInProg) {
// resend the getMulti request ACK
sa_getMulti_resend_ack(tout_cntxt);
/* if need to release the context. Call local safe release */
if( tout_cntxt->retries > sm_config.max_retries ) cntxt_release(sa_cntxt);
} else {
// resend the reply
sa_send_reply( NULL, tout_cntxt );
if( tout_cntxt->retries > sm_config.max_retries ) {
/* need to release the context. Call local safe release */
cntxt_release(tout_cntxt);
}
}
}
}
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("cs_cntxt_age: Failed to unlock SA context rc:", status);
}
}
}
int bitset_resize(bitset_t *bitset, size_t n) {
Status_t status;
size_t orig_nbits = bitset->nbits_m;
size_t orig_nset = bitset->nset_m;
size_t orig_nwords = bitset->nwords_m;
uint32_t *orig_bits = bitset->bits_m;
if (n == bitset->nbits_m) return 1;
bitset->bits_m = NULL;
if (!bitset_init(bitset->pool_m, bitset, n)) {
bitset->nbits_m = orig_nbits;
bitset->nset_m = orig_nset;
bitset->nwords_m = orig_nwords;
bitset->bits_m = orig_bits;
return 0;
}
if (!orig_bits) return 1;
if (n > orig_nbits) {
memcpy(bitset->bits_m, orig_bits, orig_nwords*sizeof(uint32_t));
bitset->nset_m = orig_nset;
} else {
memcpy(bitset->bits_m, orig_bits, bitset->nwords_m*sizeof(uint32_t));
bitset->nset_m = count_nset(bitset);
}
if ((status = vs_pool_free(bitset->pool_m, orig_bits)) != VSTATUS_OK) {
IB_LOG_ERRORRC("can't free allocated space for bitset, rc:", status);
}
return 1;
}
int
mai_spawn_dc_reader(int qp, int dev, int port)
{
int rc;
unsigned char name[VS_NAME_MAX];
IB_ENTER(__func__, qp, dev, port, 0);
dc_reader_args.qp = qp;
dc_reader_args.dev = dev;
dc_reader_args.port = port;
/*
* start the workert thread
*/
(void) memset (name, 8, sizeof(name));
(void) sprintf((void *)name,"mai_dc");
rc = vs_thread_create (&dc_thandle,
name,
(void (*)(uint32_t, uint8_t **))dc_reader_start,
0,NULL ,
MAI_DC_STACK_SIZE);
if (rc) {
IB_LOG_ERRORRC("failed to create MAI layer server thread rc:", rc);
IB_EXIT(__func__, rc);
return rc;
} else {
/* need to wait (up to 2 seconds) for the dedicated dcthread to initialize here */
return(cs_psema_wait(&gMAI_DCTHREAD_SEMA, 2));
}
IB_LOG_INFO("INFO: MAI layer server thread created ", rc);
IB_EXIT(__func__, 0);
return VSTATUS_OK;
}
Status_t
sm_fsm_discovering(Mai_t *maip, char *nodename)
{
Status_t status;
STL_SM_INFO theirSmInfo;
Lid_t slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)STL_GET_SMP_DATA(maip), &theirSmInfo);
//
// The SM in the DISCOVERING state does not have any valid transitions via
// the Set(SMInfo) method.
//
maip->base.status = MAD_STATUS_BAD_ATTR;
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)STL_GET_SMP_DATA(maip));
status = mai_stl_reply(fd_async, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_discovering - bad mai_reply rc:", status);
}
IB_LOG_WARN_FMT(__func__,
"No transitions allowed from DISCOVERING state; Got %s request from [%s] SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
getAmod(maip->base.amod), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID, maip->base.tid);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_routing_alloc_cost_matrix(Topology_t *topop)
{
Status_t status;
size_t bytesCost;
/* Allocate space for the cost array. */
bytesCost = topop->max_sws * topop->max_sws * sizeof(uint16_t);
if (bytesCost > topop->bytes) {
topop->bytes = 0;
if (topop->cost != NULL) {
(void)vs_pool_free(&sm_pool, (void *)topop->cost);
topop->cost = NULL;
}
status = vs_pool_alloc(&sm_pool, bytesCost, (void *)&topop->cost);
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("can't malloc cost array rc:", status);
IB_EXIT(__func__, status);
return status;
}
topop->bytes = bytesCost;
}
return VSTATUS_OK;
}
/*
* clear the contents of the context pool
*/
void sa_cntxt_clear(void) {
int i;
Status_t status;
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_clear: Failed to lock SA context, rc:", status);
} else {
memset( sa_hash, 0, sizeof( sa_hash ));
memset( sa_cntxt_pool, 0, sizeof( sa_cntxt_t ) * sa_max_cntxt);
sa_cntxt_free_list = NULL ;
for( i = 0 ; i < sa_max_cntxt ; ++i ) {
sa_cntxt_insert_head( sa_cntxt_free_list, &sa_cntxt_pool[i] );
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_clear: Failed to unlock SA contex, rc:", status);
}
}
}
uint32_t fe_vieo_init(uint8_t *logName)
{
uint32_t rc = SUCCESS;
static const char func[] = "fe_vieo_init";
IB_ENTER(func, 0, 0, 0, 0);
fd_dm = INVALID_HANDLE;
fdsa = INVALID_HANDLE;
// initialize MAI subsystem
mai_set_num_end_ports( MIN(fe_config.subnet_size, MAI_MAX_QUEUED_DEFAULT) );
mai_init();
IB_LOG_INFO("Device = ", fe_config.hca);
rc = if3_register_fe(fe_config.hca,fe_config.port,(void *)FE_SERVICE_NAME,FE_SERVICE_ID, IF3_REGFORCE_PORT,&fdsa);
if (rc != VSTATUS_OK) {
if (fe_config.debug) IB_LOG_INFINI_INFORC("Failed to register with IF3, will try later. rc:", rc);
} else {
rc = fe_if3_subscribe_sa();
if (rc != VSTATUS_OK) {
IB_LOG_ERRORRC("Failed to subscribe for traps in SA rc:", rc);
}
// connect to Performance Manager
if (pm_lid)
rc = if3_lid_mngr_cnx(fe_config.hca,fe_config.port,MAD_CV_VFI_PM,pm_lid,&fd_pm);
else
rc = if3_sid_mngr_cnx(fe_config.hca,fe_config.port,(void *)PM_SERVICE_NAME,PM_SERVICE_ID,
MAD_CV_VFI_PM,&fd_pm);
if (rc != VSTATUS_OK) {
IB_LOG_INFINI_INFORC("Failed to open Performance Manager, will try later. rc:", rc);
fd_pm = INVALID_HANDLE;
}
#ifdef DEVICE_MANAGER // not implemented yet
// connect to Device Manager
if (dm_lid)
rc = if3_lid_mngr_cnx(fe_config.hca,fe_config.port,DM_IF3_MCLASS,dm_lid,&fd_dm);
else
rc = if3_sid_mngr_cnx(fe_config.hca,fe_config.port,DM_SERVICE_NAME,DM_SERVICE_ID,
DM_IF3_MCLASS,&fd_dm);
if (rc != VSTATUS_OK) {
IB_LOG_INFINI_INFORC("Failed to open Device Manager, wil try later. rc:", rc);
fd_dm = INVALID_HANDLE;
}
#endif
}
IB_EXIT(func, 0);
return(rc);
}
void bitset_free(bitset_t *bitset) {
Status_t status;
if (bitset->bits_m) {
if ((status = vs_pool_free(bitset->pool_m, bitset->bits_m)) != VSTATUS_OK) {
IB_LOG_ERRORRC("can't free allocated space for bitset, rc:", status);
}
}
memset(bitset, 0, sizeof(bitset_t));
}
Status_t
state_event_timeout(void) {
uint64_t now;
Status_t status;
static uint64_t lastStandbyCheck=0;
IB_ENTER(__func__, 0, 0, 0, 0);
//
// If this is the STANDBY state, then we need to heartbeat the Master.
// If this is the MASTER or DISCOVERING state, then we need to do a topology sweep.
//
// Note that sweeping in DISCOVERING is required to avoid deadlocks when we
// abort a sweep before transitioning out of DISCOVERING.
//
if (lastStandbyCheck == 0) {
(void)vs_time_get(&lastStandbyCheck);
}
if (sm_state == SM_STATE_STANDBY) {
#if defined(IB_STACK_OPENIB)
status = ib_refresh_devport();
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("cannot refresh sm pkeys rc:", status);
}
#endif
status = sm_check_Master();
if (status != VSTATUS_OK) {
/* Master in trouble, start discovery */
(void)sm_transition(SM_STATE_DISCOVERING);
sm_trigger_sweep(SM_SWEEP_REASON_MASTER_TROUBLE);
}
} else if (sm_state == SM_STATE_DISCOVERING || sm_state == SM_STATE_MASTER) {
(void)vs_time_get(&now);
if ((topology_wakeup_time != 0ull) && (now >= topology_wakeup_time)) {
topology_wakeup_time = 0ull;
lastStandbyCheck = now;
sm_trigger_sweep(SM_SWEEP_REASON_SCHEDULED);
} else if (sm_config.monitor_standby_enable &&
topology_wakeup_time != 0ull &&
sm_state == SM_STATE_MASTER &&
((now - lastStandbyCheck) > sm_masterCheckInterval * sm_config.master_ping_max_fail)) {
sm_dbsync_standbyCheck();
lastStandbyCheck = now;
}
} else if (sm_state == SM_STATE_NOTACTIVE) {
// Notify master sm that we are going away. Most HSMs and the X switches
// don't currently support capabilityMask trap indicating IsSm off.
status = sm_check_Master();
}
IB_EXIT(__func__, VSTATUS_OK);
return(VSTATUS_OK);
}
Status_t
sa_cntxt_release( sa_cntxt_t* sa_cntxt )
{
int bucket;
Status_t status;
IB_ENTER( "sa_cntxt_release", sa_cntxt, 0, 0, 0 );
if (!sa_cntxt) {
IB_LOG_ERROR0("sa_cntxt_release: SA context is NULL!!!");
return VSTATUS_OK ;
}
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_release: Failed to lock SA context rc:", status);
} else {
if( sa_cntxt->ref == 0 ) {
IB_LOG_INFINI_INFO0("sa_cntxt_release: reference count is already zero");
} else {
--sa_cntxt->ref;
if( sa_cntxt->ref == 0 ) {
// This context needs to be removed from hash
if( sa_cntxt->hashed ) {
bucket = sa_cntxt->lid % SA_CNTXT_HASH_TABLE_DEPTH;
sa_cntxt_delete_entry( sa_hash[ bucket ], sa_cntxt );
}
sa_cntxt->prev = sa_cntxt->next = NULL ;
sa_cntxt_retire( sa_cntxt );
}
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_release: Failed to unlock SA context rc:", status);
}
}
IB_EXIT( "sa_cntxt_release", VSTATUS_OK );
return VSTATUS_OK ;
}
//
// find context entry matching input mad
//
sa_cntxt_t *sa_cntxt_find( Mai_t* mad ) {
uint64_t now ;
int bucket;
Status_t status;
sa_cntxt_t* sa_cntxt;
sa_cntxt_t* req_cntxt = NULL;
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_find: Failed to lock SA context rc:", status);
} else {
vs_time_get( &now );
// Search the hash table for the context
bucket = mad->addrInfo.slid % SA_CNTXT_HASH_TABLE_DEPTH ;
sa_cntxt = sa_hash[ bucket ];
while( sa_cntxt ) {
if( sa_cntxt->lid == mad->addrInfo.slid && sa_cntxt->tid == mad->base.tid ) {
req_cntxt = sa_cntxt ;
sa_cntxt = sa_cntxt->next ;
req_cntxt->tstamp = now ;
break ;
} else {
sa_cntxt = sa_cntxt->next;
}
}
// Table is sorted with respect to timeout
if( req_cntxt ) {
// Touch current context
req_cntxt->tstamp = now ;
sa_cntxt_delete_entry( sa_hash[ bucket ], req_cntxt );
sa_cntxt_insert_head( sa_hash[ bucket ], req_cntxt );
// A get on an existing context reserves it
req_cntxt->ref ++ ;
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_find: Failed to unlock SA context rc:", status);
}
}
IB_EXIT("sa_cntxt_find", req_cntxt );
return req_cntxt;
} // end cntxt_find
int
mai_dc_read(IBhandle_t fd)
{
int rc;
IB_ENTER(__func__, fd, 0, 0, 0);
if (gMAI_USE_DEDICATED_DCTHREAD) {
if (gMAI_DCTHREAD_HANDLE != MAI_INVALID) {
IB_LOG_WARNX("down call thread handle is busy handle:",
gMAI_DCTHREAD_HANDLE);
IB_EXIT(__func__, VSTATUS_BUSY);
return VSTATUS_BUSY;
}
/*
* Remember the handle associated with the thread
*/
gMAI_DCTHREAD_HANDLE = fd;
/* release the waiting initiator thread */
cs_vsema(&gMAI_DCTHREAD_SEMA);
}
mai_dc_read_exit = 0;
if (mai_dc_read_exit == 1) {
IB_LOG_VERBOSE0("Mai DC Task exiting OK.");
return VSTATUS_OK;
}
do {
rc = mai_recv(fd, &mad, DC_THREAD_TIMEOUT);
} while (rc == VSTATUS_TIMEOUT);
if (rc != VSTATUS_TIMEOUT) {
if (rc == VSTATUS_CONNECT_GONE) {
IB_LOG_VERBOSERC("mai is shutting down rc:", rc);
} else {
IB_LOG_ERRORRC("bad status rc:", rc);
}
}
if (gMAI_USE_DEDICATED_DCTHREAD) {
/*
* Reset the handle associated with the DC thread
*/
gMAI_DCTHREAD_HANDLE = MAI_INVALID;
}
IB_EXIT(__func__, rc);
return rc;
}
/**********************************************************************
*
* FUNCTION
* vs_pool_alloc
*
* DESCRIPTION
* Does fundamental common validation for vs_pool_alloc
* parameters and invokes environment specific implmentation.
*
* INPUTS
*
* OUTPUTS
* Status_t - On success VSTATUS_OK is returned, otherwise
* the cause of the error.
*
*
* HISTORY
*
* NAME DATE REMARKS
* DKJ 03/07/02 Initial creation of function.
**********************************************************************/
Status_t
vs_pool_alloc (Pool_t * handle, size_t length, void **loc)
{
Status_t rc;
IB_ENTER (function, (unint) handle, (uint32_t) length, (unint) loc,
(uint32_t) 0U);
if (handle == 0)
{
IB_LOG_ERROR0 ("handle is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (handle->magic != VSPOOL_MAGIC)
{
IB_LOG_ERRORX ("invalid handle magic:", handle->magic);
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (loc == 0)
{
IB_LOG_ERROR0 ("loc is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (length == (size_t) 0x00U)
{
IB_LOG_ERROR0 ("length is zero");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
// TBD - do we need to lock here?
rc = vs_lock (&handle->lock);
if (rc != VSTATUS_OK)
{
IB_LOG_ERRORRC("vs_lock failed rc:", rc);
IB_EXIT (function, VSTATUS_NXIO);
return VSTATUS_NXIO;
}
rc = vs_implpool_alloc (handle, length, loc);
(void) vs_unlock (&handle->lock);
IB_EXIT (function, rc);
return rc;
}
Status_t
sm_fsm_notactive(Mai_t *maip, char *nodename)
{
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)STL_GET_SMP_DATA(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_STANDBY: // C14-54.1.1
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received request to transition to STANDBY from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
new_state = SM_STATE_STANDBY;
break;
case SM_AMOD_DISABLE:
IB_LOG_WARN_FMT(__func__,
"[%s] SM received request to transition to NOTACTIVE from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
new_state = SM_STATE_NOTACTIVE;
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid transition request %s (%u) from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), maip->base.amod,
nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)STL_GET_SMP_DATA(maip));
status = mai_stl_reply(fd_async, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_notactive - bad mai_reply rc:", status);
}
/* make appropriate transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE)
(void)sm_transition(new_state);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
/**********************************************************************
*
* FUNCTION
* vs_pool_delete
*
* DESCRIPTION
* Does fundamental common validation for vs_pool_delete
* parameters and invokes environment specific implmentation.
*
* INPUTS
*
* OUTPUTS
* Status_t - On success VSTATUS_OK is returned, otherwise
* the cause of the error.
*
*
* HISTORY
*
* NAME DATE REMARKS
* DKJ 03/07/02 Initial creation of function.
**********************************************************************/
Status_t
vs_pool_delete (Pool_t * handle)
{
Status_t rc;
IB_ENTER (function, (unint) handle, (uint32_t) 0U, (uint32_t) 0U,
(uint32_t) 0U);
if (handle == 0)
{
IB_LOG_ERROR0 ("handle is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (handle->magic != VSPOOL_MAGIC)
{
IB_LOG_ERRORX ("invalid handle magic:", handle->magic);
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
rc = vs_lock (&handle->lock);
if (rc != VSTATUS_OK)
{
IB_LOG_ERRORRC("vs_lock failed rc:", rc);
IB_EXIT (function, VSTATUS_NXIO);
return VSTATUS_NXIO;
}
rc = vs_implpool_delete (handle);
if (rc == VSTATUS_OK)
{
handle->magic = ~VSPOOL_MAGIC;
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
}
else
{
(void) vs_unlock (&handle->lock);
}
(void) memset (handle, 0, sizeof (Pool_t));
IB_EXIT (function, rc);
return rc;
}
/**********************************************************************
*
* FUNCTION
* vs_pool_size
*
* DESCRIPTION
* Does fundamental common validation for vs_pool_size
* parameters and invokes environment specific implmentation.
*
* INPUTS
*
* OUTPUTS
* number of bytes allocated from pool.
**********************************************************************/
Status_t
vs_pool_size (Pool_t * handle, uint64_t *numBytesAlloc)
{
Status_t rc;
IB_ENTER (function, (unint) handle, 0, 0, 0);
if (handle == 0)
{
IB_LOG_ERROR0 ("handle is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (handle->magic != VSPOOL_MAGIC)
{
IB_LOG_ERRORX ("invalid handle magic:", handle->magic);
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (numBytesAlloc == 0)
{
IB_LOG_ERROR0 ("numBytesAlloc is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
// TBD - do we need to lock here?
rc = vs_lock (&handle->lock);
if (rc != VSTATUS_OK)
{
IB_LOG_ERRORRC("vs_lock failed rc:", rc);
*numBytesAlloc = 0;
IB_EXIT (function, VSTATUS_NXIO);
return VSTATUS_NXIO;
}
rc = vs_implpool_size (handle, numBytesAlloc);
(void) vs_unlock (&handle->lock);
IB_EXIT (function, rc);
return rc;
}
/**********************************************************************
*
* FUNCTION
* vs_pool_free
*
* DESCRIPTION
* Does fundamental common validation for vs_pool_free
* parameters and invokes environment specific implmentation.
*
* INPUTS
*
* OUTPUTS
* Status_t - On success VSTATUS_OK is returned, otherwise
* the cause of the error.
*
*
* HISTORY
*
* NAME DATE REMARKS
* DKJ 03/07/02 Initial creation of function.
**********************************************************************/
Status_t
vs_pool_free (Pool_t * handle, void *loc)
{
Status_t rc;
IB_ENTER (function, (unint) handle, (unint) loc, (uint32_t) 0U,
(uint32_t) 0U);
if (handle == 0)
{
IB_LOG_ERROR0 ("handle is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (handle->magic != VSPOOL_MAGIC)
{
IB_LOG_ERRORX ("invalid handle magic:", handle->magic);
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (loc == 0)
{
IB_LOG_ERROR0 ("loc is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
rc = vs_lock (&handle->lock);
if (rc != VSTATUS_OK)
{
IB_LOG_ERRORRC("vs_lock failed rc:", rc);
IB_EXIT (function, VSTATUS_NXIO);
return VSTATUS_NXIO;
}
rc = vs_implpool_free (handle, loc);
(void) vs_unlock (&handle->lock);
IB_EXIT (function, rc);
return rc;
}
void fe_shutdown(void)
{
Status_t rc = SUCCESS;
NetError nerr;
IB_ENTER(__func__, 0, 0, 0, 0);
/* Close Performance Manager Connecttion */
if (fd_pm != INVALID_HANDLE) {
if3_close_mngr_cnx(fd_pm);
} else {
/* Complete termination of the RMPP connection. */
(void)if3_close_mngr_rmpp_cnx(fd_pm);
}
/* Close Device Manager Connecttion */
if (fd_dm != INVALID_HANDLE) {
if3_close_mngr_cnx(fd_dm);
}
/* Deregister FE Manager and Unsubscribe from SA */
if (fdsa != INVALID_HANDLE) {
fe_if3_unsubscribe_sa(TRUE);
if3_deregister_fe(fdsa);
} else {
/* Complete termination of the RMPP connection. */
(void)if3_close_mngr_rmpp_cnx(fdsa);
}
/* Shutdown the net and free the pool */
nerr = fe_net_shutdown();
if (nerr != NET_NO_ERROR) {
IB_LOG_ERROR("Error shutting down libnet. nerr:", nerr);
}
rc = vs_pool_delete(&fe_pool);
if (rc != VSTATUS_OK) {
IB_LOG_ERRORRC("Error destroying fe memory pool. rc:", rc);
}
IB_EXIT(__func__, 0);
return;
}
int bitset_init(Pool_t* pool, bitset_t *bitset, size_t nbits) {
Status_t status;
bitset->pool_m = pool;
bitset->bits_m = NULL;
bitset->nset_m = 0;
bitset->nbits_m = nbits;
bitset->nwords_m = (nbits/32);
if (nbits%32) {
bitset->nwords_m++;
}
status = vs_pool_alloc(pool, sizeof(uint32_t)*bitset->nwords_m, (void *)&bitset->bits_m);
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("can't allocate space for bitset, rc:", status);
memset(bitset, 0, sizeof(bitset_t));
return 0;
}
memset(bitset->bits_m, 0, sizeof(uint32_t)*bitset->nwords_m);
return 1;
}
void
sa_main_reader(uint32_t argc, uint8_t ** argv) {
Status_t status;
Mai_t in_mad;
Filter_t filter;
sa_cntxt_t *sa_cntxt=NULL;
uint64_t now, delta, max_delta;
int tries=0, retry=0;
uint64_t reqTimeToLive=0;
SAContextGet_t cntxGetStatus=0;
int numContextBusy=0;
IB_ENTER("sa_main_reader", 0, 0, 0, 0);
sa_main_reader_exit = 0;
/*
* Create the SubnAdm(*) MAD filter for the SA thread.
*/
SA_Filter_Init(&filter);
filter.value.mclass = MAD_CV_SUBN_ADM;
filter.mask.mclass = 0xff;
filter.value.method = 0x00;
filter.mask.method = 0x80;
filter.mai_filter_check_packet = sa_reader_filter;
MAI_SET_FILTER_NAME (&filter, "SA Reader");
if (mai_filter_create(fd_sa, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("sa_main_reader: can't create SubnAdm(*) filter");
(void)vs_thread_exit(&sm_threads[SM_THREAD_SA_READER].handle);
}
timeMftLastUpdated = 0;
/*
* calculate request time to live on queue
* ~ 3.2secs for defaults: sa_packetLifetime=18 and sa_respTimeValue=18
*/
reqTimeToLive = 4ull * ( (2*(1 << sm_config.sa_packet_lifetime_n2)) + (1 << sm_config.sa_resp_time_n2) );
while (1) {
status = mai_recv(fd_sa, &in_mad, VTIMER_1S/4);
if (sa_main_reader_exit == 1){
#ifdef __VXWORKS__
ESM_LOG_ESMINFO("sa_main_reader: exiting OK.", 0);
#endif
break;
}
/* don't process messages if not master SM or still doing first sweep */
if ((sm_state != SM_STATE_MASTER) ||
(topology_passcount < 1)) {
continue;
}
/*
* If the mai layer shuts down we end up in this infinite loop here.
* This may happen on initialization
*/
if( status != VSTATUS_OK ){
if (status != VSTATUS_TIMEOUT)
IB_LOG_ERRORRC("sa_main_reader: error on mai_recv rc:", status);
} else {
/*
* Drop new requests that have been sitting on SA reader queue for too long
*/
if (in_mad.intime) {
/* PR 110586 - On some RHEL 5 systems, we've seen weird issues with gettimeofday() [used by vs_time_get()]
* where once in a while the time difference calculated from successive calls to gettimeofday()
* results in a negative value. Due to this, we might actually consider a request stale even if
* its not. Work around this by making calls to gettimeofday() till it returns us some
* sane values. Just to be extra cautious, bound the retries so that we don't get stuck in the loop.
*/
tries = 0;
/* Along with negative values also check for unreasonably high values of delta*/
max_delta = 30*reqTimeToLive;
do {
vs_time_get( &now );
delta = now - in_mad.intime;
tries++;
if ((now < in_mad.intime) || (delta > max_delta)) {
vs_thread_sleep(1);
retry = 1;
} else {
retry = 0;
}
} while (retry && tries < 20);
if (delta > reqTimeToLive) {
INCREMENT_COUNTER(smCounterSaDroppedRequests);
if (smDebugPerf || saDebugPerf) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"Dropping stale %s[%s] request from LID[0x%x], TID="FMT_U64"; On queue for %d.%d seconds.",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid), in_mad.addrInfo.slid,
in_mad.base.tid, (int)(delta/1000000), (int)((delta - delta/1000000*1000000))/1000);
}
/* drop the request without returning a response; sender will retry */
continue;
}
}
/*
* get a context to process request; sa_cntxt can be:
* 1. NULL if resources are scarce
* 2. NULL if request is dup of existing request
* 3. in progress getMulti request context
* 4. New context for a brand new request
*/
cntxGetStatus = sa_cntxt_get( &in_mad, (void *)&sa_cntxt );
if (cntxGetStatus == ContextAllocated) {
/* process the new request */
sa_process_mad( &in_mad, sa_cntxt );
/*
* This may not necessarily release context based on if someone else has reserved it
*/
if(sa_cntxt) sa_cntxt_release( sa_cntxt );
} else if (cntxGetStatus == ContextExist) {
INCREMENT_COUNTER(smCounterSaDuplicateRequests);
/* this is a duplicate request */
if (saDebugPerf || saDebugRmpp) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"SA_READER received duplicate %s[%s] from LID [0x%x] with TID ["FMT_U64"] ",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid),in_mad.addrInfo.slid, in_mad.base.tid);
}
} else if (cntxGetStatus == ContextNotAvailable) {
INCREMENT_COUNTER(smCounterSaContextNotAvailable);
/* we are swamped, return BUSY to caller */
if (saDebugPerf || saDebugRmpp) { /* log msg before send changes method and lids */
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"NO CONTEXT AVAILABLE, returning MAD_STATUS_BUSY to %s[%s] request from LID [0x%x], TID ["FMT_U64"]!",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid), in_mad.addrInfo.slid, in_mad.base.tid);
}
in_mad.base.status = MAD_STATUS_BUSY;
sa_send_reply( &in_mad, sa_cntxt );
if ((++numContextBusy % sa_max_cntxt) == 0) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"Had to drop %d SA requests since start due to no available contexts",
numContextBusy);
}
} else if (cntxGetStatus == ContextExistGetMulti) {
/* continue processing the getMulti request */
sa_process_getmulti( &in_mad, sa_cntxt );
if(sa_cntxt) sa_cntxt_release( sa_cntxt );
} else {
IB_LOG_WARN("sa_main_reader: Invalid sa_cntxt_get return code:", cntxGetStatus);
}
}
/*
* signal sm_top to reprogram the MFTs
* Wait one second to allow mcmember requests to accumulate before asking
*/
vs_time_get( &now );
if (sa_mft_reprog && timeMftLastUpdated == 0) {
timeMftLastUpdated = now;
} else if (sa_mft_reprog && (now - timeMftLastUpdated) > VTIMER_1S) {
topology_wakeup_time = 0ull;
if ((status = vs_lock(&sa_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_main_reader: Failed to lock sa_lock rc:", status);
} else {
sm_McGroups_Need_Prog = 1; /* tells Topoloy thread that MFT reprogramming is needed */
(void)vs_unlock(&sa_lock);
}
sm_trigger_sweep(SM_SWEEP_REASON_MCMEMBER);
/* clear the indicators */
timeMftLastUpdated = 0;
sa_mft_reprog = 0;
}
}
/* cleanup before exit, but allow some time for the other threads to flush out first */
(void)vs_thread_sleep(VTIMER_1S);
(void)sa_SubscriberDelete();
(void)sa_ServiceRecDelete();
(void)sa_McGroupDelete();
if (mai_filter_delete(fd_sa, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("sa_main_reader: can't delete SubnAdm(*) filter");
}
//IB_LOG_INFINI_INFO0("sa_main_reader thread: Exiting OK");
}
Status_t
sm_fsm_standby(Mai_t *maip, char *nodename)
{
int i;
uint8_t ipath[64];
uint8_t *rpath;
uint8_t *path;
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
STL_SM_INFO smInfoCopy;
Lid_t slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
Lid_t dlid=maip->addrInfo.dlid; // original destination lid(us); mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, 0, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)STL_GET_SMP_DATA(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_DISCOVER: // C14-48
new_state = SM_STATE_DISCOVERING;
break;
case SM_AMOD_DISABLE: // C14-19
new_state = SM_STATE_NOTACTIVE;
break;
case SM_AMOD_HANDOVER: // C14-50
new_state = SM_STATE_MASTER;
break;
case SM_AMOD_ACKNOWLEDGE: // from previous HANDOVER
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
break;
}
if (maip->base.status > MAD_STATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid AMOD[%d] from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), maip->base.amod, nodename,
maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"SM in [%s] state after processing %s smInfo control from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
(new_state < 0 ? sm_getStateText(sm_smInfo.u.s.SMStateCurrent):sm_getStateText(new_state)), getAmod(maip->base.amod),
nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)STL_GET_SMP_DATA(maip));
status = mai_stl_reply(fd_async, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_standby - bad mai_reply rc:", status);
}
(void)vs_lock(&new_topology_lock);
/* make appropriate none master transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE && new_state != SM_STATE_MASTER) {
(void)sm_transition(new_state);
if (new_state == SM_STATE_DISCOVERING) {
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION); // wakeup the topology thread to start sweep
}
}
/*
* If this was a HANDOVER, we need to ACK back.
*/
if (maip->base.amod == SM_AMOD_HANDOVER) {
maip->addrInfo.destqp &= 0x00ffffff;
if (maip->base.mclass == MAD_CV_SUBN_LR) {
path = NULL;
sm_topop->dlid = slid; // this is original sender of SmInfo
sm_topop->slid = dlid; // this is us
IB_LOG_INFINI_INFO_FMT(__func__, "sending LR HANDOVER ACK to node %s, Lid [0x%x], portguid "FMT_U64,
nodename, slid, theirSmInfo.PortGUID);
} else {
DRStlSmp_t *drsmp = (DRStlSmp_t *)maip->data;
path = ipath;
memset((void *)ipath, 0, 64);
ipath[0] = maip->base.hopCount;
rpath = drsmp->RetPath;
for (i = 1; i <= maip->base.hopCount; i++) {
ipath[i] = rpath[maip->base.hopCount + 1 - i];
}
IB_LOG_INFINI_INFO_FMT(__func__, "sending DR HANDOVER ACK to node %s, portguid "FMT_U64,
nodename, theirSmInfo.PortGUID);
}
smInfoCopy = sm_smInfo;
status = SM_Set_SMInfo(fd_sminfo, SM_AMOD_ACKNOWLEDGE, path, &smInfoCopy, sm_config.mkey);
if (status != VSTATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM did not receive response to Handover Acknowledgement from SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(new_state), nodename, slid, theirSmInfo.PortGUID);
}
/* make transition to MASTER state */
(void)sm_transition(new_state);
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION);
}
(void)vs_unlock(&new_topology_lock);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_fsm_master(Mai_t *maip, char *nodename)
{
Status_t status;
long new_state=-1;
STL_SM_INFO theirSmInfo;
uint8_t ipath[64];
uint8_t *rpath;
uint8_t *path;
int i, wakeTpThread=0;
STL_SM_INFO smInfoCopy;
Lid_t slid=maip->addrInfo.slid; // original source lid; mai_reply swaps slid-dlid
Lid_t dlid=maip->addrInfo.dlid; // original destination lid(us); mai_reply swaps slid-dlid
IB_ENTER(__func__, maip->base.amod, sm_state, 0, 0);
BSWAPCOPY_STL_SM_INFO((STL_SM_INFO *)STL_GET_SMP_DATA(maip), &theirSmInfo);
switch (maip->base.amod) {
case SM_AMOD_HANDOVER: // C14-61
new_state = SM_STATE_MASTER;
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received '%s' from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
case SM_AMOD_ACKNOWLEDGE:
if (theirSmInfo.u.s.SMStateCurrent < SM_STATE_STANDBY) { // C14-38.1.1 almost
maip->base.status = MAD_STATUS_BAD_FIELD;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid Handover Ack from remote SM %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64"; remote not in STANDBY state [%s]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid, sm_getStateText(theirSmInfo.u.s.SMStateCurrent));
} else {
new_state = SM_STATE_STANDBY;
wakeTpThread = 1;
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM received '%s' from SM node %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), nodename, maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
}
break;
default:
maip->base.status = MAD_STATUS_BAD_ATTR;
IB_LOG_WARN_FMT(__func__,
"[%s] SM received invalid MASTER transition [%s] from remote [%s] SM %s, LID [0x%x], portguid ["FMT_U64"], TID="FMT_U64,
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), getAmod(maip->base.amod), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename,
maip->addrInfo.slid, theirSmInfo.PortGUID, maip->base.tid);
break;
}
/*
* Reply to this Set(SMInfo).
*/
sm_smInfo.ActCount++;
BSWAPCOPY_STL_SM_INFO(&sm_smInfo, (STL_SM_INFO *)STL_GET_SMP_DATA(maip));
status = mai_stl_reply(fd_async, maip, sizeof(STL_SM_INFO));
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sm_fsm_master - bad mai_reply rc:", status);
}
(void)vs_lock(&new_topology_lock);
/* make appropriate transition if necessary */
if (new_state >= SM_STATE_NOTACTIVE && new_state != sm_state)
(void)sm_transition(new_state);
if (wakeTpThread) {
/* wakeup the topology thread to clean up SA tables */
sm_trigger_sweep(SM_SWEEP_REASON_HANDOFF);
}
/*
* If this was a HANDOVER, we need to ACK back.
*/
if (maip->base.amod == SM_AMOD_HANDOVER) {
maip->addrInfo.destqp &= 0x00ffffff;
if (maip->base.mclass == MAD_CV_SUBN_LR) {
path = NULL;
sm_topop->dlid = slid; // this is original sender of SmInfo
sm_topop->slid = dlid; // this is us
IB_LOG_INFINI_INFO_FMT(__func__, "sending LR HANDOVER ACK to node %s, Lid [0x%x], portguid "FMT_U64,
nodename, slid, theirSmInfo.PortGUID);
} else {
DRStlSmp_t *drsmp = (DRStlSmp_t *)maip->data;
path = ipath;
memset((void *)ipath, 0, 64);
ipath[0] = maip->base.hopCount;
rpath = drsmp->RetPath;
for (i = 1; i <= maip->base.hopCount; i++) {
ipath[i] = rpath[maip->base.hopCount + 1 - i];
}
IB_LOG_INFINI_INFO_FMT(__func__, "sending DR HANDOVER ACK to node %s, portguid "FMT_U64,
nodename, theirSmInfo.PortGUID);
}
smInfoCopy = sm_smInfo;
status = SM_Set_SMInfo(fd_sminfo, SM_AMOD_ACKNOWLEDGE, path, &smInfoCopy, sm_config.mkey);
if (status != VSTATUS_OK) {
IB_LOG_WARN_FMT(__func__,
"[%s] SM did not receive response to Handover Acknowledgement from [%s] SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), sm_getStateText(theirSmInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID);
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"[%s] SM successfully acknowleded Handover from remote SM node %s, LID [0x%x], portguid ["FMT_U64"]",
sm_getStateText(sm_smInfo.u.s.SMStateCurrent), nodename, slid, theirSmInfo.PortGUID);
}
/* make transition to MASTER state */
(void)sm_transition(new_state);
sm_trigger_sweep(SM_SWEEP_REASON_STATE_TRANSITION); // wakeup the topology thread
}
(void)vs_unlock(&new_topology_lock);
IB_EXIT(__func__, 0);
return(VSTATUS_OK);
}
Status_t
sm_check_Master() {
Status_t status;
STL_SM_INFO theirSmInfo;
STL_PORT_INFO portInfo;
uint8_t path[64];
static uint32_t fsmCheckMasterFailed=0; // count of fails to check master
static uint32_t fsmMultMaxFail = 1; // multiplier for sm_config.master_ping_max_fail
IB_ENTER(__func__, 0, 0, 0, 0);
(void)memset((void *)path, 0, 64);
if ((status = SM_Get_PortInfo(fd_sminfo, 1<<24, path, &portInfo)) != VSTATUS_OK) {
IB_LOG_ERRORRC("failed to get master SM Lid from my PortInfo, rc:", status);
// having a local problem
// reset count, must be healthy before we can consider becoming master
fsmCheckMasterFailed = 0;
goto stay_standby;
}
if (portInfo.LID == 0 || portInfo.MasterSMLID == 0
|| portInfo.PortStates.s.PortState != IB_PORT_ACTIVE
) {
if (smDebugPerf) {
if (portInfo.PortStates.s.PortState != IB_PORT_ACTIVE) {
IB_LOG_INFINI_INFO("our portInfo indicates state not active; portState=", (int)portInfo.PortStates.s.PortState);
} else
if (portInfo.MasterSMLID == 0) {
IB_LOG_INFINI_INFOX("our portInfo smLid is not set yet; Lid=", portInfo.LID);
}
}
// stay in standby until link comes up
if (portInfo.PortStates.s.PortState > IB_PORT_DOWN) {
IB_LOG_WARN0("Switching to DISCOVERY state; Local port uninitialized");
goto discovering;
}
goto stay_standby;
}
// make sure we aren't trying to talk to ourself during the handover window
if (portInfo.LID == portInfo.MasterSMLID) {
// we're talking to ourself. if a master SM doesn't come along and
// reprogram our SM LID in the timeout period, attempt a discovery.
// Since for large fabrics, the master can take some time to program
// our LID, use a higher upper limit for failure count to give the
// master enough time to program our SM LID.
fsmMultMaxFail = 2;
if (++fsmCheckMasterFailed >= fsmMultMaxFail * sm_config.master_ping_max_fail) {
IB_LOG_WARN0("Switching to DISCOVERY state; Timed out waiting for SM LID to get reprogrammed");
goto discovering;
}
if (smDebugPerf) {
IB_LOG_INFINI_INFOX("our portInfo smLid is not set yet; smLid=", portInfo.MasterSMLID);
}
goto stay_standby; // not yet at threshold
}
sm_topop->slid = portInfo.LID;
sm_topop->dlid = portInfo.MasterSMLID;
if ((status = SM_Get_SMInfo(fd_sminfo, 0, NULL, &theirSmInfo)) != VSTATUS_OK) {
if (++fsmCheckMasterFailed >= fsmMultMaxFail * sm_config.master_ping_max_fail) {
IB_LOG_WARNX("Switching to DISCOVERY state; Failed to get SmInfo from master SM at LID:", portInfo.MasterSMLID);
goto discovering;
} else {
IB_LOG_INFINI_INFO_FMT(__func__,
"failed to get SmInfo from master SM at LID[0x%X], retry count=%d",
portInfo.MasterSMLID, fsmCheckMasterFailed);
goto stay_standby; // not yet at threshold
}
}
sm_saw_another_sm = TRUE;
/*
* PR 105313 - restart results in 2 standby SMs
* Must check the state we get back to make sure master is still master
*/
if (theirSmInfo.u.s.SMStateCurrent != SM_STATE_MASTER) {
IB_LOG_WARN_FMT(__func__,
"SmInfo from SM at SMLID[0x%X] indicates SM is no longer master, switching to DISCOVERY state",
portInfo.MasterSMLID);
goto discovering;
}
// all OK, save data about master, reset fail count and threshold
sm_topop->sm_count = theirSmInfo.ActCount;
sm_topop->sm_key = theirSmInfo.SM_Key;
fsmCheckMasterFailed = 0;
fsmMultMaxFail = 1;
if (smDebugPerf) {
IB_LOG_INFINI_INFO_FMT(__func__,
"Master SM["FMT_U64"] at LID=0x%x has priority[%d] and smKey ["FMT_U64"]",
theirSmInfo.PortGUID, portInfo.MasterSMLID, theirSmInfo.u.s.Priority, theirSmInfo.SM_Key);
}
stay_standby:
status = VSTATUS_OK;
done:
IB_EXIT(__func__, status);
return(status);
discovering:
// reset count and threshold in case stay standby
fsmCheckMasterFailed = 0;
fsmMultMaxFail = 1;
status = VSTATUS_BAD;
goto done;
}
/*
* Very simple hashing implemented. This function is modular and can be
* changed to use any algorithm, if hashing turns out to be bad.
* Returns new context for new requests, existing context for in progress
* getMulti requests and NULL for duplicate requests
*/
SAContextGet_t
sa_cntxt_get( Mai_t* mad, void **context)
{
uint64_t now ;
int bucket;
Status_t status;
sa_cntxt_t* sa_cntxt;
SAContextGet_t getStatus=0;
sa_cntxt_t* req_cntxt = NULL;
IB_ENTER( "sa_cntxt_get", mad, 0, 0, 0 );
if ((status = vs_lock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_get: Failed to lock SA context rc:", status);
} else {
vs_time_get( &now );
// Search the hash table for the context
bucket = mad->addrInfo.slid % SA_CNTXT_HASH_TABLE_DEPTH ;
sa_cntxt = sa_hash[ bucket ];
while( sa_cntxt ) {
if( sa_cntxt->lid == mad->addrInfo.slid &&
sa_cntxt->tid == mad->base.tid ) {
req_cntxt = sa_cntxt ;
break ;
} else {
sa_cntxt = sa_cntxt->next;
}
}
if( req_cntxt ) {
if ( req_cntxt->method == SA_CM_GETMULTI && req_cntxt->reqInProg ) {
/* In progress getMulti request. Touch and reserve it. */
getStatus = ContextExistGetMulti;
req_cntxt->tstamp = now ;
sa_cntxt_delete_entry( sa_hash[ bucket ], req_cntxt );
sa_cntxt_insert_head( sa_hash[ bucket ], req_cntxt );
/* A get on an existing context reserves it */
req_cntxt->ref ++ ;
} else {
/* dup of an existing request */
getStatus = ContextExist;
req_cntxt = NULL;
}
} else {
/* Allocate a new context and set appropriate status */
req_cntxt = sa_cntxt_free_list ;
if( req_cntxt ) {
getStatus = ContextAllocated;
sa_cntxt_delete_entry( sa_cntxt_free_list, req_cntxt );
req_cntxt->ref = 1; /* set ref count to 1 */
INCR_SA_CNTXT_NALLOC();
DECR_SA_CNTXT_NFREE();
req_cntxt->lid = mad->addrInfo.slid ;
req_cntxt->tid = mad->base.tid ;
req_cntxt->method = mad->base.method ;
req_cntxt->tstamp = now;
} else {
/* out of context */
getStatus = ContextNotAvailable;
}
}
if ((status = vs_unlock(&sa_cntxt_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_cntxt_get: Failed to unlock SA context rc:", status);
}
}
/* return new context or existing getmulti context or NULL */
*context = req_cntxt;
IB_EXIT("sa_cntxt_get", getStatus );
return getStatus;
}
int
sa_main(void) {
Status_t status;
int i;
IB_ENTER("sa_main", 0, 0, 0, 0);
if (sa_SubscriberInit() != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: Can't allocate Subscriber hash table");
return 1;
}
if (sa_ServiceRecInit() != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: Can't allocate Service Record hash table");
return 1;
}
//
// Zero context hash table
//
memset( sa_hash, 0, sizeof( sa_hash ));
sa_cntxt_pool = NULL;
IB_LOG_VERBOSE("sa_main: Allocating SA context pool with num entries=", sa_max_cntxt);
status = vs_pool_alloc(&sm_pool, sizeof(sa_cntxt_t) * sa_max_cntxt, (void *)&sa_cntxt_pool);
if (status != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: Can't allocate SA context pool");
return 2;
}
memset( sa_cntxt_pool, 0, sizeof( sa_cntxt_t ) * sa_max_cntxt);
sa_cntxt_free_list = NULL ;
for( i = 0 ; i < sa_max_cntxt ; ++i ) {
sa_cntxt_insert_head( sa_cntxt_free_list, &sa_cntxt_pool[i] );
}
sa_cntxt_nfree = sa_max_cntxt;
sa_cntxt_nalloc = 0;
// initialize SA context lock
status = vs_lock_init(&sa_cntxt_lock, VLOCK_FREE, VLOCK_THREAD);
if (status != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_main: can't initialize SA context pool lock rc:", status);
return 3;
}
//
// Allocate the SA storage pool.
//
status = vs_pool_alloc(&sm_pool, sa_data_length, (void*)&sa_data);
if (status != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: can't allocate sa data");
return 4;
}
//
// Fill in my ClassPortInfo_t and add it to the database.
//
(void)memset((void *)&saClassPortInfo, 0, sizeof(STL_CLASS_PORT_INFO));
saClassPortInfo.BaseVersion = STL_BASE_VERSION; //MAD_BVERSION;
saClassPortInfo.ClassVersion = STL_SA_CLASS_VERSION; //SA_MAD_CVERSION;
saClassPortInfo.CapMask =
STL_CLASS_PORT_CAPMASK_CM2 |
STL_SA_CAPABILITY_MULTICAST_SUPPORT |
STL_SA_CAPABILITY_PORTINFO_CAPMASK_MATCH |
STL_SA_CAPABILITY_PA_SERVICES_SUPPORT;
saClassPortInfo.u1.s.CapMask2 =
STL_SA_CAPABILITY2_QOS_SUPPORT |
STL_SA_CAPABILITY2_MFTTOP_SUPPORT |
STL_SA_CAPABILITY2_FULL_PORTINFO |
STL_SA_CAPABILITY2_EXT_SUPPORT;
saClassPortInfo.u1.s.RespTimeValue = sm_config.sa_resp_time_n2;
saClassPortInfo.u3.s.RedirectQP = 1;
saClassPortInfo.u5.s.TrapHopLimit = 0xff;
saClassPortInfo.u5.s.TrapQP = 1;
//
// Init Sa Groups table and Set up the default Multicast group if one is set.
//
status = sa_McGroupInit();
if (status != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: can't initialize SA McMember/Groups table lock");
return 5;
}
sa_SetDefBcGrp();
//
// Init SA caching
//
status = sa_cache_init();
if (status != VSTATUS_OK) {
IB_FATAL_ERROR("sa_main: can't initialize SA caching");
return 6;
}
return 0;
}
int
main(int argc, char* argv[])
{
int rc,i;
uint64_t tid;
IBhandle_t fd = -1,fh;
Mai_t mad, rmad;
uint8_t name[16];
/* Parse command line arguments */
if (argc > 1)
parse_cmd_line(argc,argv);
/* Initialize MAD */
memset(&rmad,0,sizeof(rmad));
/* Initialize MAD */
memset(&mad,0,sizeof(mad));
/* Initialize MAI subsystem */
mai_init();
rc=ib_init_devport(&ib_dev, &ib_port, NULL);
if (rc)
{
printf("ib_init_devport failed, %d\n",rc);
return rc;
}
/* Create MAI handle to used to communicate */
rc = mai_open(MAI_GSI_QP,ib_dev,ib_port,&fd);
if (rc)
{
printf("MAI_OPEN failed, %d\n",rc);
return rc;
}
/* Initialize MAD that will be sent */
if(oob)
mad.type = MAI_TYPE_INTERNAL;
else
mad.type = MAI_TYPE_EXTERNAL;
mad.base.method = method;
mad.base.mclass = mclass;
mad.base.aid = aid;
AddrInfo_Init(&mad, slid, dlid, 0, STL_DEFAULT_FM_PKEY, MAI_GSI_QP, MAI_GSI_QP, GSI_WELLKNOWN_QKEY);
mad.base.bversion = MAD_BVERSION;
mad.base.cversion = MAD_CVERSION;
mad.active = MAI_ACT_BASE | MAI_ACT_DATA | MAI_ACT_TYPE |
MAI_ACT_ADDRINFO;
/* Create filter to listen for response */
rc = mai_filter_method(fd,VFILTER_SHARE,MAI_TYPE_ANY, &fh, mclass,
RESPONSE);
if (rc)
{
printf("Cannot create filter to listen for responses \n");
return rc;
}
if(use_event)
{
sprintf((char *) name, "client");
rc = vs_event_create(&event, name, (Eventset_t) 0x00U);
if (rc)
{
IB_LOG_ERRORRC("vs_event_create failed rc:", rc);
return rc;
}
}
/* Loop which sends MADS to receiver */
for (i = 0; i < loop; i++)
{
/* Get a transaction ID for message we are about to send */
rc = mai_alloc_tid(fd, mclass, &tid);
/* Assign transaction ID to the MAD to be sent */
mad.base.tid = tid;
/* Send command to receiver */
rc = mai_send(fd,&mad);
if (rc)
{
printf("MAI send failed %d \n",rc);
return rc;
}
printf("MAD sent\n");
printf("Waiting for acknowledgement\n");
if(use_event)
{
Eventset_t events = (Eventset_t) 0U;
do{
rc = vs_event_wait(event.event_handle,
timeout,0x1,&events);
}while(rc == VSTATUS_AGAIN);
if(rc == VSTATUS_OK)
{
rc = mai_recv(fd,&rmad,MAI_RECV_NOWAIT);
}
printf("Event wait returned %d\n",rc);
}
else
{
/* Get acknowledgement */
rc = mai_recv(fd,&rmad,timeout);
}
if (rc != VSTATUS_OK)
{
printf("Acknowledgement not received \n");
return rc;
}
printf("Acknowledgement received\n");
printf("Passed Loop %d\n ",i);
//sleep(1);
}
/* Delete filter */
mai_filter_hdelete(fd,fh);
/* Close MAI channel */
rc = mai_close(fd);
if (rc)
{
printf("Close failed\n");
return rc;
}
printf("Test successful!!\n");
return rc;
}
/**********************************************************************
*
* FUNCTION
* vs_pool_create
*
* DESCRIPTION
* Does fundamental common validation for vs_pool_create
* parameters and invokes environment specific implmentation.
*
* INPUTS
*
* OUTPUTS
* Status_t - On success VSTATUS_OK is returned, otherwise
* the cause of the error.
*
*
* HISTORY
*
* NAME DATE REMARKS
* DKJ 03/07/02 Initial creation of function.
* DKJ 03/21/02 Updated prototype
**********************************************************************/
Status_t
vs_pool_create (Pool_t * handle, uint32_t options,
unsigned char * name, void *address, size_t size)
{
Status_t rc;
size_t namesize;
const uint32_t valid_options = 0;
IB_ENTER (function, (unint) handle, options, (unint) address,
(uint32_t) size);
if (handle == 0)
{
IB_LOG_ERROR0 ("handle is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
(void) memset (handle, 0, sizeof (Pool_t));
rc = vs_lock_init (&handle->lock, VLOCK_LOCKED, VLOCK_THREAD);
if (rc != VSTATUS_OK)
{
IB_LOG_ERRORRC("vs_lock_init failed rc:", rc);
IB_EXIT (function, VSTATUS_NOMEM);
return VSTATUS_NOMEM;
}
if (name == 0)
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR0 ("name is null");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
for (namesize = (size_t) 0U; namesize < sizeof (handle->name); namesize++)
{
handle->name[namesize] = name[namesize];
if (name[namesize] == (unsigned char) 0x00U)
{
break;
}
}
if (namesize >= sizeof (handle->name))
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR0 ("name doesn't contain a terminator");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if ((options & ~valid_options) != (uint32_t) 0x00U)
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR ("Invalid options specified: ", options);
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
handle->options = options;
if (size == (size_t) 0x00U)
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR0 ("Pool size is zero");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
#ifdef __VXWORKS__
/* cap size on VxWorks only */
if (size > (size_t) 0xFFFFFFF0U)
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR ("Pool size is too big: high:", (uint32_t) (size));
IB_LOG_ERROR ("Pool size is too big: low:", (uint32_t) (size));
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
#endif
handle->pagesize = vs_pool_page_size ();
if ((options & VMEM_PAGE) == VMEM_PAGE)
{
if (size < handle->pagesize)
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
IB_LOG_ERROR ("size is less than pagesize:", size);
IB_EXIT (function, VSTATUS_NOMEM);
return VSTATUS_NOMEM;
}
}
rc = vs_implpool_create (handle, options, name, address, size);
if (rc == VSTATUS_OK)
{
handle->magic = VSPOOL_MAGIC;
(void) vs_unlock (&handle->lock);
}
else
{
(void) vs_unlock (&handle->lock);
(void) vs_lock_delete (&handle->lock);
}
IB_EXIT (function, rc);
return rc;
}
Status_t
sa_SMInfoRecord_GetTable(Mai_t *maip, uint32_t *records) {
uint8_t *data;
uint32_t bytes;
STL_SA_MAD samad;
Status_t status;
SmRecp smrecp;
CS_HashTableItr_t itr;
STL_SMINFO_RECORD * record;
IB_ENTER("sa_SMInfoRecord_GetTable", maip, *records, 0, 0);
*records = 0;
data = sa_data;
bytes = Calculate_Padding(sizeof(STL_SMINFO_RECORD));
//
// Verify the size of the data received for the request
//
if ( maip->datasize-sizeof(STL_SA_MAD_HEADER) < sizeof(STL_SMINFO_RECORD) ) {
IB_LOG_ERROR_FMT("sa_SMInfoRecord_GetTable",
"invalid MAD length; size of STL_SMINFO_RECORD[%lu], datasize[%d]", sizeof(STL_SMINFO_RECORD), maip->datasize-sizeof(STL_SA_MAD_HEADER));
maip->base.status = MAD_STATUS_SA_REQ_INVALID;
IB_EXIT("sa_SMInfoRecord_GetTable", MAD_STATUS_SA_REQ_INVALID);
return (MAD_STATUS_SA_REQ_INVALID);
}
BSWAPCOPY_STL_SA_MAD((STL_SA_MAD*)maip->data, &samad, sizeof(STL_SMINFO_RECORD));
/*
* Create the template mask for the lookup.
*/
status = sa_create_template_mask(maip->base.aid, samad.header.mask);
if (status != VSTATUS_OK) {
IB_EXIT("sa_SMInfoRecord_GetTable", VSTATUS_OK);
return(VSTATUS_OK);
}
if ((status = vs_lock(&smRecords.smLock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_SMInfoRecord_GetTable: Can't lock SM Record table, rc:",
status);
} else if (cs_hashtable_count(smRecords.smMap) > 0) {
cs_hashtable_iterator(smRecords.smMap, &itr);
do {
smrecp = cs_hashtable_iterator_value(&itr);
// Support ElapsedTime...
// Work with the outbound copy (Do not modify the SMs db)
record = (STL_SMINFO_RECORD *)data;
memcpy(record, &smrecp->smInfoRec, sizeof(*record));
// If this SM is Master, then fill in the elapsed time.
// [Verify that the master is us... just in case]
if (record->SMInfo.u.s.SMStateCurrent == SM_STATE_MASTER) {
if (smrecp->portguid != sm_smInfo.PortGUID) {
IB_LOG_WARN64("sa_SMInfoRecord_GetTable: Master SM is not us:", smrecp->portguid);
}
record->SMInfo.ElapsedTime = (uint32_t)difftime(time(NULL),sm_masterStartTime);
} else if ( (record->SMInfo.u.s.SMStateCurrent == SM_STATE_STANDBY) &&
(smrecp->dbsync.fullSyncStatus > DBSYNC_STAT_UNINITIALIZED) &&
(smrecp->dbsync.fullTimeLastSync!=0)) {
record->SMInfo.ElapsedTime = (uint32_t)difftime(time(NULL), smrecp->dbsync.fullTimeLastSync);
} else {
record->SMInfo.ElapsedTime = 0;
IB_LOG_INFO64("sa_SMInfoRecord_GetTable: SM never syncd:", smrecp->portguid);
}
record->Reserved = 0;
BSWAP_STL_SMINFO_RECORD(record);
(void)sa_template_test_mask(samad.header.mask, samad.data, &data, sizeof(STL_SMINFO_RECORD),
bytes, records);
} while (cs_hashtable_iterator_advance(&itr));
}
(void)vs_unlock(&smRecords.smLock);
IB_EXIT("sa_SMInfoRecord_GetTable", VSTATUS_OK);
return(VSTATUS_OK);
}
void
sa_main_writer(uint32_t argc, uint8_t ** argv) {
Status_t status;
Mai_t in_mad;
Filter_t filter;
sa_cntxt_t *sa_cntxt;
uint64_t now, srLastAged=0, cacheLastCleaned=0;
uint32_t records=0;
IB_ENTER("sa_main_writer", 0, 0, 0, 0);
sa_main_writer_exit = 0;
//
// Create the SubnAdm(*) MAD filter for the SA thread.
//
SA_Filter_Init(&filter);
filter.value.mclass = MAD_CV_SUBN_ADM;
filter.mask.mclass = 0xff;
filter.value.method = 0x00;
filter.mask.method = 0x80;
filter.mai_filter_check_packet = sa_writer_filter;
MAI_SET_FILTER_NAME (&filter, "SA Writer");
if (mai_filter_create(fd_sa_w, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("esm_saw: can't create SubnAdm(*) filter");
(void)vs_thread_exit(&sm_threads[SM_THREAD_SA_WRITER].handle);
}
while (1) {
status = mai_recv(fd_sa_w, &in_mad, VTIMER_1S/4);
if (status != VSTATUS_OK && status != VSTATUS_TIMEOUT) {
IB_LOG_ERRORRC("sa_main_writer: error on mai_recv rc:", status);
vs_thread_sleep(VTIMER_1S/10);
}
if (sa_main_writer_exit == 1){
#ifdef __VXWORKS__
ESM_LOG_ESMINFO("SA Writer Task exiting OK.", 0);
#endif
break;
}
/* don't process messages if not master SM or still doing first sweep */
if (sm_state != SM_STATE_MASTER || topology_passcount < 1) {
continue;
}
/*
* process the rmpp ack and send out the next set of segments
*/
if (status == VSTATUS_OK) {
/* locate and process in flight rmpp request */
sa_cntxt = sa_cntxt_find( &in_mad );
if (sa_cntxt) {
sa_process_inflight_rmpp_request( &in_mad, sa_cntxt );
/*
* This may not necessarily release context
* based on if someone else has reserved it
*/
sa_cntxt_release( sa_cntxt );
} else {
INCREMENT_COUNTER(smCounterSaDeadRmppPacket);
if (saDebugRmpp) {
IB_LOG_INFINI_INFO_FMT( "sa_main_writer",
"dropping %s[%s] RMPP packet from LID[0x%x], TID ["FMT_U64"] already completed/aborted",
sa_getMethodText((int)in_mad.base.method), sa_getAidName(in_mad.base.aid),
in_mad.addrInfo.slid, in_mad.base.tid);
}
}
}
/* age contexts if more than 1 second since last time */
vs_time_get( &now );
if ((now - timeLastAged) > (VTIMER_1S)) {
(void) sa_cntxt_age();
}
/* age the service records */
if ((now - srLastAged) > 5*VTIMER_1S) {
srLastAged = now;
if ((status = sa_ServiceRecord_Age(&records)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_main_writer: failed to age service records, rc:", status);
} else if (records) {
if (smDebugPerf) IB_LOG_INFINI_INFO("sa_main_writer: Number of service records aged out was", records);
}
}
/* clean the SA cache */
if ((now - cacheLastCleaned) > SA_CACHE_CLEAN_INTERVAL) {
cacheLastCleaned = now;
(void)vs_lock(&saCache.lock);
sa_cache_clean();
(void)vs_unlock(&saCache.lock);
}
}
/* clean up cache before exit */
sa_cache_clean();
(void)vs_lock_delete(&saCache.lock);
//IB_LOG_INFINI_INFO0("sa_main_writer thread: Exiting OK");
} // SA_MAIN_WRITER
