/**********************************************************************
*
* FUNCTION
* cs_psema
*
* DESCRIPTION
* Decrement a semaphore (wait or take operation)
*
* INPUTS
* handle - A pointer to a semaphore object filled in by the
* cs_sema_create() call.
*
* OUTPUTS
* Status_t - On success VSTATUS_OK is returned, otherwise
* the cause of the error.
*
* RESPONSIBLE ENGINEER:
* CS Developers.
*
* HISTORY
*
* NAME DATE REMARKS
* MGR 03/14/02 Initial creation of function.
* MGR 04/18/02 PR1728. vs_psema should return VSTATUS_AGAIN
* error code, if thread has been killed.
*
**********************************************************************/
Status_t
cs_psema_wait (Sema_t *handle, int timeout)
{
Status_t rc=VSTATUS_OK;
IB_ENTER (function, (unint)handle, 0U, 0U, 0U);
/* Validate handle */
if (handle == (Sema_t *) 0)
{
IB_LOG_ERROR0 ("NULL handle pointer");
IB_EXIT (function, VSTATUS_ILLPARM);
return VSTATUS_ILLPARM;
}
if (handle->magic != SEMAPHORE_MAGIC)
{
IB_LOG_ERRORX ("Invalid semaphore specified:", handle->magic);
IB_EXIT (function, VSTATUS_NXIO);
return VSTATUS_NXIO;
}
if (timeout == CS_SEMA_WAIT_FOREVER) {
return(cs_psema(handle));
} else {
#ifdef __VXWORKS__
if (ERROR == semTake(handle->semId, timeout*sysClkRateGet()))
{
IB_LOG_WARN0("Timed out trying to take semaphore");
rc = VSTATUS_NXIO;
} else {
handle->count--;
}
#elif defined (__LINUX__)
// LINUX USER SEMA
int i=0, rc=-1;
do {
rc = sem_trywait((sem_t *)&handle->osdSema);
++i;
(void)vs_thread_sleep(VTIMER_1S);
} while (i<timeout && rc);
if (rc != 0) {
IB_LOG_WARN0("Timed out trying to take semaphore");
rc = VSTATUS_NXIO;
}
#endif
}
IB_EXIT (function, rc);
return rc;
}
void
sa_main_writer(uint32_t argc, uint8_t ** argv) {
Status_t status;
Mai_t in_mad;
Filter_t filter;
sa_cntxt_t *sa_cntxt;
uint64_t now, srLastAged=0, cacheLastCleaned=0;
uint32_t records=0;
IB_ENTER("sa_main_writer", 0, 0, 0, 0);
sa_main_writer_exit = 0;
//
// Create the SubnAdm(*) MAD filter for the SA thread.
//
SA_Filter_Init(&filter);
filter.value.mclass = MAD_CV_SUBN_ADM;
filter.mask.mclass = 0xff;
filter.value.method = 0x00;
filter.mask.method = 0x80;
filter.mai_filter_check_packet = sa_writer_filter;
MAI_SET_FILTER_NAME (&filter, "SA Writer");
if (mai_filter_create(fd_sa_w, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("esm_saw: can't create SubnAdm(*) filter");
(void)vs_thread_exit(&sm_threads[SM_THREAD_SA_WRITER].handle);
}
while (1) {
status = mai_recv(fd_sa_w, &in_mad, VTIMER_1S/4);
if (status != VSTATUS_OK && status != VSTATUS_TIMEOUT) {
IB_LOG_ERRORRC("sa_main_writer: error on mai_recv rc:", status);
vs_thread_sleep(VTIMER_1S/10);
}
if (sa_main_writer_exit == 1){
#ifdef __VXWORKS__
ESM_LOG_ESMINFO("SA Writer Task exiting OK.", 0);
#endif
break;
}
/* don't process messages if not master SM or still doing first sweep */
if (sm_state != SM_STATE_MASTER || topology_passcount < 1) {
continue;
}
/*
* process the rmpp ack and send out the next set of segments
*/
if (status == VSTATUS_OK) {
/* locate and process in flight rmpp request */
sa_cntxt = sa_cntxt_find( &in_mad );
if (sa_cntxt) {
sa_process_inflight_rmpp_request( &in_mad, sa_cntxt );
/*
* This may not necessarily release context
* based on if someone else has reserved it
*/
sa_cntxt_release( sa_cntxt );
} else {
INCREMENT_COUNTER(smCounterSaDeadRmppPacket);
if (saDebugRmpp) {
IB_LOG_INFINI_INFO_FMT( "sa_main_writer",
"dropping %s[%s] RMPP packet from LID[0x%x], TID ["FMT_U64"] already completed/aborted",
sa_getMethodText((int)in_mad.base.method), sa_getAidName(in_mad.base.aid),
in_mad.addrInfo.slid, in_mad.base.tid);
}
}
}
/* age contexts if more than 1 second since last time */
vs_time_get( &now );
if ((now - timeLastAged) > (VTIMER_1S)) {
(void) sa_cntxt_age();
}
/* age the service records */
if ((now - srLastAged) > 5*VTIMER_1S) {
srLastAged = now;
if ((status = sa_ServiceRecord_Age(&records)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_main_writer: failed to age service records, rc:", status);
} else if (records) {
if (smDebugPerf) IB_LOG_INFINI_INFO("sa_main_writer: Number of service records aged out was", records);
}
}
/* clean the SA cache */
if ((now - cacheLastCleaned) > SA_CACHE_CLEAN_INTERVAL) {
cacheLastCleaned = now;
(void)vs_lock(&saCache.lock);
sa_cache_clean();
(void)vs_unlock(&saCache.lock);
}
}
/* clean up cache before exit */
sa_cache_clean();
(void)vs_lock_delete(&saCache.lock);
//IB_LOG_INFINI_INFO0("sa_main_writer thread: Exiting OK");
} // SA_MAIN_WRITER
void
sa_main_reader(uint32_t argc, uint8_t ** argv) {
Status_t status;
Mai_t in_mad;
Filter_t filter;
sa_cntxt_t *sa_cntxt=NULL;
uint64_t now, delta, max_delta;
int tries=0, retry=0;
uint64_t reqTimeToLive=0;
SAContextGet_t cntxGetStatus=0;
int numContextBusy=0;
IB_ENTER("sa_main_reader", 0, 0, 0, 0);
sa_main_reader_exit = 0;
/*
* Create the SubnAdm(*) MAD filter for the SA thread.
*/
SA_Filter_Init(&filter);
filter.value.mclass = MAD_CV_SUBN_ADM;
filter.mask.mclass = 0xff;
filter.value.method = 0x00;
filter.mask.method = 0x80;
filter.mai_filter_check_packet = sa_reader_filter;
MAI_SET_FILTER_NAME (&filter, "SA Reader");
if (mai_filter_create(fd_sa, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("sa_main_reader: can't create SubnAdm(*) filter");
(void)vs_thread_exit(&sm_threads[SM_THREAD_SA_READER].handle);
}
timeMftLastUpdated = 0;
/*
* calculate request time to live on queue
* ~ 3.2secs for defaults: sa_packetLifetime=18 and sa_respTimeValue=18
*/
reqTimeToLive = 4ull * ( (2*(1 << sm_config.sa_packet_lifetime_n2)) + (1 << sm_config.sa_resp_time_n2) );
while (1) {
status = mai_recv(fd_sa, &in_mad, VTIMER_1S/4);
if (sa_main_reader_exit == 1){
#ifdef __VXWORKS__
ESM_LOG_ESMINFO("sa_main_reader: exiting OK.", 0);
#endif
break;
}
/* don't process messages if not master SM or still doing first sweep */
if ((sm_state != SM_STATE_MASTER) ||
(topology_passcount < 1)) {
continue;
}
/*
* If the mai layer shuts down we end up in this infinite loop here.
* This may happen on initialization
*/
if( status != VSTATUS_OK ){
if (status != VSTATUS_TIMEOUT)
IB_LOG_ERRORRC("sa_main_reader: error on mai_recv rc:", status);
} else {
/*
* Drop new requests that have been sitting on SA reader queue for too long
*/
if (in_mad.intime) {
/* PR 110586 - On some RHEL 5 systems, we've seen weird issues with gettimeofday() [used by vs_time_get()]
* where once in a while the time difference calculated from successive calls to gettimeofday()
* results in a negative value. Due to this, we might actually consider a request stale even if
* its not. Work around this by making calls to gettimeofday() till it returns us some
* sane values. Just to be extra cautious, bound the retries so that we don't get stuck in the loop.
*/
tries = 0;
/* Along with negative values also check for unreasonably high values of delta*/
max_delta = 30*reqTimeToLive;
do {
vs_time_get( &now );
delta = now - in_mad.intime;
tries++;
if ((now < in_mad.intime) || (delta > max_delta)) {
vs_thread_sleep(1);
retry = 1;
} else {
retry = 0;
}
} while (retry && tries < 20);
if (delta > reqTimeToLive) {
INCREMENT_COUNTER(smCounterSaDroppedRequests);
if (smDebugPerf || saDebugPerf) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"Dropping stale %s[%s] request from LID[0x%x], TID="FMT_U64"; On queue for %d.%d seconds.",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid), in_mad.addrInfo.slid,
in_mad.base.tid, (int)(delta/1000000), (int)((delta - delta/1000000*1000000))/1000);
}
/* drop the request without returning a response; sender will retry */
continue;
}
}
/*
* get a context to process request; sa_cntxt can be:
* 1. NULL if resources are scarce
* 2. NULL if request is dup of existing request
* 3. in progress getMulti request context
* 4. New context for a brand new request
*/
cntxGetStatus = sa_cntxt_get( &in_mad, (void *)&sa_cntxt );
if (cntxGetStatus == ContextAllocated) {
/* process the new request */
sa_process_mad( &in_mad, sa_cntxt );
/*
* This may not necessarily release context based on if someone else has reserved it
*/
if(sa_cntxt) sa_cntxt_release( sa_cntxt );
} else if (cntxGetStatus == ContextExist) {
INCREMENT_COUNTER(smCounterSaDuplicateRequests);
/* this is a duplicate request */
if (saDebugPerf || saDebugRmpp) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"SA_READER received duplicate %s[%s] from LID [0x%x] with TID ["FMT_U64"] ",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid),in_mad.addrInfo.slid, in_mad.base.tid);
}
} else if (cntxGetStatus == ContextNotAvailable) {
INCREMENT_COUNTER(smCounterSaContextNotAvailable);
/* we are swamped, return BUSY to caller */
if (saDebugPerf || saDebugRmpp) { /* log msg before send changes method and lids */
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"NO CONTEXT AVAILABLE, returning MAD_STATUS_BUSY to %s[%s] request from LID [0x%x], TID ["FMT_U64"]!",
sa_getMethodText((int)in_mad.base.method), sa_getAidName((int)in_mad.base.aid), in_mad.addrInfo.slid, in_mad.base.tid);
}
in_mad.base.status = MAD_STATUS_BUSY;
sa_send_reply( &in_mad, sa_cntxt );
if ((++numContextBusy % sa_max_cntxt) == 0) {
IB_LOG_INFINI_INFO_FMT( "sa_main_reader",
"Had to drop %d SA requests since start due to no available contexts",
numContextBusy);
}
} else if (cntxGetStatus == ContextExistGetMulti) {
/* continue processing the getMulti request */
sa_process_getmulti( &in_mad, sa_cntxt );
if(sa_cntxt) sa_cntxt_release( sa_cntxt );
} else {
IB_LOG_WARN("sa_main_reader: Invalid sa_cntxt_get return code:", cntxGetStatus);
}
}
/*
* signal sm_top to reprogram the MFTs
* Wait one second to allow mcmember requests to accumulate before asking
*/
vs_time_get( &now );
if (sa_mft_reprog && timeMftLastUpdated == 0) {
timeMftLastUpdated = now;
} else if (sa_mft_reprog && (now - timeMftLastUpdated) > VTIMER_1S) {
topology_wakeup_time = 0ull;
if ((status = vs_lock(&sa_lock)) != VSTATUS_OK) {
IB_LOG_ERRORRC("sa_main_reader: Failed to lock sa_lock rc:", status);
} else {
sm_McGroups_Need_Prog = 1; /* tells Topoloy thread that MFT reprogramming is needed */
(void)vs_unlock(&sa_lock);
}
sm_trigger_sweep(SM_SWEEP_REASON_MCMEMBER);
/* clear the indicators */
timeMftLastUpdated = 0;
sa_mft_reprog = 0;
}
}
/* cleanup before exit, but allow some time for the other threads to flush out first */
(void)vs_thread_sleep(VTIMER_1S);
(void)sa_SubscriberDelete();
(void)sa_ServiceRecDelete();
(void)sa_McGroupDelete();
if (mai_filter_delete(fd_sa, &filter, VFILTER_SHARE) != VSTATUS_OK) {
IB_LOG_ERROR0("sa_main_reader: can't delete SubnAdm(*) filter");
}
//IB_LOG_INFINI_INFO0("sa_main_reader thread: Exiting OK");
}