static void safTerminate(SaInvocationT invocation, const SaNameT *compName)
{
SaAisErrorT rc = SA_AIS_OK;
if(gClMsgInit)
{
ClBoolT lockStatus = CL_TRUE;
ClTimerTimeOutT timeout = {.tsSec = 0, .tsMilliSec = 0};
clOsalMutexLock(&gClMsgFinalizeLock);
while(gClMsgSvcRefCnt > 0)
{
clOsalCondWait(&gClMsgFinalizeCond, &gClMsgFinalizeLock, timeout);
}
safMsgFinalize(&lockStatus);
if(lockStatus)
{
clOsalMutexUnlock(&gClMsgFinalizeLock);
}
}
rc = saAmfComponentUnregister(amfHandle, compName, NULL);
clCpmClientFinalize(amfHandle);
//clCpmResponse(cpmHandle, invocation, CL_OK);
saAmfResponse(amfHandle, invocation, SA_AIS_OK);
return;
}
static void clMsgRegisterWithCpm(void)
{
SaAisErrorT rc = SA_AIS_OK;
SaAmfCallbacksT callbacks = {0};
SaVersionT version = {0};
version.releaseCode = 'B';
version.majorVersion = 0x01;
version.minorVersion = 0x01;
callbacks.saAmfHealthcheckCallback = NULL;
callbacks.saAmfComponentTerminateCallback = safTerminate;
callbacks.saAmfCSISetCallback = NULL;
callbacks.saAmfCSIRemoveCallback = NULL;
callbacks.saAmfProtectionGroupTrackCallback = NULL;
rc = saAmfInitialize(&amfHandle, &callbacks, &version);
if( rc != SA_AIS_OK)
{
clLogError("MSG", "INI", "saAmfInitialize failed with error code [0x%x].", rc);
return ;
}
rc = saAmfComponentNameGet(amfHandle, &appName);
rc = saAmfComponentRegister(amfHandle, &appName, NULL);
}
void clPyGlueStart(int block)
{
ClRcT rc;
clRunPython("eoApp.Start()");
if (block)
{
ClTimerTimeOutT forever={0};
rc = clOsalMutexLock(&pyMutex);
CL_ASSERT(rc==CL_OK);
clOsalCondWait (&event,&pyMutex,forever);
}
}
ClRcT clMsgFinalizeBlocker(void)
{
ClRcT rc = CL_OK;
SaTimeT timeout = (SaTimeT)(CL_MSG_FIN_BLOCK_TIME * 1000000000LL);
ClTimerTimeOutT tempTime = {0};
CL_OSAL_MUTEX_LOCK(&gFinBlockMutex);
if(gMsgMoveStatus == MSG_MOVE_DONE)
goto out;
clLogDebug("FIN", "BLOCK", "Message service finalize will be blocked for [%llu ns].", timeout);
if(gMsgMoveStatus == MSG_MOVE_FIN_UNINIT)
{
rc = clOsalTaskCreateDetached("allClosedQueueMoverThread", CL_OSAL_SCHED_OTHER, CL_OSAL_THREAD_PRI_NOT_APPLICABLE, 0, clMsgClosedQueueMoveThread, NULL);
if(rc != CL_OK)
{
clLogError("FIN", "BLOCK", "Failed to create a Queue mover thread. error code [0x%x].", rc);
goto error_out;
}
clMsgTimeConvert(&tempTime, timeout);
rc = clOsalCondWait(gFinBlockCond, &gFinBlockMutex, tempTime);
if(CL_GET_ERROR_CODE(rc) == CL_ERR_TIMEOUT)
{
clLogError("FIN", "BLOCK", "Finalize blocking timed out. Timeout is [%lld ns]. error code [0x%x].", timeout, rc);
}
else if(rc != CL_OK)
{
clLogError("FIN", "BLOCK", "Failed at Conditional Wait. error code [0x%x].", rc);
}
else
{
clLogDebug("FIN", "BLOCK", "Message queues are failed over to [0x%x] node.", gQMoveDestNode);
}
}
gMsgMoveStatus = MSG_MOVE_DONE;
goto out;
error_out:
out:
CL_OSAL_MUTEX_UNLOCK(&gFinBlockMutex);
return rc;
}
static void safTerminate(SaInvocationT invocation, const SaNameT *compName)
{
if(gClMsgInit)
{
ClBoolT lockStatus = CL_TRUE;
ClTimerTimeOutT timeout = { 0, 0};
clOsalMutexLock(&gClMsgFinalizeLock);
while(gClMsgSvcRefCnt > 0)
{
clOsalCondWait(&gClMsgFinalizeCond, &gClMsgFinalizeLock, timeout);
}
safMsgFinalize(&lockStatus);
if(lockStatus)
{
clOsalMutexUnlock(&gClMsgFinalizeLock);
}
}
IGNORE_RETURN(saAmfComponentUnregister(amfHandle, compName, NULL));
clCpmClientFinalize(amfHandle);
//clCpmResponse(cpmHandle, invocation, CL_OK);
saAmfResponse(amfHandle, invocation, SA_AIS_OK);
return;
}
/*
* To all the application we need to pass 1. Component Name 2. Ioc Address So
* as of now we have decided that these will be passed as environment variable
*/
ClRcT clEoInitialize(ClInt32T argc, ClCharT *argv[])
{
ClRcT rc = CL_OK;
ClEoExecutionObjT *pThis = NULL;
ClTimerTimeOutT waitForExit = { 0, 0 };
clEoProgName = argv[0];
clLog(CL_LOG_SEV_INFO, CL_LOG_AREA, CL_LOG_CTXT_INI, "Process [%s] started. PID [%d]", clEoProgName, (int)getpid());
clEoStaticQueueInit();
clASPInitialize();
rc = clEoMyEoObjectGet(&pThis);
if(rc != CL_OK)
{
clLog(CL_LOG_SEV_CRITICAL, CL_LOG_AREA, CL_LOG_CTXT_INI, "Exiting : EO my object get failed. error [%x0x].\n", rc);
exit(1);
}
/*
* This should keep track of blocking APP initialize.
*/
clEoRefInc(pThis);
/* Call the application's initialize function */
rc = eoConfig.clEoCreateCallout(argc, argv);
if (rc != CL_OK)
{
clLog(CL_LOG_SEV_CRITICAL, CL_LOG_AREA, CL_LOG_CTXT_INI, "Application initialization failed, error [0x%x]", rc);
exit(1);
}
if(eoConfig.appType == CL_EO_USE_THREAD_FOR_APP)
{
clEoUnblock(pThis);
}
/*
* We block on the exit path waiting for a terminate.
*/
clOsalMutexLock(&pThis->eoMutex);
while (pThis->refCnt > 0)
{
clOsalCondWait(&pThis->eoCond, &pThis->eoMutex, waitForExit);
}
clOsalMutexUnlock(&pThis->eoMutex);
clEoTearDown();
clLog(CL_LOG_SEV_INFO, CL_LOG_AREA, CL_LOG_CTXT_FIN, "Process [%s] exited normally", argv[0]);
if (clDbgNoKillComponents)
{
clLog(CL_LOG_SEV_CRITICAL, CL_LOG_AREA, CL_LOG_CTXT_FIN,
"In debug mode and 'clDbgNoKillComponents' is set, so this process will pause, not exit.");
while(1) sleep(10000); /* Using sleep here instead of Osal because Osal is likely shutdown */
}
return CL_OK;
}
/*
* The recovery thread for entity threshold.
*/
static ClPtrT clAmsEntityTriggerRecoveryThread(ClPtrT pArg)
{
#define MAX_RECOVERY_MASK (0x3)
static ClInt32T numRecovery = 0;
ClTimerTimeOutT timeout = { 0, 0 };
ClTimerTimeOutT delay = { 0, 500 };
static ClAmsListHeadT recoveryList = { 0, CL_LIST_HEAD_INITIALIZER(recoveryList.list) };
clOsalMutexLock(&gClAmsEntityTriggerRecoveryCtrl.list.mutex);
while(gClAmsEntityTriggerRecoveryCtrl.running == CL_TRUE)
{
ClListHeadT *pNext = NULL;
while(!gClAmsEntityTriggerRecoveryCtrl.list.numElements)
{
numRecovery = 0;
clOsalCondWait(&gClAmsEntityTriggerRecoveryCtrl.list.cond,
&gClAmsEntityTriggerRecoveryCtrl.list.mutex,
timeout);
if(gClAmsEntityTriggerRecoveryCtrl.running == CL_FALSE)
goto out_drain;
}
/*
* We are here when the recovery list isn't empty.
* We shove the existing recovery batch into the temp list
* and reset the main list to release the lock and go lockless.
*/
recoveryList.numElements = gClAmsEntityTriggerRecoveryCtrl.list.numElements;
gClAmsEntityTriggerRecoveryCtrl.list.numElements = 0;
pNext = gClAmsEntityTriggerRecoveryCtrl.list.list.pNext;
clListDelInit(&gClAmsEntityTriggerRecoveryCtrl.list.list);
clListAddTail(pNext, &recoveryList.list);
clOsalMutexUnlock(&gClAmsEntityTriggerRecoveryCtrl.list.mutex);
while(gClAmsEntityTriggerRecoveryCtrl.running == CL_TRUE
&&
recoveryList.numElements > 0
&&
!CL_LIST_HEAD_EMPTY(&recoveryList.list))
{
ClListHeadT *pFirst = recoveryList.list.pNext;
ClAmsEntityTriggerT *pEntityTrigger = CL_LIST_ENTRY(pFirst, ClAmsEntityTriggerT, list);
/*
* Take a break incase you have done some recoveries
*/
if(numRecovery && !(numRecovery & MAX_RECOVERY_MASK))
{
clOsalTaskDelay(delay);
}
++numRecovery;
numRecovery &= 0xffff;
/*
* Unlink the entry from the recovery list.
*/
clListDel(pFirst);
clAmsEntityTriggerRecovery(pEntityTrigger);
clHeapFree(pEntityTrigger);
--recoveryList.numElements;
}
clOsalMutexLock(&gClAmsEntityTriggerRecoveryCtrl.list.mutex);
if(gClAmsEntityTriggerRecoveryCtrl.running == CL_FALSE)
{
goto out_drain;
}
}
/*
* Drain the entries holding the lock.
*/
out_drain:
clAmsListHeadDelete(&recoveryList);
clAmsListHeadDelete(&gClAmsEntityTriggerRecoveryCtrl.list);
clOsalMutexUnlock(&gClAmsEntityTriggerRecoveryCtrl.list.mutex);
return NULL;
}
ClRcT clCkptSvrInitialize(void)
{
ClRcT rc = CL_OK;
/* Variable related ckptDataBackup feature thats not supported
ClUint8T ckptRead = 0;
*/
ClTimerTimeOutT timeOut = {0};
ClIocNodeAddressT deputy = 0;
ClIocNodeAddressT master = 0;
ClBoolT addressUpdate = CL_FALSE;
SaNameT appName = {0};
/*
* Allocate the memory for server control block.
*/
rc = ckptSvrCbAlloc(&gCkptSvr);
CKPT_ERR_CHECK(CL_CKPT_SVR,CL_LOG_SEV_ERROR,
("Checkpoint service CB create failed rc[0x %x]\n", rc), rc);
/*
* Mark server as not ready.
*/
gCkptSvr->serverUp = CL_FALSE;
gCkptSvr->condVarWaiting = CL_TRUE;
/*
* Create condition variable that indicate the receipt
* of master and deputy ckpt addresses from gms.
*/
clOsalCondCreate(&gCkptSvr->condVar);
clOsalMutexCreate(&gCkptSvr->mutexVar);
gCkptSvr->masterInfo.masterAddr = CL_CKPT_UNINIT_ADDR;
gCkptSvr->masterInfo.deputyAddr = CL_CKPT_UNINIT_ADDR;
gCkptSvr->masterInfo.compId = CL_CKPT_UNINIT_VALUE;
gCkptSvr->masterInfo.clientHdlCount = 0;
gCkptSvr->masterInfo.masterHdlCount = 0;
/*
* Initialize gms to get the master and deputy addresses.
*/
clOsalMutexLock(&gCkptSvr->ckptClusterSem);
rc = clCkptMasterAddressesSet();
master = gCkptSvr->masterInfo.masterAddr;
deputy = gCkptSvr->masterInfo.deputyAddr;
clOsalMutexUnlock(&gCkptSvr->ckptClusterSem);
rc = clEoMyEoObjectGet(&gCkptSvr->eoHdl);
rc = clEoMyEoIocPortGet(&gCkptSvr->eoPort);
/*
* Install the ckpt native table.
*/
rc = clCkptEoClientInstall();
CL_ASSERT(rc == CL_OK);
rc = clCkptEoClientTableRegister(CL_IOC_CKPT_PORT);
CL_ASSERT(rc == CL_OK);
/*
* Initialize the event client library.
*/
ckptEventSvcInitialize();
/*
* Initialize ckpt lib for persistent memory.
* Obtain ckptRead flag that will tell whether ckpt server needs to
* read data from persistent memory or not.
*/
/*
Feature not yet supported see bug 6017 -- Do not forget to uncomment ckptDataBackupFinalize
rc = ckptDataBackupInitialize(&ckptRead);
*/
if(rc != CL_OK)
{
clLogWrite(CL_LOG_HANDLE_APP, CL_LOG_SEV_WARNING, NULL,
CL_LOG_MESSAGE_2_LIBRARY_INIT_FAILED, "ckpt", rc);
}
if( gCkptSvr->localAddr == master )
{
/*
* The node coming up is master. If deputy exists, syncup with deputy
* else syncup with persistant memory (if existing).
*/
rc = clCpmComponentNameGet(gCkptSvr->amfHdl, &appName);
if(clCpmIsCompRestarted(appName))
{
if((deputy != CL_CKPT_UNINIT_ADDR) && ( (ClInt32T) deputy != -1))
{
rc = ckptMasterDatabaseSyncup(deputy);
/*
* If deputy server is not reachable then pull info from the
* persistent memory.
*/
if(CL_RMD_TIMEOUT_UNREACHABLE_CHECK(rc))
{
clLogNotice(CL_CKPT_AREA_ACTIVE, "DBS",
"Database Syncup with ckpt master failed with RMD timeout error. rc 0x%x",rc);
/* This is dead code as the feature is not supported
if(ckptRead == 1)
{
rc = ckptPersistentMemoryRead();
}
*/
}
}
}
/* This is dead code as the feature is not supported
else
{
if(ckptRead == 1)
{
rc = ckptPersistentMemoryRead();
}
}
*/
}
/*TODO:This check is only for seleting the deputy or for what */
clOsalMutexLock(gCkptSvr->mutexVar);
if( (clIocLocalAddressGet() != master) &&
(gCkptSvr->condVarWaiting == CL_TRUE) )
{
/*
* Ensure that the node gets the master and deputy addresses before
* proceeding further.
*/
timeOut.tsSec = 5;
timeOut.tsMilliSec = 0;
clOsalCondWait(gCkptSvr->condVar, gCkptSvr->mutexVar, timeOut);
gCkptSvr->condVarWaiting = CL_FALSE;
addressUpdate = CL_TRUE;
}
clOsalMutexUnlock(gCkptSvr->mutexVar);
/*
* We double check again incase we had waited for the gms cluster track.
* but with the right lock sequence below instead of shoving it in the
* condwait above.
*/
if(addressUpdate == CL_TRUE)
{
clOsalMutexLock(&gCkptSvr->ckptClusterSem);
master = gCkptSvr->masterInfo.masterAddr;
deputy = gCkptSvr->masterInfo.deputyAddr;
clOsalMutexUnlock(&gCkptSvr->ckptClusterSem);
}
/*
* Announce the arrival to peer in the n/w. Master server announces to
* all and other servers announce to master server.
*/
ckptSvrArrvlAnnounce();
/*
* If the node coming up is deputy, syncup the metadata and ckpt info
* from the master. Treat all SC capable nodes except the master as deputy.
*/
if( gCkptSvr->localAddr != master && clCpmIsSCCapable())
{
/*
* Just freezing the deputy till syncup, not to receive any master
* related calls, this is to ensure that deputy is in full synup with
* master and any calls from master to update deputy will sleep on the
* lock.
*/
CKPT_LOCK(gCkptSvr->masterInfo.ckptMasterDBSem);
ckptMasterDatabaseSyncup(gCkptSvr->masterInfo.masterAddr);
CKPT_UNLOCK(gCkptSvr->masterInfo.ckptMasterDBSem);
}
/*
* Server is up.
*/
gCkptSvr->serverUp = CL_TRUE;
return rc;
exitOnError:
{
return rc;
}
}
ClRcT cpmGmsInitialize(void)
{
ClRcT rc = CL_OK;
ClTimerTimeOutT timeOut = {0, 0};
ClGmsCallbacksT cpmGmsCallbacks = { NULL, cpmClusterTrackCallBack, NULL, NULL };
gpClCpm->version.releaseCode = 'B';
gpClCpm->version.majorVersion = 0x01;
gpClCpm->version.minorVersion = 0x01;
rc = clGmsInitialize(&gpClCpm->cpmGmsHdl,
&cpmGmsCallbacks,
&gpClCpm->version);
if (CL_OK != rc)
{
clLogError(CPM_LOG_AREA_CPM, CPM_LOG_CTX_CPM_GMS,
"Failed to do GMS initialization, error [%#x]",
rc);
gpClCpm->cpmGmsHdl = CL_HANDLE_INVALID_VALUE;
goto failure;
}
rc = clOsalMutexLock(&gpClCpm->cpmGmsMutex);
CL_CPM_CHECK_1(CL_LOG_SEV_ERROR, CL_CPM_LOG_1_OSAL_MUTEX_LOCK_ERR, rc, rc,
CL_LOG_HANDLE_APP);
rc = clGmsClusterTrack(gpClCpm->cpmGmsHdl, CL_GMS_TRACK_CHANGES_ONLY | CL_GMS_TRACK_CURRENT, NULL);
if (CL_OK != rc)
{
clOsalMutexUnlock(&gpClCpm->cpmGmsMutex);
clLogError(CPM_LOG_AREA_CPM, CPM_LOG_CTX_CPM_GMS,
"The GMS cluster track function failed, error [%#x]",
rc);
goto failure;
}
/*
* Wait for the GMS callback
*/
retry:
#ifdef VXWORKS_BUILD
timeOut.tsSec = gpClCpm->cpmGmsTimeout + 20;
#else
timeOut.tsSec = gpClCpm->cpmGmsTimeout + 60; /* There is no reason to not wait for a long time. 100% cpu could cause GMS to come up slowly */
#endif
timeOut.tsMilliSec = 0;
clLogInfo(CPM_LOG_AREA_CPM, CPM_LOG_CTX_CPM_GMS,
"Node [%s] waiting for GMS cluster track callback for [%d.%d] secs",
gpClCpm->pCpmLocalInfo->nodeName, timeOut.tsSec, timeOut.tsMilliSec);
rc = clOsalCondWait(&gpClCpm->cpmGmsCondVar,
&gpClCpm->cpmGmsMutex,
timeOut);
if (CL_OK != rc)
{
if(gpClCpm->trackCallbackInProgress)
{
clLogWarning(CPM_LOG_AREA_CPM, CPM_LOG_CTX_CPM_GMS,
"GMS cluster track callback in progress. Waiting for it to complete ...");
goto retry;
}
clLogError(CPM_LOG_AREA_CPM, CPM_LOG_CTX_CPM_GMS,
"Failed to receive GMS cluster track callback, "
"error [%#x].",
rc);
clOsalMutexUnlock(&gpClCpm->cpmGmsMutex);
goto failure;
}
rc = clOsalMutexUnlock(&gpClCpm->cpmGmsMutex);
CL_CPM_CHECK_1(CL_LOG_SEV_ERROR, CL_CPM_LOG_1_OSAL_MUTEX_UNLOCK_ERR, rc,
rc, CL_LOG_HANDLE_APP);
return CL_OK;
failure:
return rc;
}
static void *
clLogDeferredHandler(void *pData)
{
ClRcT rc = CL_OK;
ClTimerTimeOutT timeOut ={.tsSec = 0, .tsMilliSec = 0};
ClUint16T numRecs = 0;
ClLogDeferredHeaderT msg[CL_LOG_MAX_FLUSH_COUNT];
static ClUint16T lastReadIdx = 0xFFFF;
static ClUint16T lastOverwriteFlag = 0xFFFF;
static ClUint16T numNotFlushed = 0;
while( gUtilLibInitialized )
{
rc = clOsalMutexLock(&gLogMutex);
if( CL_OK != rc )
{
gUtilLibInitialized = CL_FALSE;
goto threadExit;
}
/*
* Sleep for two seconds, it may be woken up by
* writer for flushing.
* Disable the deferred flusher thread as its obsoleted by other changes to flush to log streams
* now.
*/
if(1 /*clLogUtilNumRecordsTBW() <= (2 * CL_LOG_MAX_FLUSH_COUNT)*/)
{
//fprintf(stderr, "sleeping....getpid[%d] readIdx [%d] writeIdx [%d]\n", getpid(), readIdx,
// writeIdx);
rc = clOsalCondWait(&gLogCond, &gLogMutex, timeOut);
if( (CL_OK != rc) && (CL_GET_ERROR_CODE(rc) != CL_ERR_TIMEOUT))
{
clOsalMutexUnlock(&gLogMutex);
gUtilLibInitialized = CL_FALSE;
goto threadExit;
}
}
if( lastOverwriteFlag != overWriteFlag )
{
/* last over write flag is not matching with overWriteFlag,
* so readIdx has been modifiedi by writer,
* so making numNotFlushed to 0
*/
numNotFlushed = 0;
}
if( (numNotFlushed != 0) && (numNotFlushed != numRecs) )
{
if( readIdx < numNotFlushed )
{
readIdx = CL_LOG_MAX_NUM_MSGS - ( numNotFlushed - readIdx);
}
else
{
readIdx -= numNotFlushed;
}
}
/* niether everything nor nothing */
if( (numNotFlushed == 0) || (lastReadIdx != readIdx) )
{
/*
* if the readIdx has been changed while this thread was
* not having the lock variable, lastOverwriteFlag has been
* modified, this means I have to change my buffer.
*/
clLogNumFlushableRecordsGet(msg, &numRecs, &lastReadIdx,
&lastOverwriteFlag);
//fprintf(stderr, "msg repopulation, numNotFlushed: [%d] lastReadIdx [%d] readIdx [%d] numRecs [%d]\n",
// numNotFlushed, lastReadIdx, readIdx, numRecs);
}
numNotFlushed = 0;
if( gUtilLibInitialized == CL_FALSE )
{
clOsalMutexUnlock(&gLogMutex);
goto threadExit;
}
/* unlock it */
rc = clOsalMutexUnlock(&gLogMutex);
if( CL_OK != rc )
{
gUtilLibInitialized = CL_FALSE;
goto threadExit;
}
/* Do the flushing */
rc = clLogDoLogWrite(msg, numRecs, &numNotFlushed);
if( CL_OK != rc )
{
goto threadExit;
}
}
threadExit:
if( CL_OK == rc )
{
clLogDoLogWrite(msg, numRecs, &numNotFlushed);
}
gUtilLibInitialized = CL_FALSE;
return NULL;
}
static ClUint16T
clLogUtilNumRecordsTBW(void)
{
ClUint16T numRecs = 0;
//fprintf(stderr, "readIdx: %d writeIdx: %d\n", readIdx, writeIdx);
if( (readIdx == writeIdx) && (overWriteFlag == 1))
{
numRecs = CL_LOG_MAX_NUM_MSGS - 1;
}
else if( (readIdx <= writeIdx) )
{
numRecs = writeIdx - readIdx;
}
else if( (readIdx > writeIdx) && (overWriteFlag == 0) )
{
numRecs = (CL_LOG_MAX_NUM_MSGS - readIdx) + writeIdx;
}
else
{
fprintf(stderr, "readIdx & writeIdx updation went wrong readIdx [%d] writeIdx [%d] overWriteFlag [%d]\n",
readIdx, writeIdx, overWriteFlag);
/* Should not occur */
}
return numRecs;
}