void CSPStore::Close()
{
if (m_connected)
{
if (m_hFlaim != 0)
{
FlmDbClose(&m_hFlaim);
m_hFlaim = 0;
}
m_connected = false;
}
} // CSPStore::disconnect()
/****************************************************************************
Desc:
****************************************************************************/
TestBase::~TestBase()
{
if( m_pLogger)
{
m_pLogger->Release();
}
if( m_pDisplayer)
{
m_pDisplayer->Release();
}
if( m_pReporter)
{
m_pReporter->Release();
}
if( m_hDb != HFDB_NULL)
{
FlmDbClose( &m_hDb);
}
}
/****************************************************************************
Desc:
****************************************************************************/
RCODE TestBase::shutdownTestState(
const char * pszDibName,
FLMBOOL bRemoveDib)
{
RCODE rc = FERR_OK;
if( bRemoveDib)
{
if( m_hDb != HFDB_NULL)
{
FlmDbClose( &m_hDb);
}
if( RC_BAD( rc = FlmDbRemove( pszDibName, NULL, NULL, TRUE)))
{
goto Exit;
}
}
Exit:
return( rc);
}
/****************************************************************************
Desc: Thread that will build an index in the background.
Caller will create a pDb to use. This pDb must be
freed at the conclusion of the routine.
****************************************************************************/
FSTATIC RCODE FLMAPI flmBackgroundIndexBuildThrd(
IF_Thread * pThread)
{
RCODE rc = FERR_OK;
IXD * pIxd;
F_BKGND_IX * pBackgroundIx = (F_BKGND_IX *)pThread->getParm1();
FLMBOOL bStartedTrans;
FLMBOOL bDbInitialized;
FLMUINT uiContainerNum;
FLMUINT uiFirstDrn;
FLMUINT uiIndexNum;
FDB * pDb = NULL;
FLMBOOL bForcedShutdown = FALSE;
FLMBOOL bHitEnd;
FINDEX_STATUS savedIxStatus;
FlmRecord * pReusableRec = NULL;
char szMsg[ 128];
FLMINT iErrorLine = 0;
FLMBOOL bLimitedMode = FALSE;
pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING);
if( (pReusableRec = f_new FlmRecord) != NULL)
{
if( RC_BAD( pReusableRec->preallocSpace( 512, 1024 * 64)))
{
pReusableRec->Release();
pReusableRec = NULL;
}
}
Loop_Again:
rc = FERR_OK;
uiIndexNum = pBackgroundIx->indexStatus.uiIndexNum;
flmAssert( pThread->getThreadAppId() == uiIndexNum);
bDbInitialized = FALSE;
bStartedTrans = FALSE;
pDb = NULL;
// We could loop forever on flmOpenFile errors, check if we should exit.
if( pThread->getShutdownFlag())
{
bForcedShutdown = TRUE;
goto Exit;
}
if( RC_BAD( rc = flmOpenFile( pBackgroundIx->pFile,
NULL, NULL, NULL, 0, TRUE, NULL, NULL,
pBackgroundIx->pFile->pszDbPassword, &pDb)))
{
// If the file is being closed, this is not an error.
if( pBackgroundIx->pFile->uiFlags & DBF_BEING_CLOSED)
{
bForcedShutdown = TRUE;
rc = FERR_OK;
}
else
{
iErrorLine = (FLMINT)__LINE__;
}
goto Exit;
}
flmAssert( pDb->pSFileHdl);
bDbInitialized = TRUE;
if (RC_BAD( rc = fdbInit( pDb, FLM_NO_TRANS, 0, 0, &bStartedTrans)))
{
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
flmAssert( !bStartedTrans);
pDb->uiFlags |= FDB_BACKGROUND_INDEXING;
for(;;)
{
// Set the thread's status
pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING);
// See if we should shut down.
if( pThread->getShutdownFlag())
{
bForcedShutdown = TRUE;
break;
}
// Obtain the file lock
flmAssert( !(pDb->uiFlags & FDB_HAS_FILE_LOCK));
if( RC_BAD( rc = pDb->pFile->pFileLockObj->lock( pDb->hWaitSem,
TRUE, FLM_NO_TIMEOUT, FLM_BACKGROUND_LOCK_PRIORITY,
pDb->pDbStats ? &pDb->pDbStats->LockStats : NULL)))
{
if( rc == FERR_IO_FILE_LOCK_ERR)
{
// This would only happen if we were signaled to shut down.
// So, it's ok to exit
flmAssert( pThread->getShutdownFlag());
bForcedShutdown = TRUE;
rc = FERR_OK;
}
else
{
iErrorLine = (FLMINT)__LINE__;
}
goto Exit;
}
// The lock needs to be marked as implicit so that flmCommitDbTrans
// will unlock the file and allow the next update transaction to
// begin before all writes are complete.
pDb->uiFlags |= (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT);
// If there are higher priority waiters in the lock queue,
// or we are being told to shut down, we want to relinquish.
if( pThread->getShutdownFlag() ||
pDb->pFile->pFileLockObj->haveHigherPriorityWaiter(
FLM_BACKGROUND_LOCK_PRIORITY))
{
if (RC_BAD( rc = pDb->pFile->pFileLockObj->unlock()))
{
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT);
continue;
}
// Start an update transaction
if( RC_BAD( rc = flmBeginDbTrans( pDb, FLM_UPDATE_TRANS, FLM_NO_TIMEOUT,
FLM_DONT_POISON_CACHE)))
{
if( rc == FERR_IO_FILE_LOCK_ERR)
{
// This would only happen if we were signaled to shut down.
// So, it's ok to exit
flmAssert( pThread->getShutdownFlag());
bForcedShutdown = TRUE;
rc = FERR_OK;
}
else
{
iErrorLine = (FLMINT)__LINE__;
}
goto Exit;
}
bStartedTrans = TRUE;
if( RC_BAD( rc = fdictGetIndex( pDb->pDict, pDb->pFile->bInLimitedMode,
uiIndexNum, NULL, &pIxd, TRUE)))
{
// Index may have been deleted by another transaction, or
// there may have been some other error.
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
// If we're running in limited mode, then we can't mess with encrypted
// indexes. On the other hand, since the index is marked as offline,
// but not suspended, this thread has to exist, or else it will cause
// all kinds of problems elsewhere. So, in such a case, we will simply
// sleep in an inifinite loop until the shutdown flag is set.
// (We consider this acceptable becase running in limited mode is not
// the norm, and Flaim probably won't be up for very long in this mode.)
if( pDb->pFile->bInLimitedMode && pIxd->uiEncId)
{
bLimitedMode = TRUE;
goto Exit;
}
// Look up the tracker info to determine where we need to being indexing
if (RC_BAD( rc = flmGetIxTrackerInfo( pDb,
pBackgroundIx->indexStatus.uiIndexNum, &uiContainerNum,
&uiFirstDrn, NULL, &pBackgroundIx->indexStatus.bSuspended)))
{
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
// If the index is suspended, this thread should have been
// shut down. The suspending thread will keep the database
// locked until we exit. So, if we have the database locked,
// the index better not be suspended.
flmAssert( !pBackgroundIx->indexStatus.bSuspended &&
!(pIxd->uiFlags & IXD_SUSPENDED));
if (pIxd->uiContainerNum)
{
uiContainerNum = pIxd->uiContainerNum;
if( uiFirstDrn == DRN_LAST_MARKER)
{
goto Exit;
}
}
else
{
if( uiFirstDrn == DRN_LAST_MARKER && uiContainerNum == 0xFFFFFFFF)
{
goto Exit;
}
else
{
// The container number from the tracker record
// may not be a real container.
// Determine what the next actual container number is.
if (uiContainerNum != FLM_DATA_CONTAINER)
{
while( uiContainerNum < pDb->pDict->uiIttCnt)
{
ITT * pItt = &pDb->pDict->pIttTbl [uiContainerNum];
if (ITT_IS_CONTAINER( pItt))
{
break;
}
else
{
uiContainerNum++;
}
}
if (uiContainerNum >= pDb->pDict->uiIttCnt)
{
uiContainerNum = FLM_DATA_CONTAINER;
}
}
}
}
// Setup the DRN range we want to index.
uiFirstDrn++;
flmAssert( pIxd->uiLastDrnIndexed == uiFirstDrn - 1);
// Set the thread's status
pThread->setThreadStatus( "Indexing %u:%u",
(unsigned)uiContainerNum, (unsigned)uiFirstDrn);
// Read and index up to the highest drn (or record higher than uiEndDrn)
// or until time runs out. The 500 is millisecs to take for the transaction.
f_memcpy( &savedIxStatus,
&pBackgroundIx->indexStatus, sizeof( FINDEX_STATUS));
if( RC_BAD( rc = flmIndexSetOfRecords( pDb,
uiIndexNum, uiContainerNum, uiFirstDrn, DRN_LAST_MARKER,
NULL, NULL, NULL, NULL,
&pBackgroundIx->indexStatus, &bHitEnd, pThread, pReusableRec)))
{
// Lock the mutex while copying the saved index status back to
// the main index status so that someone requesting the index status
// won't see the status while the memcpy is in progress.
f_mutexLock( gv_FlmSysData.hShareMutex);
f_memcpy( &pBackgroundIx->indexStatus,
&savedIxStatus, sizeof( FINDEX_STATUS));
f_mutexUnlock( gv_FlmSysData.hShareMutex);
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
if( pBackgroundIx->indexStatus.uiRecordsProcessed -
savedIxStatus.uiRecordsProcessed)
{
if( RC_BAD( rc = pDb->pFile->pRfl->logIndexSet( uiIndexNum,
uiContainerNum, uiFirstDrn,
pBackgroundIx->indexStatus.uiLastRecordIdIndexed)))
{
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
}
// Commit the transaction (even if we didn't do any indexing work).
if( RC_BAD( rc = flmCommitDbTrans( pDb, 0, FALSE)))
{
iErrorLine = (FLMINT)__LINE__;
goto Exit;
}
bStartedTrans = FALSE;
pBackgroundIx->indexStatus.uiTransactions++;
if( bHitEnd)
{
// flmIndexSetOfRecords brought the index on-line
if( gv_FlmSysData.UpdateEvents.pEventCBList)
{
flmDoEventCallback( F_EVENT_UPDATES,
F_EVENT_INDEXING_COMPLETE, (void *)uiIndexNum,
(void *)0);
}
// Log a message
flmLogIndexingProgress( uiIndexNum, 0);
break;
}
}
Exit:
pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING);
if( bStartedTrans)
{
(void)flmAbortDbTrans( pDb);
bStartedTrans = FALSE;
}
if( pDb && pDb->uiFlags & FDB_HAS_FILE_LOCK)
{
(void)pDb->pFile->pFileLockObj->unlock();
pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT);
}
if( bDbInitialized)
{
fdbExit( pDb);
bDbInitialized = FALSE;
}
if( pDb)
{
(void)FlmDbClose( (HFDB *) &pDb);
}
if( RC_BAD(rc) && !bForcedShutdown)
{
if (rc == FERR_MEM || rc == FERR_IO_DISK_FULL ||
rc == FERR_MUST_WAIT_CHECKPOINT)
{
// Log the error
f_sprintf( (char *)szMsg,
"Background indexing thread %u (index %u)",
(unsigned)pThread->getThreadId(), (unsigned)uiIndexNum);
flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine);
// Sleep a half second and try again.
pThread->sleep( 500);
goto Loop_Again;
}
else
{
f_sprintf( (char *)szMsg,
"Background indexing thread %u (index %u) -- unrecoverable error.",
(unsigned)pThread->getThreadId(), (unsigned)uiIndexNum);
flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine);
}
}
if( pReusableRec)
{
flmAssert( pReusableRec->getRefCount() == 1);
pReusableRec->Release();
}
if( bLimitedMode)
{
flmAssert( RC_OK( rc));
for (;;)
{
if( pThread->getShutdownFlag())
{
break;
}
pThread->sleep( 1000);
}
}
// Set the thread's app ID to 0, so that it will not
// be found now that the thread is terminating (we don't
// want flmBackgroundIndexGet to find the thread).
pThread->setThreadAppId( 0);
// Free the background index structure
f_mutexLock( gv_FlmSysData.hShareMutex);
f_free( &pBackgroundIx);
pThread->setParm1( NULL);
f_mutexUnlock( gv_FlmSysData.hShareMutex);
return( rc);
}
/*******************************************************************************
Desc: Copies a database, including roll-forward log files.
*******************************************************************************/
FLMEXP RCODE FLMAPI FlmDbCopy(
const char * pszSrcDbName,
const char * pszSrcDataDir,
const char * pszSrcRflDir,
const char * pszDestDbName,
const char * pszDestDataDir,
const char * pszDestRflDir,
STATUS_HOOK fnStatusCallback,
void * UserData)
{
RCODE rc = FERR_OK;
FLMBYTE * pucLastCommittedLogHdr;
HFDB hDb = HFDB_NULL;
FDB * pDb;
FLMBOOL bDbLocked = FALSE;
FLMUINT uiDbVersion;
// Make sure the destination database is closed
if (RC_BAD( rc = FlmConfig( FLM_CLOSE_FILE,
(void *)pszDestDbName, (void *)pszDestDataDir)))
{
goto Exit;
}
gv_FlmSysData.pFileHdlCache->closeUnusedFiles();
// Open the database so we can force a checkpoint.
if (RC_BAD( rc = FlmDbOpen( pszSrcDbName, pszSrcDataDir, pszSrcRflDir,
0, NULL, &hDb)))
{
goto Exit;
}
pDb = (FDB *)hDb;
// Need to lock the database, because we want to do a checkpoint
// and then the copy immediately after without letting other
// threads have the opportunity to get in and update the
// database.
if (RC_BAD( rc = FlmDbLock( hDb, FLM_LOCK_EXCLUSIVE, 0, FLM_NO_TIMEOUT)))
{
goto Exit;
}
bDbLocked = TRUE;
// Force a checkpoint
if (RC_BAD( rc = FlmDbCheckpoint( hDb, FLM_NO_TIMEOUT)))
{
goto Exit;
}
pucLastCommittedLogHdr = &pDb->pFile->ucLastCommittedLogHdr[ 0];
// Get the low and high RFL log file numbers from the log
// header.
uiDbVersion = pDb->pFile->FileHdr.uiVersionNum;
// Once we get this far, we have exclusive access to the database
// and we have forced a checkpoint. The database's contents are
// guaranteed to be on disk at this point, and they will not
// change.
rc = flmCopyDb( uiDbVersion, pszSrcDbName, pszSrcDataDir, pszSrcRflDir,
pszDestDbName, pszDestDataDir, pszDestRflDir,
fnStatusCallback, UserData);
Exit:
// Unlock and close the database
if (bDbLocked)
{
FlmDbUnlock( hDb);
}
if (hDb != HFDB_NULL)
{
(void)FlmDbClose( &hDb);
(void)FlmConfig( FLM_CLOSE_FILE, (void *)pszSrcDbName,
(void *)pszSrcDataDir);
}
return( rc);
}
/********************************************************************
Desc: Loads the database with objects.
*********************************************************************/
RCODE gigaLoadDatabase( void)
{
RCODE rc = NE_FLM_OK;
FLMBOOL bTransActive = FALSE;
FLMBOOL bCommitTrans = FALSE;
FLMUINT uiObjsInTrans = 0;
FLMUINT uiChar = 0;
FLMUINT bSuspend = FALSE;
FlmRecord * pNewRec = NULL;
// Set cache size, if specified on command line.
if( gv_uiCacheSize)
{
if( RC_BAD( rc = FlmSetHardMemoryLimit( 0, FALSE, 0,
gv_uiCacheSize, 0)))
{
gigaOutputRcErr( "setting cache size", rc);
goto Exit;
}
}
// Set block cache percentage, if it is not default.
if( gv_uiBlockCachePercentage != 50)
{
if( RC_BAD( rc = FlmConfig( FLM_BLOCK_CACHE_PERCENTAGE,
(void *)gv_uiBlockCachePercentage, (void *)0)))
{
gigaOutputRcErr( "setting block cache percentage", rc);
goto Exit;
}
}
// Set the maximum and low dirty cache, if one was specified
if( gv_uiMaxDirtyCache)
{
if( RC_BAD( rc = FlmConfig( FLM_MAX_DIRTY_CACHE,
(void *)gv_uiMaxDirtyCache, (void *)gv_uiLowDirtyCache)))
{
gigaOutputRcErr( "setting maximum dirty cache", rc);
goto Exit;
}
}
// Set checkpoint interval, if one is specified.
if( gv_uiCPInterval != 0xFFFFFFFF)
{
if( RC_BAD( rc = FlmConfig( FLM_MAX_CP_INTERVAL,
(void *)gv_uiCPInterval, (void *)0)))
{
gigaOutputRcErr( "setting checkpoint interval", rc);
goto Exit;
}
}
// Enable/Disable direct I/O
if( RC_BAD( rc = FlmConfig( FLM_DIRECT_IO_STATE,
(void *)!gv_bDisableDirectIO, NULL)))
{
goto Exit;
}
// Create the database.
(void)FlmDbRemove( gv_szDibName, gv_szDataDir, gv_szRflDir, TRUE);
if( RC_BAD( rc = FlmDbCreate( gv_szDibName, gv_szDataDir, gv_szRflDir,
NULL, gv_pszGigaDictionary, NULL, &gv_hDb)))
{
gigaOutputRcErr( "creating database", rc);
goto Exit;
}
if( RC_BAD( rc = FlmDbConfig( gv_hDb,
FDB_RFL_FOOTPRINT_SIZE, (void *)(512 * 1024 * 1024), NULL)))
{
goto Exit;
}
if( RC_BAD( rc = FlmDbConfig( gv_hDb,
FDB_RBL_FOOTPRINT_SIZE, (void *)(512 * 1024 * 1024), NULL)))
{
goto Exit;
}
// Create the display
gv_uiTotalLoaded = 0;
gv_ui10SecTotal = 0;
f_mutexLock( gv_hWindowMutex);
FTXWinClear( gv_pWindow);
f_mutexUnlock( gv_hWindowMutex);
gigaOutputLabel( MAX_CACHE_ROW, "Maximum Cache Size (bytes)");
gigaOutputLabel( USED_CACHE_ROW, "Cache Used (bytes)");
gigaOutputLabel( ITEMS_CACHED_ROW, "Cache Used (items)");
gigaOutputLabel( DIRTY_CACHE_ROW, "Dirty Cache (bytes)");
gigaOutputLabel( LOG_CACHE_ROW, "Log Cache (bytes)");
gigaOutputLabel( FREE_CACHE_ROW, "Free Cache (bytes)");
gigaOutputLabel( CP_STATE_ROW, "Checkpoint State");
gigaUpdateMemInfo();
gigaOutputLabel( DB_NAME_ROW, "Database Name");
gigaOutputStr( DB_NAME_ROW, gv_szDibName);
gigaOutputLabel( TOTAL_TO_LOAD_ROW, "Total To Load");
gigaOutputUINT( TOTAL_TO_LOAD_ROW, gv_uiTotalToLoad);
gigaOutputLabel( TRANS_SIZE_ROW, "Transaction Size");
gigaOutputUINT( TRANS_SIZE_ROW, gv_uiTransSize);
gigaOutputLabel( TOTAL_LOADED_ROW, "Total Loaded");
gigaOutputUINT( TOTAL_LOADED_ROW, gv_uiTotalLoaded);
gigaOutputLabel( ADDS_PER_SEC_CURRENT, "Adds/Sec. (10 secs)");
gigaOutputUINT( ADDS_PER_SEC_CURRENT, 0);
gigaOutputLabel( ADDS_PER_SEC_OVERALL, "Adds/Sec. (overall)");
gigaOutputUINT( ADDS_PER_SEC_OVERALL, 0);
gigaOutputLabel( ELAPSED_TIME_ROW, "Elapsed Time");
gigaOutputStr( ELAPSED_TIME_ROW, "<none>");
if( RC_BAD( rc = gigaStartScreenThread()))
{
goto Exit;
}
gv_ui10SecStartTime = gv_uiStartTime = FLM_GET_TIMER();
gv_ui10Secs = FLM_SECS_TO_TIMER_UNITS( 10);
gv_ui1Sec = FLM_SECS_TO_TIMER_UNITS( 1);
for( ;;)
{
// See if we have been told to shut down, or if the user
// has pressed escape.
if( gv_bShutdown)
{
break;
}
// Every 127 objects, see if character was pressed and update
// count on screen.
if( (gv_uiTotalLoaded & 0x7F) == 0)
{
f_yieldCPU();
if( (uiChar = gigaSeeIfQuit()) != 0)
{
if( uiChar == FKB_ESCAPE)
{
break;
}
else if( uiChar == 's' || uiChar == 'S')
{
bSuspend = TRUE;
}
}
// Check for other keyboard options
}
else if( (gv_uiTotalLoaded & 0x7) == 0)
{
FLMUINT uiElapsedTime;
FLMUINT uiCurrTime;
uiCurrTime = FLM_GET_TIMER();
// If at least 10 seconds have elapsed, redisplay the average
// rate values.
if( (uiElapsedTime = FLM_ELAPSED_TIME( uiCurrTime,
gv_ui10SecStartTime)) >= gv_ui10Secs)
{
gigaUpdateLoadTimes();
}
}
// Start a transaction, if one is not going.
if( !bTransActive)
{
if( bSuspend)
{
uiChar = gigaGetInput(
"Load suspended, press any character to continue loading: ",
NULL);
bSuspend = FALSE;
}
if( RC_BAD( rc = gigaStartTrans()))
{
goto Exit;
}
bTransActive = TRUE;
bCommitTrans = FALSE;
uiObjsInTrans = 0;
}
// Increment the load counters and determine if this will be the
// last object of the transaction.
gv_uiTotalLoaded++;
uiObjsInTrans++;
if( uiObjsInTrans == gv_uiTransSize ||
gv_uiTotalLoaded == gv_uiTotalToLoad)
{
bCommitTrans = TRUE;
}
// Create a new object.
if( RC_BAD( rc = gigaMakeNewRecord( &pNewRec)))
{
goto Exit;
}
if( RC_BAD( rc = FlmRecordAdd( gv_hDb, FLM_DATA_CONTAINER,
NULL, pNewRec, FLM_DONT_INSERT_IN_CACHE)))
{
goto Exit;
}
// Commit when we reach the transaction size or the total to load.
// NOTE: The bCommitTrans flag is set above.
if( bCommitTrans)
{
if( RC_BAD( rc = gigaCommitTrans()))
{
goto Exit;
}
bTransActive = FALSE;
}
// See if we are done.
if( gv_uiTotalLoaded == gv_uiTotalToLoad)
{
flmAssert( !bTransActive);
break;
}
}
Exit:
if( pNewRec)
{
pNewRec->Release();
}
if( bTransActive)
{
(void)FlmDbTransAbort( gv_hDb);
}
if( gv_hDb != HFDB_NULL)
{
FlmDbCheckpoint( gv_hDb, FLM_NO_TIMEOUT);
gigaStopScreenThread();
FlmDbClose( &gv_hDb);
// This will cause us to wait for the last checkpoint
// to finish.
(void)FlmConfig( FLM_CLOSE_FILE, (void *)gv_szDibName,
(void *)gv_szDataDir);
}
gigaUpdateLoadTimes();
gigaStopScreenThread();
f_threadDestroy( &gv_pIxManagerThrd);
return( rc);
}
