void CSPStore::Close() { if (m_connected) { if (m_hFlaim != 0) { FlmDbClose(&m_hFlaim); m_hFlaim = 0; } m_connected = false; } } // CSPStore::disconnect()
/**************************************************************************** Desc: ****************************************************************************/ TestBase::~TestBase() { if( m_pLogger) { m_pLogger->Release(); } if( m_pDisplayer) { m_pDisplayer->Release(); } if( m_pReporter) { m_pReporter->Release(); } if( m_hDb != HFDB_NULL) { FlmDbClose( &m_hDb); } }
/**************************************************************************** Desc: ****************************************************************************/ RCODE TestBase::shutdownTestState( const char * pszDibName, FLMBOOL bRemoveDib) { RCODE rc = FERR_OK; if( bRemoveDib) { if( m_hDb != HFDB_NULL) { FlmDbClose( &m_hDb); } if( RC_BAD( rc = FlmDbRemove( pszDibName, NULL, NULL, TRUE))) { goto Exit; } } Exit: return( rc); }
/**************************************************************************** Desc: Thread that will build an index in the background. Caller will create a pDb to use. This pDb must be freed at the conclusion of the routine. ****************************************************************************/ FSTATIC RCODE FLMAPI flmBackgroundIndexBuildThrd( IF_Thread * pThread) { RCODE rc = FERR_OK; IXD * pIxd; F_BKGND_IX * pBackgroundIx = (F_BKGND_IX *)pThread->getParm1(); FLMBOOL bStartedTrans; FLMBOOL bDbInitialized; FLMUINT uiContainerNum; FLMUINT uiFirstDrn; FLMUINT uiIndexNum; FDB * pDb = NULL; FLMBOOL bForcedShutdown = FALSE; FLMBOOL bHitEnd; FINDEX_STATUS savedIxStatus; FlmRecord * pReusableRec = NULL; char szMsg[ 128]; FLMINT iErrorLine = 0; FLMBOOL bLimitedMode = FALSE; pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING); if( (pReusableRec = f_new FlmRecord) != NULL) { if( RC_BAD( pReusableRec->preallocSpace( 512, 1024 * 64))) { pReusableRec->Release(); pReusableRec = NULL; } } Loop_Again: rc = FERR_OK; uiIndexNum = pBackgroundIx->indexStatus.uiIndexNum; flmAssert( pThread->getThreadAppId() == uiIndexNum); bDbInitialized = FALSE; bStartedTrans = FALSE; pDb = NULL; // We could loop forever on flmOpenFile errors, check if we should exit. if( pThread->getShutdownFlag()) { bForcedShutdown = TRUE; goto Exit; } if( RC_BAD( rc = flmOpenFile( pBackgroundIx->pFile, NULL, NULL, NULL, 0, TRUE, NULL, NULL, pBackgroundIx->pFile->pszDbPassword, &pDb))) { // If the file is being closed, this is not an error. if( pBackgroundIx->pFile->uiFlags & DBF_BEING_CLOSED) { bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } flmAssert( pDb->pSFileHdl); bDbInitialized = TRUE; if (RC_BAD( rc = fdbInit( pDb, FLM_NO_TRANS, 0, 0, &bStartedTrans))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } flmAssert( !bStartedTrans); pDb->uiFlags |= FDB_BACKGROUND_INDEXING; for(;;) { // Set the thread's status pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); // See if we should shut down. if( pThread->getShutdownFlag()) { bForcedShutdown = TRUE; break; } // Obtain the file lock flmAssert( !(pDb->uiFlags & FDB_HAS_FILE_LOCK)); if( RC_BAD( rc = pDb->pFile->pFileLockObj->lock( pDb->hWaitSem, TRUE, FLM_NO_TIMEOUT, FLM_BACKGROUND_LOCK_PRIORITY, pDb->pDbStats ? &pDb->pDbStats->LockStats : NULL))) { if( rc == FERR_IO_FILE_LOCK_ERR) { // This would only happen if we were signaled to shut down. // So, it's ok to exit flmAssert( pThread->getShutdownFlag()); bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } // The lock needs to be marked as implicit so that flmCommitDbTrans // will unlock the file and allow the next update transaction to // begin before all writes are complete. pDb->uiFlags |= (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); // If there are higher priority waiters in the lock queue, // or we are being told to shut down, we want to relinquish. if( pThread->getShutdownFlag() || pDb->pFile->pFileLockObj->haveHigherPriorityWaiter( FLM_BACKGROUND_LOCK_PRIORITY)) { if (RC_BAD( rc = pDb->pFile->pFileLockObj->unlock())) { iErrorLine = (FLMINT)__LINE__; goto Exit; } pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); continue; } // Start an update transaction if( RC_BAD( rc = flmBeginDbTrans( pDb, FLM_UPDATE_TRANS, FLM_NO_TIMEOUT, FLM_DONT_POISON_CACHE))) { if( rc == FERR_IO_FILE_LOCK_ERR) { // This would only happen if we were signaled to shut down. // So, it's ok to exit flmAssert( pThread->getShutdownFlag()); bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } bStartedTrans = TRUE; if( RC_BAD( rc = fdictGetIndex( pDb->pDict, pDb->pFile->bInLimitedMode, uiIndexNum, NULL, &pIxd, TRUE))) { // Index may have been deleted by another transaction, or // there may have been some other error. iErrorLine = (FLMINT)__LINE__; goto Exit; } // If we're running in limited mode, then we can't mess with encrypted // indexes. On the other hand, since the index is marked as offline, // but not suspended, this thread has to exist, or else it will cause // all kinds of problems elsewhere. So, in such a case, we will simply // sleep in an inifinite loop until the shutdown flag is set. // (We consider this acceptable becase running in limited mode is not // the norm, and Flaim probably won't be up for very long in this mode.) if( pDb->pFile->bInLimitedMode && pIxd->uiEncId) { bLimitedMode = TRUE; goto Exit; } // Look up the tracker info to determine where we need to being indexing if (RC_BAD( rc = flmGetIxTrackerInfo( pDb, pBackgroundIx->indexStatus.uiIndexNum, &uiContainerNum, &uiFirstDrn, NULL, &pBackgroundIx->indexStatus.bSuspended))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } // If the index is suspended, this thread should have been // shut down. The suspending thread will keep the database // locked until we exit. So, if we have the database locked, // the index better not be suspended. flmAssert( !pBackgroundIx->indexStatus.bSuspended && !(pIxd->uiFlags & IXD_SUSPENDED)); if (pIxd->uiContainerNum) { uiContainerNum = pIxd->uiContainerNum; if( uiFirstDrn == DRN_LAST_MARKER) { goto Exit; } } else { if( uiFirstDrn == DRN_LAST_MARKER && uiContainerNum == 0xFFFFFFFF) { goto Exit; } else { // The container number from the tracker record // may not be a real container. // Determine what the next actual container number is. if (uiContainerNum != FLM_DATA_CONTAINER) { while( uiContainerNum < pDb->pDict->uiIttCnt) { ITT * pItt = &pDb->pDict->pIttTbl [uiContainerNum]; if (ITT_IS_CONTAINER( pItt)) { break; } else { uiContainerNum++; } } if (uiContainerNum >= pDb->pDict->uiIttCnt) { uiContainerNum = FLM_DATA_CONTAINER; } } } } // Setup the DRN range we want to index. uiFirstDrn++; flmAssert( pIxd->uiLastDrnIndexed == uiFirstDrn - 1); // Set the thread's status pThread->setThreadStatus( "Indexing %u:%u", (unsigned)uiContainerNum, (unsigned)uiFirstDrn); // Read and index up to the highest drn (or record higher than uiEndDrn) // or until time runs out. The 500 is millisecs to take for the transaction. f_memcpy( &savedIxStatus, &pBackgroundIx->indexStatus, sizeof( FINDEX_STATUS)); if( RC_BAD( rc = flmIndexSetOfRecords( pDb, uiIndexNum, uiContainerNum, uiFirstDrn, DRN_LAST_MARKER, NULL, NULL, NULL, NULL, &pBackgroundIx->indexStatus, &bHitEnd, pThread, pReusableRec))) { // Lock the mutex while copying the saved index status back to // the main index status so that someone requesting the index status // won't see the status while the memcpy is in progress. f_mutexLock( gv_FlmSysData.hShareMutex); f_memcpy( &pBackgroundIx->indexStatus, &savedIxStatus, sizeof( FINDEX_STATUS)); f_mutexUnlock( gv_FlmSysData.hShareMutex); iErrorLine = (FLMINT)__LINE__; goto Exit; } if( pBackgroundIx->indexStatus.uiRecordsProcessed - savedIxStatus.uiRecordsProcessed) { if( RC_BAD( rc = pDb->pFile->pRfl->logIndexSet( uiIndexNum, uiContainerNum, uiFirstDrn, pBackgroundIx->indexStatus.uiLastRecordIdIndexed))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } } // Commit the transaction (even if we didn't do any indexing work). if( RC_BAD( rc = flmCommitDbTrans( pDb, 0, FALSE))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } bStartedTrans = FALSE; pBackgroundIx->indexStatus.uiTransactions++; if( bHitEnd) { // flmIndexSetOfRecords brought the index on-line if( gv_FlmSysData.UpdateEvents.pEventCBList) { flmDoEventCallback( F_EVENT_UPDATES, F_EVENT_INDEXING_COMPLETE, (void *)uiIndexNum, (void *)0); } // Log a message flmLogIndexingProgress( uiIndexNum, 0); break; } } Exit: pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); if( bStartedTrans) { (void)flmAbortDbTrans( pDb); bStartedTrans = FALSE; } if( pDb && pDb->uiFlags & FDB_HAS_FILE_LOCK) { (void)pDb->pFile->pFileLockObj->unlock(); pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); } if( bDbInitialized) { fdbExit( pDb); bDbInitialized = FALSE; } if( pDb) { (void)FlmDbClose( (HFDB *) &pDb); } if( RC_BAD(rc) && !bForcedShutdown) { if (rc == FERR_MEM || rc == FERR_IO_DISK_FULL || rc == FERR_MUST_WAIT_CHECKPOINT) { // Log the error f_sprintf( (char *)szMsg, "Background indexing thread %u (index %u)", (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine); // Sleep a half second and try again. pThread->sleep( 500); goto Loop_Again; } else { f_sprintf( (char *)szMsg, "Background indexing thread %u (index %u) -- unrecoverable error.", (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine); } } if( pReusableRec) { flmAssert( pReusableRec->getRefCount() == 1); pReusableRec->Release(); } if( bLimitedMode) { flmAssert( RC_OK( rc)); for (;;) { if( pThread->getShutdownFlag()) { break; } pThread->sleep( 1000); } } // Set the thread's app ID to 0, so that it will not // be found now that the thread is terminating (we don't // want flmBackgroundIndexGet to find the thread). pThread->setThreadAppId( 0); // Free the background index structure f_mutexLock( gv_FlmSysData.hShareMutex); f_free( &pBackgroundIx); pThread->setParm1( NULL); f_mutexUnlock( gv_FlmSysData.hShareMutex); return( rc); }
/******************************************************************************* Desc: Copies a database, including roll-forward log files. *******************************************************************************/ FLMEXP RCODE FLMAPI FlmDbCopy( const char * pszSrcDbName, const char * pszSrcDataDir, const char * pszSrcRflDir, const char * pszDestDbName, const char * pszDestDataDir, const char * pszDestRflDir, STATUS_HOOK fnStatusCallback, void * UserData) { RCODE rc = FERR_OK; FLMBYTE * pucLastCommittedLogHdr; HFDB hDb = HFDB_NULL; FDB * pDb; FLMBOOL bDbLocked = FALSE; FLMUINT uiDbVersion; // Make sure the destination database is closed if (RC_BAD( rc = FlmConfig( FLM_CLOSE_FILE, (void *)pszDestDbName, (void *)pszDestDataDir))) { goto Exit; } gv_FlmSysData.pFileHdlCache->closeUnusedFiles(); // Open the database so we can force a checkpoint. if (RC_BAD( rc = FlmDbOpen( pszSrcDbName, pszSrcDataDir, pszSrcRflDir, 0, NULL, &hDb))) { goto Exit; } pDb = (FDB *)hDb; // Need to lock the database, because we want to do a checkpoint // and then the copy immediately after without letting other // threads have the opportunity to get in and update the // database. if (RC_BAD( rc = FlmDbLock( hDb, FLM_LOCK_EXCLUSIVE, 0, FLM_NO_TIMEOUT))) { goto Exit; } bDbLocked = TRUE; // Force a checkpoint if (RC_BAD( rc = FlmDbCheckpoint( hDb, FLM_NO_TIMEOUT))) { goto Exit; } pucLastCommittedLogHdr = &pDb->pFile->ucLastCommittedLogHdr[ 0]; // Get the low and high RFL log file numbers from the log // header. uiDbVersion = pDb->pFile->FileHdr.uiVersionNum; // Once we get this far, we have exclusive access to the database // and we have forced a checkpoint. The database's contents are // guaranteed to be on disk at this point, and they will not // change. rc = flmCopyDb( uiDbVersion, pszSrcDbName, pszSrcDataDir, pszSrcRflDir, pszDestDbName, pszDestDataDir, pszDestRflDir, fnStatusCallback, UserData); Exit: // Unlock and close the database if (bDbLocked) { FlmDbUnlock( hDb); } if (hDb != HFDB_NULL) { (void)FlmDbClose( &hDb); (void)FlmConfig( FLM_CLOSE_FILE, (void *)pszSrcDbName, (void *)pszSrcDataDir); } return( rc); }
/******************************************************************** Desc: Loads the database with objects. *********************************************************************/ RCODE gigaLoadDatabase( void) { RCODE rc = NE_FLM_OK; FLMBOOL bTransActive = FALSE; FLMBOOL bCommitTrans = FALSE; FLMUINT uiObjsInTrans = 0; FLMUINT uiChar = 0; FLMUINT bSuspend = FALSE; FlmRecord * pNewRec = NULL; // Set cache size, if specified on command line. if( gv_uiCacheSize) { if( RC_BAD( rc = FlmSetHardMemoryLimit( 0, FALSE, 0, gv_uiCacheSize, 0))) { gigaOutputRcErr( "setting cache size", rc); goto Exit; } } // Set block cache percentage, if it is not default. if( gv_uiBlockCachePercentage != 50) { if( RC_BAD( rc = FlmConfig( FLM_BLOCK_CACHE_PERCENTAGE, (void *)gv_uiBlockCachePercentage, (void *)0))) { gigaOutputRcErr( "setting block cache percentage", rc); goto Exit; } } // Set the maximum and low dirty cache, if one was specified if( gv_uiMaxDirtyCache) { if( RC_BAD( rc = FlmConfig( FLM_MAX_DIRTY_CACHE, (void *)gv_uiMaxDirtyCache, (void *)gv_uiLowDirtyCache))) { gigaOutputRcErr( "setting maximum dirty cache", rc); goto Exit; } } // Set checkpoint interval, if one is specified. if( gv_uiCPInterval != 0xFFFFFFFF) { if( RC_BAD( rc = FlmConfig( FLM_MAX_CP_INTERVAL, (void *)gv_uiCPInterval, (void *)0))) { gigaOutputRcErr( "setting checkpoint interval", rc); goto Exit; } } // Enable/Disable direct I/O if( RC_BAD( rc = FlmConfig( FLM_DIRECT_IO_STATE, (void *)!gv_bDisableDirectIO, NULL))) { goto Exit; } // Create the database. (void)FlmDbRemove( gv_szDibName, gv_szDataDir, gv_szRflDir, TRUE); if( RC_BAD( rc = FlmDbCreate( gv_szDibName, gv_szDataDir, gv_szRflDir, NULL, gv_pszGigaDictionary, NULL, &gv_hDb))) { gigaOutputRcErr( "creating database", rc); goto Exit; } if( RC_BAD( rc = FlmDbConfig( gv_hDb, FDB_RFL_FOOTPRINT_SIZE, (void *)(512 * 1024 * 1024), NULL))) { goto Exit; } if( RC_BAD( rc = FlmDbConfig( gv_hDb, FDB_RBL_FOOTPRINT_SIZE, (void *)(512 * 1024 * 1024), NULL))) { goto Exit; } // Create the display gv_uiTotalLoaded = 0; gv_ui10SecTotal = 0; f_mutexLock( gv_hWindowMutex); FTXWinClear( gv_pWindow); f_mutexUnlock( gv_hWindowMutex); gigaOutputLabel( MAX_CACHE_ROW, "Maximum Cache Size (bytes)"); gigaOutputLabel( USED_CACHE_ROW, "Cache Used (bytes)"); gigaOutputLabel( ITEMS_CACHED_ROW, "Cache Used (items)"); gigaOutputLabel( DIRTY_CACHE_ROW, "Dirty Cache (bytes)"); gigaOutputLabel( LOG_CACHE_ROW, "Log Cache (bytes)"); gigaOutputLabel( FREE_CACHE_ROW, "Free Cache (bytes)"); gigaOutputLabel( CP_STATE_ROW, "Checkpoint State"); gigaUpdateMemInfo(); gigaOutputLabel( DB_NAME_ROW, "Database Name"); gigaOutputStr( DB_NAME_ROW, gv_szDibName); gigaOutputLabel( TOTAL_TO_LOAD_ROW, "Total To Load"); gigaOutputUINT( TOTAL_TO_LOAD_ROW, gv_uiTotalToLoad); gigaOutputLabel( TRANS_SIZE_ROW, "Transaction Size"); gigaOutputUINT( TRANS_SIZE_ROW, gv_uiTransSize); gigaOutputLabel( TOTAL_LOADED_ROW, "Total Loaded"); gigaOutputUINT( TOTAL_LOADED_ROW, gv_uiTotalLoaded); gigaOutputLabel( ADDS_PER_SEC_CURRENT, "Adds/Sec. (10 secs)"); gigaOutputUINT( ADDS_PER_SEC_CURRENT, 0); gigaOutputLabel( ADDS_PER_SEC_OVERALL, "Adds/Sec. (overall)"); gigaOutputUINT( ADDS_PER_SEC_OVERALL, 0); gigaOutputLabel( ELAPSED_TIME_ROW, "Elapsed Time"); gigaOutputStr( ELAPSED_TIME_ROW, "<none>"); if( RC_BAD( rc = gigaStartScreenThread())) { goto Exit; } gv_ui10SecStartTime = gv_uiStartTime = FLM_GET_TIMER(); gv_ui10Secs = FLM_SECS_TO_TIMER_UNITS( 10); gv_ui1Sec = FLM_SECS_TO_TIMER_UNITS( 1); for( ;;) { // See if we have been told to shut down, or if the user // has pressed escape. if( gv_bShutdown) { break; } // Every 127 objects, see if character was pressed and update // count on screen. if( (gv_uiTotalLoaded & 0x7F) == 0) { f_yieldCPU(); if( (uiChar = gigaSeeIfQuit()) != 0) { if( uiChar == FKB_ESCAPE) { break; } else if( uiChar == 's' || uiChar == 'S') { bSuspend = TRUE; } } // Check for other keyboard options } else if( (gv_uiTotalLoaded & 0x7) == 0) { FLMUINT uiElapsedTime; FLMUINT uiCurrTime; uiCurrTime = FLM_GET_TIMER(); // If at least 10 seconds have elapsed, redisplay the average // rate values. if( (uiElapsedTime = FLM_ELAPSED_TIME( uiCurrTime, gv_ui10SecStartTime)) >= gv_ui10Secs) { gigaUpdateLoadTimes(); } } // Start a transaction, if one is not going. if( !bTransActive) { if( bSuspend) { uiChar = gigaGetInput( "Load suspended, press any character to continue loading: ", NULL); bSuspend = FALSE; } if( RC_BAD( rc = gigaStartTrans())) { goto Exit; } bTransActive = TRUE; bCommitTrans = FALSE; uiObjsInTrans = 0; } // Increment the load counters and determine if this will be the // last object of the transaction. gv_uiTotalLoaded++; uiObjsInTrans++; if( uiObjsInTrans == gv_uiTransSize || gv_uiTotalLoaded == gv_uiTotalToLoad) { bCommitTrans = TRUE; } // Create a new object. if( RC_BAD( rc = gigaMakeNewRecord( &pNewRec))) { goto Exit; } if( RC_BAD( rc = FlmRecordAdd( gv_hDb, FLM_DATA_CONTAINER, NULL, pNewRec, FLM_DONT_INSERT_IN_CACHE))) { goto Exit; } // Commit when we reach the transaction size or the total to load. // NOTE: The bCommitTrans flag is set above. if( bCommitTrans) { if( RC_BAD( rc = gigaCommitTrans())) { goto Exit; } bTransActive = FALSE; } // See if we are done. if( gv_uiTotalLoaded == gv_uiTotalToLoad) { flmAssert( !bTransActive); break; } } Exit: if( pNewRec) { pNewRec->Release(); } if( bTransActive) { (void)FlmDbTransAbort( gv_hDb); } if( gv_hDb != HFDB_NULL) { FlmDbCheckpoint( gv_hDb, FLM_NO_TIMEOUT); gigaStopScreenThread(); FlmDbClose( &gv_hDb); // This will cause us to wait for the last checkpoint // to finish. (void)FlmConfig( FLM_CLOSE_FILE, (void *)gv_szDibName, (void *)gv_szDataDir); } gigaUpdateLoadTimes(); gigaStopScreenThread(); f_threadDestroy( &gv_pIxManagerThrd); return( rc); }