/************************************************************************ Desc : Outputs an update event callback. *************************************************************************/ void flmUpdEventCallback( FDB * pDb, FEventType eEventType, HFDB hDb, RCODE rc, FLMUINT uiDrn, FLMUINT uiContainer, FlmRecord * pNewRecord, FlmRecord * pOldRecord) { FLM_UPDATE_EVENT UpdEvent; UpdEvent.uiThreadId = f_threadId(); UpdEvent.hDb = hDb; UpdEvent.uiTransID = (FLMUINT)( (pDb->uiTransType) ? pDb->LogHdr.uiCurrTransID : 0); UpdEvent.rc = rc; UpdEvent.uiDrn = uiDrn; UpdEvent.uiContainer = uiContainer; UpdEvent.pNewRecord = pNewRecord; UpdEvent.pOldRecord = pOldRecord; flmDoEventCallback( F_EVENT_UPDATES, eEventType, &UpdEvent, NULL); }
/**************************************************************************** Desc: Index a set of documents or until time runs out. ****************************************************************************/ RCODE F_Db::indexSetOfRows( FLMUINT uiIndexNum, FLMUINT64 ui64StartRowId, FLMUINT64 ui64EndRowId, IF_IxStatus * pIxStatus, IF_IxClient * pIxClient, SFLM_INDEX_STATUS * pIndexStatus, FLMBOOL * pbHitEnd, IF_Thread * pThread) { RCODE rc = NE_SFLM_OK; FLMUINT64 ui64RowId; FLMUINT64 ui64LastRowId = 0; F_INDEX * pIndex = NULL; F_TABLE * pTable; IF_LockObject * pDatabaseLockObj = m_pDatabase->m_pDatabaseLockObj; FLMBOOL bHitEnd = FALSE; FLMUINT uiCurrTime; FLMUINT uiLastStatusTime = 0; FLMUINT uiStartTime; FLMUINT uiMinTU; FLMUINT uiStatusIntervalTU; FLMUINT64 ui64RowsProcessed = 0; FLMBOOL bRelinquish = FALSE; FLMBYTE ucKey[ FLM_MAX_NUM_BUF_SIZE]; FLMUINT uiKeyLen; void * pvTmpPoolMark = m_tempPool.poolMark(); F_Btree * pbtree = NULL; FLMBOOL bNeg; FLMUINT uiBytesProcessed; F_Row * pRow = NULL; uiMinTU = FLM_MILLI_TO_TIMER_UNITS( 500); uiStatusIntervalTU = FLM_SECS_TO_TIMER_UNITS( 10); uiStartTime = FLM_GET_TIMER(); if (RC_BAD( rc = krefCntrlCheck())) { goto Exit; } pIndex = m_pDict->getIndex( uiIndexNum); flmAssert( pIndex); flmAssert( !(pIndex->uiFlags & IXD_SUSPENDED)); // Get a btree if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbtree))) { goto Exit; } pTable = m_pDict->getTable( pIndex->uiTableNum); flmAssert( pTable); if (RC_BAD( rc = pbtree->btOpen( this, &pTable->lfInfo, FALSE, TRUE))) { goto Exit; } uiKeyLen = sizeof( ucKey); if (RC_BAD( rc = flmNumber64ToStorage( ui64StartRowId, &uiKeyLen, ucKey, FALSE, TRUE))) { goto Exit; } if( RC_BAD( rc = pbtree->btLocateEntry( ucKey, sizeof( ucKey), &uiKeyLen, FLM_INCL))) { if (rc == NE_SFLM_EOF_HIT || rc == NE_SFLM_NOT_FOUND) { rc = NE_SFLM_OK; bHitEnd = TRUE; goto Commit_Keys; } goto Exit; } for (;;) { // See what row we're on if (RC_BAD( rc = flmCollation2Number( uiKeyLen, ucKey, &ui64RowId, &bNeg, &uiBytesProcessed))) { goto Exit; } if (ui64RowId > ui64EndRowId) { break; } if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( this, pIndex->uiTableNum, ui64RowId, &pRow))) { goto Exit; } if (RC_BAD( rc = buildKeys( pIndex, pTable, pRow, TRUE, NULL))) { goto Exit; } // See if there is an indexing callback if (pIxClient) { if (RC_BAD( rc = pIxClient->doIndexing( this, uiIndexNum, pIndex->uiTableNum, pRow))) { goto Exit; } } ui64LastRowId = ui64RowId; ui64RowsProcessed++; if (pIndexStatus) { pIndexStatus->ui64RowsProcessed++; pIndexStatus->ui64LastRowIndexed = ui64LastRowId; } // Get the current time uiCurrTime = FLM_GET_TIMER(); // Break out if someone is waiting for an update transaction. if (pThread) { if (pThread->getShutdownFlag()) { bRelinquish = TRUE; break; } if (pDatabaseLockObj->getWaiterCount()) { // See if our minimum run time has elapsed if (FLM_ELAPSED_TIME( uiCurrTime, uiStartTime) >= uiMinTU) { if (ui64RowsProcessed < 50) { // If there are higher priority waiters in the lock queue, // we want to relinquish. if (pDatabaseLockObj->haveHigherPriorityWaiter( FLM_BACKGROUND_LOCK_PRIORITY)) { bRelinquish = TRUE; break; } } else { bRelinquish = TRUE; break; } } } else { // Even if no one has requested a lock for a long time, we // still want to periodically commit our transaction so // we won't lose more than uiMaxCPInterval timer units worth // of work if we crash. We will run until we exceed the checkpoint // interval and we see that someone (the checkpoint thread) is // waiting for the write lock. if (FLM_ELAPSED_TIME( uiCurrTime, uiStartTime) > gv_SFlmSysData.uiMaxCPInterval && m_pDatabase->m_pWriteLockObj->getWaiterCount()) { bRelinquish = TRUE; break; } } } if (FLM_ELAPSED_TIME( uiCurrTime, uiLastStatusTime) >= uiStatusIntervalTU) { uiLastStatusTime = uiCurrTime; if( pIxStatus) { if( RC_BAD( rc = pIxStatus->reportIndex( ui64LastRowId))) { goto Exit; } } // Send indexing completed event notification if( gv_SFlmSysData.EventHdrs[ SFLM_EVENT_UPDATES].pEventCBList) { flmDoEventCallback( SFLM_EVENT_UPDATES, SFLM_EVENT_INDEXING_PROGRESS, this, f_threadId(), 0, uiIndexNum, ui64LastRowId, NE_SFLM_OK); } // Log a progress message flmLogIndexingProgress( uiIndexNum, ui64LastRowId); } // Need to go to the next row. if( RC_BAD( rc = pbtree->btNextEntry( ucKey, sizeof( ucKey), &uiKeyLen))) { if (rc == NE_SFLM_EOF_HIT) { rc = NE_SFLM_OK; bHitEnd = TRUE; break; } goto Exit; } } Commit_Keys: if (RC_BAD( rc = keysCommit( TRUE))) { goto Exit; } // If at the end, change index state. if (bHitEnd) { if (RC_BAD( rc = setIxStateInfo( uiIndexNum, 0, 0))) { goto Exit; } // setIxStateInfo may have changed to a new dictionary, so pIxd is no // good after this point pIndex = NULL; } else if (ui64RowsProcessed) { if (RC_BAD( rc = setIxStateInfo( uiIndexNum, ui64LastRowId, IXD_OFFLINE))) { goto Exit; } // setIxStateInfo may have changed to a new dictionary, so pIndex is no // good after this point pIndex = NULL; } // Log the rows that were indexed, if any if (ui64LastRowId) { if (RC_BAD( rc = m_pDatabase->m_pRfl->logIndexSet( this, uiIndexNum, ui64StartRowId, ui64LastRowId))) { goto Exit; } } Exit: // We want to make one last call if we are in the foreground or if // we actually did some indexing. if (gv_SFlmSysData.EventHdrs[ SFLM_EVENT_UPDATES].pEventCBList) { flmDoEventCallback( SFLM_EVENT_UPDATES, SFLM_EVENT_INDEXING_PROGRESS, this, f_threadId(), 0, uiIndexNum, (FLMUINT64)(bHitEnd ? (FLMUINT64)0 : ui64LastRowId), NE_SFLM_OK); } flmLogIndexingProgress( uiIndexNum, (FLMUINT64)(bHitEnd ? (FLMUINT64)0 : ui64LastRowId)); if (pIxStatus) { (void) pIxStatus->reportIndex( ui64LastRowId); } if (pbHitEnd) { *pbHitEnd = bHitEnd; } krefCntrlFree(); m_tempPool.poolReset( pvTmpPoolMark); if (pbtree) { gv_SFlmSysData.pBtPool->btpReturnBtree( &pbtree); } if (pRow) { pRow->ReleaseRow(); } return( rc); }
/**************************************************************************** Desc: Thread that will build an index in the background. Caller will create a pDb to use. This pDb must be freed at the conclusion of the routine. ****************************************************************************/ FSTATIC RCODE FLMAPI flmBackgroundIndexBuildThrd( IF_Thread * pThread) { RCODE rc = FERR_OK; IXD * pIxd; F_BKGND_IX * pBackgroundIx = (F_BKGND_IX *)pThread->getParm1(); FLMBOOL bStartedTrans; FLMBOOL bDbInitialized; FLMUINT uiContainerNum; FLMUINT uiFirstDrn; FLMUINT uiIndexNum; FDB * pDb = NULL; FLMBOOL bForcedShutdown = FALSE; FLMBOOL bHitEnd; FINDEX_STATUS savedIxStatus; FlmRecord * pReusableRec = NULL; char szMsg[ 128]; FLMINT iErrorLine = 0; FLMBOOL bLimitedMode = FALSE; pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING); if( (pReusableRec = f_new FlmRecord) != NULL) { if( RC_BAD( pReusableRec->preallocSpace( 512, 1024 * 64))) { pReusableRec->Release(); pReusableRec = NULL; } } Loop_Again: rc = FERR_OK; uiIndexNum = pBackgroundIx->indexStatus.uiIndexNum; flmAssert( pThread->getThreadAppId() == uiIndexNum); bDbInitialized = FALSE; bStartedTrans = FALSE; pDb = NULL; // We could loop forever on flmOpenFile errors, check if we should exit. if( pThread->getShutdownFlag()) { bForcedShutdown = TRUE; goto Exit; } if( RC_BAD( rc = flmOpenFile( pBackgroundIx->pFile, NULL, NULL, NULL, 0, TRUE, NULL, NULL, pBackgroundIx->pFile->pszDbPassword, &pDb))) { // If the file is being closed, this is not an error. if( pBackgroundIx->pFile->uiFlags & DBF_BEING_CLOSED) { bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } flmAssert( pDb->pSFileHdl); bDbInitialized = TRUE; if (RC_BAD( rc = fdbInit( pDb, FLM_NO_TRANS, 0, 0, &bStartedTrans))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } flmAssert( !bStartedTrans); pDb->uiFlags |= FDB_BACKGROUND_INDEXING; for(;;) { // Set the thread's status pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); // See if we should shut down. if( pThread->getShutdownFlag()) { bForcedShutdown = TRUE; break; } // Obtain the file lock flmAssert( !(pDb->uiFlags & FDB_HAS_FILE_LOCK)); if( RC_BAD( rc = pDb->pFile->pFileLockObj->lock( pDb->hWaitSem, TRUE, FLM_NO_TIMEOUT, FLM_BACKGROUND_LOCK_PRIORITY, pDb->pDbStats ? &pDb->pDbStats->LockStats : NULL))) { if( rc == FERR_IO_FILE_LOCK_ERR) { // This would only happen if we were signaled to shut down. // So, it's ok to exit flmAssert( pThread->getShutdownFlag()); bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } // The lock needs to be marked as implicit so that flmCommitDbTrans // will unlock the file and allow the next update transaction to // begin before all writes are complete. pDb->uiFlags |= (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); // If there are higher priority waiters in the lock queue, // or we are being told to shut down, we want to relinquish. if( pThread->getShutdownFlag() || pDb->pFile->pFileLockObj->haveHigherPriorityWaiter( FLM_BACKGROUND_LOCK_PRIORITY)) { if (RC_BAD( rc = pDb->pFile->pFileLockObj->unlock())) { iErrorLine = (FLMINT)__LINE__; goto Exit; } pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); continue; } // Start an update transaction if( RC_BAD( rc = flmBeginDbTrans( pDb, FLM_UPDATE_TRANS, FLM_NO_TIMEOUT, FLM_DONT_POISON_CACHE))) { if( rc == FERR_IO_FILE_LOCK_ERR) { // This would only happen if we were signaled to shut down. // So, it's ok to exit flmAssert( pThread->getShutdownFlag()); bForcedShutdown = TRUE; rc = FERR_OK; } else { iErrorLine = (FLMINT)__LINE__; } goto Exit; } bStartedTrans = TRUE; if( RC_BAD( rc = fdictGetIndex( pDb->pDict, pDb->pFile->bInLimitedMode, uiIndexNum, NULL, &pIxd, TRUE))) { // Index may have been deleted by another transaction, or // there may have been some other error. iErrorLine = (FLMINT)__LINE__; goto Exit; } // If we're running in limited mode, then we can't mess with encrypted // indexes. On the other hand, since the index is marked as offline, // but not suspended, this thread has to exist, or else it will cause // all kinds of problems elsewhere. So, in such a case, we will simply // sleep in an inifinite loop until the shutdown flag is set. // (We consider this acceptable becase running in limited mode is not // the norm, and Flaim probably won't be up for very long in this mode.) if( pDb->pFile->bInLimitedMode && pIxd->uiEncId) { bLimitedMode = TRUE; goto Exit; } // Look up the tracker info to determine where we need to being indexing if (RC_BAD( rc = flmGetIxTrackerInfo( pDb, pBackgroundIx->indexStatus.uiIndexNum, &uiContainerNum, &uiFirstDrn, NULL, &pBackgroundIx->indexStatus.bSuspended))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } // If the index is suspended, this thread should have been // shut down. The suspending thread will keep the database // locked until we exit. So, if we have the database locked, // the index better not be suspended. flmAssert( !pBackgroundIx->indexStatus.bSuspended && !(pIxd->uiFlags & IXD_SUSPENDED)); if (pIxd->uiContainerNum) { uiContainerNum = pIxd->uiContainerNum; if( uiFirstDrn == DRN_LAST_MARKER) { goto Exit; } } else { if( uiFirstDrn == DRN_LAST_MARKER && uiContainerNum == 0xFFFFFFFF) { goto Exit; } else { // The container number from the tracker record // may not be a real container. // Determine what the next actual container number is. if (uiContainerNum != FLM_DATA_CONTAINER) { while( uiContainerNum < pDb->pDict->uiIttCnt) { ITT * pItt = &pDb->pDict->pIttTbl [uiContainerNum]; if (ITT_IS_CONTAINER( pItt)) { break; } else { uiContainerNum++; } } if (uiContainerNum >= pDb->pDict->uiIttCnt) { uiContainerNum = FLM_DATA_CONTAINER; } } } } // Setup the DRN range we want to index. uiFirstDrn++; flmAssert( pIxd->uiLastDrnIndexed == uiFirstDrn - 1); // Set the thread's status pThread->setThreadStatus( "Indexing %u:%u", (unsigned)uiContainerNum, (unsigned)uiFirstDrn); // Read and index up to the highest drn (or record higher than uiEndDrn) // or until time runs out. The 500 is millisecs to take for the transaction. f_memcpy( &savedIxStatus, &pBackgroundIx->indexStatus, sizeof( FINDEX_STATUS)); if( RC_BAD( rc = flmIndexSetOfRecords( pDb, uiIndexNum, uiContainerNum, uiFirstDrn, DRN_LAST_MARKER, NULL, NULL, NULL, NULL, &pBackgroundIx->indexStatus, &bHitEnd, pThread, pReusableRec))) { // Lock the mutex while copying the saved index status back to // the main index status so that someone requesting the index status // won't see the status while the memcpy is in progress. f_mutexLock( gv_FlmSysData.hShareMutex); f_memcpy( &pBackgroundIx->indexStatus, &savedIxStatus, sizeof( FINDEX_STATUS)); f_mutexUnlock( gv_FlmSysData.hShareMutex); iErrorLine = (FLMINT)__LINE__; goto Exit; } if( pBackgroundIx->indexStatus.uiRecordsProcessed - savedIxStatus.uiRecordsProcessed) { if( RC_BAD( rc = pDb->pFile->pRfl->logIndexSet( uiIndexNum, uiContainerNum, uiFirstDrn, pBackgroundIx->indexStatus.uiLastRecordIdIndexed))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } } // Commit the transaction (even if we didn't do any indexing work). if( RC_BAD( rc = flmCommitDbTrans( pDb, 0, FALSE))) { iErrorLine = (FLMINT)__LINE__; goto Exit; } bStartedTrans = FALSE; pBackgroundIx->indexStatus.uiTransactions++; if( bHitEnd) { // flmIndexSetOfRecords brought the index on-line if( gv_FlmSysData.UpdateEvents.pEventCBList) { flmDoEventCallback( F_EVENT_UPDATES, F_EVENT_INDEXING_COMPLETE, (void *)uiIndexNum, (void *)0); } // Log a message flmLogIndexingProgress( uiIndexNum, 0); break; } } Exit: pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); if( bStartedTrans) { (void)flmAbortDbTrans( pDb); bStartedTrans = FALSE; } if( pDb && pDb->uiFlags & FDB_HAS_FILE_LOCK) { (void)pDb->pFile->pFileLockObj->unlock(); pDb->uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); } if( bDbInitialized) { fdbExit( pDb); bDbInitialized = FALSE; } if( pDb) { (void)FlmDbClose( (HFDB *) &pDb); } if( RC_BAD(rc) && !bForcedShutdown) { if (rc == FERR_MEM || rc == FERR_IO_DISK_FULL || rc == FERR_MUST_WAIT_CHECKPOINT) { // Log the error f_sprintf( (char *)szMsg, "Background indexing thread %u (index %u)", (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine); // Sleep a half second and try again. pThread->sleep( 500); goto Loop_Again; } else { f_sprintf( (char *)szMsg, "Background indexing thread %u (index %u) -- unrecoverable error.", (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); flmLogError( rc, (char *)szMsg, __FILE__, iErrorLine); } } if( pReusableRec) { flmAssert( pReusableRec->getRefCount() == 1); pReusableRec->Release(); } if( bLimitedMode) { flmAssert( RC_OK( rc)); for (;;) { if( pThread->getShutdownFlag()) { break; } pThread->sleep( 1000); } } // Set the thread's app ID to 0, so that it will not // be found now that the thread is terminating (we don't // want flmBackgroundIndexGet to find the thread). pThread->setThreadAppId( 0); // Free the background index structure f_mutexLock( gv_FlmSysData.hShareMutex); f_free( &pBackgroundIx); pThread->setParm1( NULL); f_mutexUnlock( gv_FlmSysData.hShareMutex); return( rc); }