/**************************************************************************** Desc: Read the block from disk. ****************************************************************************/ RCODE F_BtreeBlk::readBlk( IF_FileHdl * pFileHdl, FLMUINT uiBlkAddr) { RCODE rc = NE_FLM_OK; FLMUINT uiBytesRead; if (RC_BAD( rc = pFileHdl->read( uiBlkAddr * DYNSSET_BLOCK_SIZE, DYNSSET_BLOCK_SIZE, m_pucBlkBuf, &uiBytesRead))) { RC_UNEXPECTED_ASSERT( rc); goto Exit; } if (blkAddr() != uiBlkAddr) { // Most likely a coding error rather than an I/O error. goto Exit; } Exit: return( rc); }
/**************************************************************************** Desc: Thread that will delete block chains from deleted indexes and tables in the background. ****************************************************************************/ RCODE SQFAPI F_Database::maintenanceThread( IF_Thread * pThread) { RCODE rc = NE_SFLM_OK; F_Database * pDatabase = (F_Database *)pThread->getParm1(); F_Db * pDb; F_Row * pRow; FLMUINT64 ui64MaintRowId; FLMBOOL bStartedTrans; FLMBOOL bShutdown; F_DbSystem * pDbSystem; FSTableCursor * pTableCursor; FLMUINT uiBlkAddress; FLMBOOL bIsNull; FLMUINT uiBlocksToFree; FLMUINT uiBlocksFreed; Retry: rc = NE_SFLM_OK; pDb = NULL; pRow = NULL; bStartedTrans = FALSE; bShutdown = FALSE; pDbSystem = NULL; pTableCursor = NULL; if( (pDbSystem = f_new F_DbSystem) == NULL) { rc = RC_SET( NE_SFLM_MEM); goto Exit; } pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING); if( RC_BAD( rc = pDbSystem->internalDbOpen( pDatabase, &pDb))) { // If the file is being closed, this is not an error. if( pDatabase->getFlags() & DBF_BEING_CLOSED) { rc = NE_SFLM_OK; bShutdown = TRUE; } goto Exit; } pDbSystem->Release(); pDbSystem = NULL; if ((pTableCursor = f_new FSTableCursor) == NULL) { rc = RC_SET( NE_SFLM_MEM); goto Exit; } for( ;;) { pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); if( RC_BAD( rc = pDb->beginBackgroundTrans( pThread))) { goto Exit; } bStartedTrans = TRUE; pTableCursor->resetCursor(); if (RC_BAD( rc = pTableCursor->setupRange( pDb, SFLM_TBLNUM_BLOCK_CHAINS, 1, FLM_MAX_UINT64, FALSE))) { goto Exit; } // Free up to 25 blocks per transaction. uiBlocksToFree = 25; while (uiBlocksToFree) { if (RC_BAD( rc = pTableCursor->nextRow( pDb, &pRow, &ui64MaintRowId))) { if (rc != NE_SFLM_EOF_HIT) { RC_UNEXPECTED_ASSERT( rc); goto Exit; } rc = NE_SFLM_OK; break; } if (RC_BAD( rc = pRow->getUINT( pDb, SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS, &uiBlkAddress, &bIsNull))) { goto Exit; } if (bIsNull) { rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); goto Exit; } if( RC_BAD( rc = pDb->maintBlockChainFree( ui64MaintRowId, uiBlkAddress, uiBlocksToFree, 0, &uiBlocksFreed))) { goto Exit; } uiBlocksToFree -= uiBlocksFreed; } bStartedTrans = FALSE; if( RC_BAD( rc = pDb->commitTrans( 0, FALSE))) { goto Exit; } pThread->setThreadStatus( FLM_THREAD_STATUS_SLEEPING); f_semWait( pDatabase->m_hMaintSem, F_WAITFOREVER); if (pThread->getShutdownFlag()) { bShutdown = TRUE; goto Exit; } } Exit: pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); if (pDbSystem) { pDbSystem->Release(); } if (pRow) { pRow->ReleaseRow(); } if( bStartedTrans) { pDb->abortTrans(); } if (pDb) { pDb->Release(); pDb = NULL; } if (!bShutdown) { flmAssert( RC_BAD( rc)); f_sleep( 250); f_semSignal( pDatabase->m_hMaintSem); goto Retry; } return( rc); }
/**************************************************************************** Desc: Move first half of entries into new block. Reset previous block to point to new block. Add new last entry in new block to parent. Fixup prev/next linkages. ****************************************************************************/ RCODE F_BtreeBlk::split( F_BtreeRoot * pRoot, FLMBYTE * pucCurEntry, // (in) Contains entry to insert FLMUINT uiCurBlkAddr, // (in) Blk addr if non-leaf FLMBYTE * pucParentEntry, // (out) Entry to insert into parent. FLMUINT * puiNewBlkAddr) // (out) New blk addr to insert into parent. { RCODE rc = NE_FLM_OK; F_BtreeBlk * pPrevBlk; F_BtreeBlk * pNewBlk = NULL; FLMBYTE * pucEntry = NULL; FLMBYTE * pucMidEntry; FLMBYTE * pucChildAddr; FLMUINT uiChildAddr; FLMUINT uiPrevBlkAddr; FLMUINT uiMid; FLMUINT uiPos; FLMUINT uiMoveBytes; FLMBOOL bInserted = FALSE; // Allocate a new block for the split. if (RC_BAD( rc = pRoot->newBlk( &pNewBlk, blkType() ))) { goto Exit; } pNewBlk->AddRef(); // Pin the block - may get flushed out. // the last half into the new block, but that would force the parent // entry to be changed. This may take a little longer, but it is much // more easier to code. // Call search entry once just to setup for insert. (void) pNewBlk->searchEntry( ENTRY_POS( 0)); // get the count and move more then half into the new block. uiMid = (entryCount() + 5) >> 1; uiChildAddr = FBTREE_END; for (uiPos = 0; uiPos < uiMid; uiPos++) { pucEntry = ENTRY_POS( uiPos); if (blkType() != ACCESS_BTREE_LEAF) { pucChildAddr = pucEntry + m_uiEntrySize; uiChildAddr = (FLMUINT)FB2UD(pucChildAddr); } // m_uiPosition automatically gets incremented. if (RC_BAD( rc = pNewBlk->insertEntry( pucEntry, uiChildAddr))) { RC_UNEXPECTED_ASSERT( rc); goto Exit; } } if (m_uiPosition < uiMid) { // Insert this entry now bInserted = TRUE; (void) pNewBlk->searchEntry( pucCurEntry); if (RC_BAD( rc = pNewBlk->insertEntry( pucCurEntry, uiCurBlkAddr))) { goto Exit; } } // Let caller insert into parent entry. This rids us of recursion. f_memcpy( pucParentEntry, pucEntry, m_uiEntrySize); // Move the rest down pucEntry = ENTRY_POS( 0); pucMidEntry = ENTRY_POS( uiMid); entryCount( entryCount() - uiMid); uiMoveBytes = entryCount() * (m_uiEntrySize + m_uiEntryOvhd); flmAssert( uiMoveBytes < DYNSSET_BLOCK_SIZE - sizeof( FixedBlkHdr)); f_memmove( pucEntry, pucMidEntry, uiMoveBytes); if( !bInserted) { // m_uiPosition -= uiMid; (void) searchEntry( pucCurEntry); if (RC_BAD( rc = insertEntry( pucCurEntry, uiCurBlkAddr))) { goto Exit; } } // VISIT: Could position stack to point to current element to insert. // Fixup the prev/next block linkages. if (prevBlk() != FBTREE_END) { if (RC_BAD( rc = pRoot->readBlk( prevBlk(), blkType(), &pPrevBlk ))) { goto Exit; } pPrevBlk->nextBlk( pNewBlk->blkAddr()); uiPrevBlkAddr = pPrevBlk->blkAddr(); } else { uiPrevBlkAddr = FBTREE_END; } pNewBlk->prevBlk( uiPrevBlkAddr); pNewBlk->nextBlk( blkAddr()); prevBlk( pNewBlk->blkAddr()); *puiNewBlkAddr = pNewBlk->blkAddr(); Exit: if (pNewBlk) { pNewBlk->Release(); } return( rc); }
/**************************************************************************** Desc: Returns disk space usage for the database ****************************************************************************/ RCODE XFLAPI F_Db::getDiskSpaceUsage( FLMUINT64 * pui64DataSize, FLMUINT64 * pui64RollbackSize, FLMUINT64 * pui64RflSize) { RCODE rc = NE_XFLM_OK; FLMBOOL bStartedTrans = FALSE; FLMUINT uiEndAddress; FLMUINT uiLastFileNumber; FLMUINT64 ui64LastFileSize; char szTmpName [F_PATH_MAX_SIZE]; char szRflDir [F_PATH_MAX_SIZE]; IF_FileHdl * pFileHdl = NULL; IF_DirHdl * pDirHdl = NULL; if (m_eTransType == XFLM_READ_TRANS) { rc = RC_SET( NE_XFLM_ILLEGAL_TRANS_OP); goto Exit; } else if (m_eTransType != XFLM_NO_TRANS) { if (!okToCommitTrans()) { rc = RC_SET( NE_XFLM_ABORT_TRANS); goto Exit; } } else { if (RC_BAD( rc = beginTrans( XFLM_UPDATE_TRANS))) { goto Exit; } bStartedTrans = TRUE; } // See if they want the database files sizes. if (pui64DataSize) { uiEndAddress = m_uiLogicalEOF; uiLastFileNumber = FSGetFileNumber( uiEndAddress); // Last file number better be in the proper range. flmAssert( uiLastFileNumber >= 1 && uiLastFileNumber <= MAX_DATA_BLOCK_FILE_NUMBER); // Get the actual size of the last file. if (RC_BAD( rc = m_pSFileHdl->getFileSize( uiLastFileNumber, &ui64LastFileSize))) { if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) { if (uiLastFileNumber > 1) { rc = NE_XFLM_OK; ui64LastFileSize = 0; } else { // Should always be a data file #1 RC_UNEXPECTED_ASSERT( rc); goto Exit; } } else { goto Exit; } } // One of two situations exists with respect to the last // file: 1) it has not been fully written out yet (blocks // are still cached, or 2) it has been written out and // extended beyond what the logical EOF shows. We want // the larger of these two possibilities. if (FSGetFileOffset( uiEndAddress) > ui64LastFileSize) { ui64LastFileSize = FSGetFileOffset( uiEndAddress); } if (uiLastFileNumber == 1) { // Only one file - use last file's size. *pui64DataSize = ui64LastFileSize; } else { // Size is the sum of full size for all files except the last one, // plus the calculated (or actual) size of the last one. (*pui64DataSize) = (FLMUINT64)(uiLastFileNumber - 1) * (FLMUINT64)m_pDatabase->m_uiMaxFileSize + ui64LastFileSize; } } // See if they want the rollback files sizes. if (pui64RollbackSize) { uiEndAddress = (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RblEOF; uiLastFileNumber = FSGetFileNumber( uiEndAddress); // Last file number better be in the proper range. flmAssert( !uiLastFileNumber || (uiLastFileNumber >= FIRST_LOG_BLOCK_FILE_NUMBER && uiLastFileNumber <= MAX_LOG_BLOCK_FILE_NUMBER)); // Get the size of the last file number. if (RC_BAD( rc = m_pSFileHdl->getFileSize( uiLastFileNumber, &ui64LastFileSize))) { if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) { if (uiLastFileNumber) { rc = NE_XFLM_OK; ui64LastFileSize = 0; } else { // Should always have rollback file #0 RC_UNEXPECTED_ASSERT( rc); goto Exit; } } else { goto Exit; } } // If the EOF offset for the last file is greater than the // actual file size, use it instead of the actual file size. if (FSGetFileOffset( uiEndAddress) > ui64LastFileSize) { ui64LastFileSize = FSGetFileOffset( uiEndAddress); } // Special case handling here because rollback file numbers start with // zero and then skip to a file number that is one beyond the // highest data file number - so the calculation for file size needs // to account for this. if (!uiLastFileNumber) { *pui64RollbackSize = ui64LastFileSize; } else { FLMUINT uiFirstLogFileNum = FIRST_LOG_BLOCK_FILE_NUMBER; // Add full size of file zero plus a full size for every file // except the last one. (*pui64RollbackSize) = (FLMUINT64)(uiLastFileNumber - uiFirstLogFileNum + 1) * (FLMUINT64)m_pDatabase->m_uiMaxFileSize + ui64LastFileSize; } } // See if they want the roll-forward log file sizes. if (pui64RflSize) { char * pszDbFileName = m_pDatabase->m_pszDbPath; *pui64RflSize = 0; // Scan the RFL directory for // RFL files. The call below to rflGetDirAndPrefix is done // to get the prefix. It will not return the correct // RFL directory name, because we are passing in a NULL // RFL directory path (which may or may not be correct). // That's OK, because we get the RFL directory directly // from the F_Rfl object anyway. if (RC_BAD( rc = rflGetDirAndPrefix( pszDbFileName, NULL, szRflDir))) { goto Exit; } // We need to get the RFL directory from the F_Rfl object. m_pDatabase->lockMutex(); f_strcpy( szRflDir, m_pDatabase->m_pRfl->getRflDirPtr()); m_pDatabase->unlockMutex(); // See if the directory exists. If not, we are done. if (gv_XFlmSysData.pFileSystem->isDir( szRflDir)) { // Open the directory and scan for RFL files. if (RC_BAD( rc = gv_XFlmSysData.pFileSystem->openDir( szRflDir, "*", &pDirHdl))) { goto Exit; } for (;;) { if (RC_BAD( rc = pDirHdl->next())) { if (rc == NE_FLM_IO_NO_MORE_FILES) { rc = NE_XFLM_OK; break; } else { goto Exit; } } pDirHdl->currentItemPath( szTmpName); // If the item looks like an RFL file name, get // its size. if (!pDirHdl->currentItemIsDir() && rflGetFileNum( szTmpName, &uiLastFileNumber)) { // Open the file and get its size. if (RC_BAD( rc = gv_XFlmSysData.pFileSystem->openFile( szTmpName, gv_XFlmSysData.uiFileOpenFlags, &pFileHdl))) { if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) { rc = NE_XFLM_OK; ui64LastFileSize = 0; } else { goto Exit; } } else { if (RC_BAD( rc = pFileHdl->size( &ui64LastFileSize))) { goto Exit; } } if (pFileHdl) { pFileHdl->Release(); pFileHdl = NULL; } (*pui64RflSize) += ui64LastFileSize; } } } } Exit: if (pFileHdl) { pFileHdl->Release(); } if (pDirHdl) { pDirHdl->Release(); } if (bStartedTrans) { abortTrans(); } return( rc); }
/**************************************************************************** Desc: Try to perform a checkpoint on the database. Returns TRUE if we need to terminate. ****************************************************************************/ FLMBOOL F_Database::tryCheckpoint( IF_Thread * pThread, CP_INFO * pCPInfo) { RCODE rc = NE_XFLM_OK; FLMBOOL bTerminate = FALSE; FLMBOOL bForceCheckpoint; FLMINT iForceReason; FLMUINT uiCurrTime; XFLM_DB_STATS * pDbStats; // See if we should terminate the thread. if (pThread->getShutdownFlag()) { // Set terminate flag to TRUE and then see if // we have been set up to do one final checkpoint // to flush dirty buffers to disk. bTerminate = TRUE; } // Determine if we need to force a checkpoint. bForceCheckpoint = FALSE; iForceReason = 0; uiCurrTime = (FLMUINT)FLM_GET_TIMER(); if (bTerminate) { bForceCheckpoint = TRUE; iForceReason = XFLM_CP_SHUTTING_DOWN_REASON; } else if (!m_pRfl->seeIfRflVolumeOk() || RC_BAD( m_CheckpointRc)) { bForceCheckpoint = TRUE; iForceReason = XFLM_CP_RFL_VOLUME_PROBLEM; } else if ((FLM_ELAPSED_TIME( uiCurrTime, m_uiLastCheckpointTime) >= gv_XFlmSysData.uiMaxCPInterval) || (!gv_XFlmSysData.uiMaxCPInterval)) { bForceCheckpoint = TRUE; iForceReason = XFLM_CP_TIME_INTERVAL_REASON; } if (gv_XFlmSysData.Stats.bCollectingStats) { // Statistics are being collected for the system. Therefore, // if we are not currently collecting statistics in the // start. If we were collecting statistics, but the // start time was earlier than the start time in the system // statistics structure, reset the statistics. if (!pCPInfo->Stats.bCollectingStats) { flmStatStart( &pCPInfo->Stats); } else if (pCPInfo->Stats.uiStartTime < gv_XFlmSysData.Stats.uiStartTime) { flmStatReset( &pCPInfo->Stats, FALSE); } (void)flmStatGetDb( &pCPInfo->Stats, this, 0, &pDbStats, NULL, NULL); } else { pDbStats = NULL; } // Lock write object - If we are forcing a checkpoint // wait until we get the lock. Otherwise, if we can't get // the lock without waiting, don't do anything. if (bForceCheckpoint || (gv_XFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache && (m_uiDirtyCacheCount + m_uiLogCacheCount) * m_uiBlockSize > gv_XFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache)) { if (RC_BAD( rc = dbWriteLock( pCPInfo->hWaitSem, pDbStats))) { // THIS SHOULD NEVER HAPPEN BECAUSE dbWriteLock will // wait forever for the lock! RC_UNEXPECTED_ASSERT( rc); goto Exit; } pThread->setThreadStatusStr( "Forcing checkpoint"); // Must wait for any RFL writes to complete. (void)m_pRfl->seeIfRflWritesDone( pCPInfo->hWaitSem, TRUE); } else { if (RC_BAD( dbWriteLock( pCPInfo->hWaitSem, pDbStats, 0))) { goto Exit; } pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); // See if we actually need to do the checkpoint. If the // current transaction ID and the last checkpoint transaction // ID are the same, no updates have occurred that would require // a checkpoint to take place. if (m_lastCommittedDbHdr.ui64RflLastCPTransID == m_lastCommittedDbHdr.ui64CurrTransID || !m_pRfl->seeIfRflWritesDone( pCPInfo->hWaitSem, FALSE)) { dbWriteUnlock(); goto Exit; } } // Do the checkpoint. (void)doCheckpoint( pCPInfo->hWaitSem, pDbStats, pCPInfo->pSFileHdl, FALSE, bForceCheckpoint, iForceReason, 0, 0); if (pDbStats) { (void)flmStatUpdate( &pCPInfo->Stats); } dbWriteUnlock(); // Set the thread's status pThread->setThreadStatus( FLM_THREAD_STATUS_SLEEPING); Exit: return( bTerminate); }
/**************************************************************************** Desc: Commit (write out) all keys that have built up in the KREF table. ****************************************************************************/ RCODE F_Db::keysCommit( FLMBOOL bCommittingTrans, FLMBOOL bSortKeys) { RCODE rc = NE_SFLM_OK; // If the Kref has not been initialized, there is no // work to do. if (m_bKrefSetup) { F_INDEX * pIndex = NULL; FLMUINT uiTotal = m_uiKrefCount; KREF_ENTRY * pKref; KREF_ENTRY ** pKrefTbl = m_pKrefTbl; FLMUINT uiKrefNum; FLMUINT uiLastIxNum; // We should not have reached this point if bAbortTrans is TRUE if( RC_BAD( m_AbortRc)) { rc = RC_SET_AND_ASSERT( m_AbortRc); goto Exit; } // Sort the KREF table, if it contains more than one key. // This will sort all keys from the same index the same. if (uiTotal > 1 && bSortKeys) { processDupKeys( NULL); uiTotal = m_uiKrefCount; } // Loop through the KREF table outputting all keys uiLastIxNum = 0; for (uiKrefNum = 0; uiKrefNum < uiTotal; uiKrefNum++) { pKref = pKrefTbl [uiKrefNum]; // See if the LFILE changed flmAssert( pKref->ui16IxNum); if (pKref->ui16IxNum != uiLastIxNum) { uiLastIxNum = pKref->ui16IxNum; pIndex = m_pDict->getIndex( uiLastIxNum); } // Flush the key to the index if (m_pKeyColl) { m_pKeyColl->addKey( this, pIndex, pKref); } else { if (RC_BAD(rc = refUpdate( pIndex, pKref, TRUE))) { if (rc != NE_SFLM_NOT_UNIQUE) { RC_UNEXPECTED_ASSERT( rc); } goto Exit; } } } if (bCommittingTrans) { krefCntrlFree(); } else { // Empty the table out so we can add more keys in this trans. m_pKrefPool->poolReset( NULL, TRUE); m_uiKrefCount = 0; m_uiTotalKrefBytes = 0; } } Exit: if( RC_BAD( rc)) { setMustAbortTrans( rc); } return( rc); }
/*************************************************************************** Desc: Update (add or delete) a single reference *****************************************************************************/ RCODE F_Db::refUpdate( LFILE * pLFile, IXD * pIxd, KREF_ENTRY * pKrefEntry, FLMBOOL bNormalUpdate) { RCODE rc = NE_XFLM_OK; F_Btree * pbtree = NULL; IXKeyCompare compareObject; // Get a btree if (RC_BAD( rc = gv_XFlmSysData.pBtPool->btpReserveBtree( &pbtree))) { goto Exit; } flmAssert( pLFile->uiRootBlk); compareObject.setIxInfo( this, pIxd); if (bNormalUpdate && pKrefEntry->bDelete) { compareObject.setOldNodeList( m_pOldNodeList); } if( RC_BAD( rc = pbtree->btOpen( this, pLFile, (pIxd->uiFlags & IXD_ABS_POS) ? TRUE : FALSE, (pIxd->pFirstData) ? TRUE : FALSE, &compareObject))) { goto Exit; } if (!pKrefEntry->bDelete) { pbtree->btResetBtree(); if( RC_BAD( rc = pbtree->btInsertEntry( (FLMBYTE *)&pKrefEntry [1], pKrefEntry->ui16KeyLen, pKrefEntry->uiDataLen ? ((FLMBYTE *)(&pKrefEntry [1])) + 1 + pKrefEntry->ui16KeyLen : NULL, pKrefEntry->uiDataLen, TRUE, TRUE))) { goto Exit; } } else { pbtree->btResetBtree(); if (RC_BAD( rc = pbtree->btRemoveEntry( (FLMBYTE *)&pKrefEntry [1], pKrefEntry->ui16KeyLen))) { if (rc == NE_XFLM_NOT_FOUND) { // Already been deleted, ignore the error condition and go on. RC_UNEXPECTED_ASSERT( rc); rc = NE_XFLM_OK; } goto Exit; } } Exit: if (pbtree) { gv_XFlmSysData.pBtPool->btpReturnBtree( &pbtree); } return( rc); }