void HousekeepStoreAccount::UpdateDirectorySize( BackupStoreDirectory& rDirectory, IOStream::pos_type new_size_in_blocks) { #ifndef NDEBUG { std::string dirFilename; MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); std::auto_ptr<RaidFileRead> dirStream( RaidFileRead::Open(mStoreDiscSet, dirFilename)); ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); } #endif IOStream::pos_type old_size_in_blocks = rDirectory.GetUserInfo1_SizeInBlocks(); if(new_size_in_blocks == old_size_in_blocks) { return; } rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) { return; } std::string parentFilename; MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); std::auto_ptr<RaidFileRead> parentStream( RaidFileRead::Open(mStoreDiscSet, parentFilename)); BackupStoreDirectory parent(*parentStream); parentStream.reset(); BackupStoreDirectory::Entry* en = parent.FindEntryByID(rDirectory.GetObjectID()); ASSERT(en); if (en->GetSizeInBlocks() != old_size_in_blocks) { BOX_WARNING("Directory " << BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << " entry in directory " << BOX_FORMAT_OBJECTID(rDirectory.GetContainerID()) << " had incorrect size " << en->GetSizeInBlocks() << ", should have been " << old_size_in_blocks); mErrorCount++; } en->SetSizeInBlocks(new_size_in_blocks); RaidFileWrite writeDir(mStoreDiscSet, parentFilename, mapNewRefs->GetRefCount(rDirectory.GetContainerID())); writeDir.Open(true /* allow overwriting */); parent.WriteToStream(writeDir); writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); }
int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) { int ErrorCount = 0; int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed(); int64_t MaxNewObjectId = GetLastObjectIDUsed(); for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; ObjectID < std::max(MaxOldObjectId, MaxNewObjectId); ObjectID++) { typedef BackupStoreRefCountDatabase::refcount_t refcount_t; refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ? rOldRefs.GetRefCount(ObjectID) : 0; refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ? this->GetRefCount(ObjectID) : 0; if (OldRefs != NewRefs) { BOX_WARNING("Reference count of object " << BOX_FORMAT_OBJECTID(ObjectID) << " changed from " << OldRefs << " to " << NewRefs); ErrorCount++; } } return ErrorCount; }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreRefCountDatabase::GetRefCount(int64_t // ObjectID) // Purpose: Get the number of references to the specified object // out of the database // Created: 2009/06/01 // // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::refcount_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const { IOStream::pos_type offset = GetOffset(ObjectID); if (GetSize() < offset + GetEntrySize()) { THROW_FILE_ERROR("Failed to read refcount database: " "attempted read of unknown refcount for object " << BOX_FORMAT_OBJECTID(ObjectID), mFilename, BackupStoreException, UnknownObjectRefCountRequested); } mapDatabaseFile->Seek(offset, SEEK_SET); refcount_t refcount; if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != sizeof(refcount)) { THROW_FILE_ERROR("Failed to read refcount database: " "short read at offset " << offset, mFilename, BackupStoreException, CouldNotLoadStoreInfo); } return ntohl(refcount); }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckObjects() // Purpose: Read in the contents of the directory, recurse to other levels, // checking objects for sanity and readability // Created: 21/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::CheckObjects() { // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything int64_t maxDir = 0; // Find the maximum directory starting ID { // Make sure the starting root dir doesn't end with '/'. std::string start(mStoreRoot); if(start.size() > 0 && ( start[start.size() - 1] == '/' || start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR)) { start.resize(start.size() - 1); } maxDir = CheckObjectsScanDir(0, 1, start); BOX_TRACE("Max dir starting ID is " << BOX_FORMAT_OBJECTID(maxDir)); } // Then go through and scan all the objects within those directories for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) { CheckObjectsDir(d); } }
bool check_reference_counts() { std::auto_ptr<BackupStoreAccountDatabase> apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); std::auto_ptr<BackupStoreRefCountDatabase> apReferences( BackupStoreRefCountDatabase::Load(account, true)); TEST_EQUAL(ExpectedRefCounts.size(), apReferences->GetLastObjectIDUsed() + 1); bool counts_ok = true; for (unsigned int i = BackupProtocolListDirectory::RootDirectory; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], apReferences->GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) { counts_ok = false; } } return counts_ok; }
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified) { int32_t IndexInDirBlock; IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock); ASSERT(piBlock != 0); uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); // Is the type the same? if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) { // Entry is of wrong type BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which has a different type than expected."); ++mNumberErrorsFound; return false; // remove this entry } // Check that the entry is not already contained. if(iflags & Flags_IsContained) { BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which is already contained."); ++mNumberErrorsFound; return false; // remove this entry } // Not already contained by another directory. // Don't set the flag until later, after we finish repairing // the directory and removing all bad entries. // Check that the container ID of the object is correct if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) { // Needs fixing... if(iflags & Flags_IsDir) { // Add to will fix later list BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has wrong container ID."); mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); ++mNumberErrorsFound; } else { // This is OK for files, they might move BOX_INFO("File ID " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has different container ID, " "probably moved"); } // Fix entry for now piBlock->mContainer[IndexInDirBlock] = DirectoryID; } // Check the object size if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) { // Wrong size, correct it. BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) << " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has wrong size " << rEntry.GetSizeInBlocks() << ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]); rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); // Mark as changed rIsModified = true; ++mNumberErrorsFound; } return true; // don't delete this entry }
// Count valid remaining entries and the number of blocks in them. void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); bool badEntry = false; bool wasAlreadyContained = false; ASSERT(piBlock != 0 || mDirsWhichContainLostDirs.find(en->GetObjectID()) != mDirsWhichContainLostDirs.end()); if (piBlock) { // Normally it would exist and this // check would not be necessary, but // we might have missing directories // that we will recreate later. // cf mDirsWhichContainLostDirs. uint8_t iflags = GetFlags(piBlock, iIndex); wasAlreadyContained = (iflags & Flags_IsContained); SetFlags(piBlock, iIndex, iflags | Flags_IsContained); } if(wasAlreadyContained) { // don't double-count objects that are // contained by another directory as well. } else if(en->IsDir()) { mNumDirectories++; } else if(!en->IsFile()) { BOX_TRACE("Not counting object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " with flags " << en->GetFlags()); } else // it's a file { // Add to sizes? // If piBlock was zero, then wasAlreadyContained // might be uninitialized; but we only process // files here, and if a file's piBlock was zero // then badEntry would be set above, so we // wouldn't be here. ASSERT(!badEntry) // It can be both old and deleted. // If neither, then it's current. if(en->IsDeleted()) { mNumDeletedFiles++; mBlocksInDeletedFiles += en->GetSizeInBlocks(); } if(en->IsOld()) { mNumOldFiles++; mBlocksInOldFiles += en->GetSizeInBlocks(); } if(!en->IsDeleted() && !en->IsOld()) { mNumCurrentFiles++; mBlocksInCurrentFiles += en->GetSizeInBlocks(); } } mapNewRefs->AddReference(en->GetObjectID()); } }
bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { bool restart = true; bool isModified = false; while(restart) { std::vector<int64_t> toDelete; restart = false; // Check for validity if(dir.CheckAndFix()) { // Wasn't quite right, and has been modified BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << " had invalid entries" << (mFixErrors ? ", fixed" : "")); ++mNumberErrorsFound; isModified = true; } // Go through, and check that every entry exists and is valid BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { // Lookup the item int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); bool badEntry = false; if(piBlock != 0) { badEntry = !CheckDirectoryEntry(*en, dir.GetObjectID(), isModified); } // Item can't be found. Is it a directory? else if(en->IsDir()) { // Store the directory for later attention mDirsWhichContainLostDirs[en->GetObjectID()] = dir.GetObjectID(); } else { // Just remove the entry badEntry = true; BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); ++mNumberErrorsFound; } // Is this entry worth keeping? if(badEntry) { toDelete.push_back(en->GetObjectID()); } } if(toDelete.size() > 0) { // Delete entries from directory for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) { BOX_ERROR("Removing directory entry " << BOX_FORMAT_OBJECTID(*d) << " from " "directory " << BOX_FORMAT_OBJECTID(dir.GetObjectID())); ++mNumberErrorsFound; dir.DeleteEntry(*d); } // Mark as modified restart = true; isModified = true; // Errors found } } return isModified; }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckDirectories() // Purpose: Check the directories // Created: 22/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::CheckDirectories() { // Phase 1 did this: // Checked that all the directories are readable // Built a list of all directories and files which exist on the store // // This phase will check all the files in the directories, make // a note of all directories which are missing, and do initial fixing. // The root directory is not contained inside another directory, so // it has no directory entry to scan, but we have to count it // somewhere, so we'll count it here. mNumDirectories++; // Scan all objects. for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); if(flags & Flags_IsDir) { // Found a directory. Read it in. std::string filename; StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); BackupStoreDirectory dir; { std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); dir.ReadFromStream(*file, IOStream::TimeOutInfinite); } // Flag for modifications bool isModified = CheckDirectory(dir); // Check the directory again, now that entries have been removed if(dir.CheckAndFix()) { // Wasn't quite right, and has been modified BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was still bad after all checks"); ++mNumberErrorsFound; isModified = true; } else if(isModified) { BOX_INFO("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was OK after fixing"); } if(isModified && mFixErrors) { BOX_WARNING("Writing modified directory to disk: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); RaidFileWrite fixed(mDiscSetNumber, filename); fixed.Open(true /* allow overwriting */); dir.WriteToStream(fixed); fixed.Commit(true /* convert to raid representation now */); } CountDirectoryEntries(dir); } } } }
BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; bool wasOldVersion = false; int64_t deletedFileSizeInBlocks = 0; // A pointer to an object which requires committing if the directory save goes OK std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { BackupStoreRefCountDatabase::refcount_t refs = mapNewRefs->GetRefCount(ObjectID); BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { BOX_ERROR("Housekeeping on account " << BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); mErrorCount++; return refs; } // Record the flags it's got set wasDeleted = pentry->IsDeleted(); wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { // Things changed since we were last around return refs; } // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); if(refs > 1) { // Not safe to merge patches if someone else has a // reference to this object, so just remove the // directory entry and return. rDirectory.DeleteEntry(ObjectID); if(wasDeleted) { rBackupStoreInfo.AdjustNumDeletedFiles(-1); } if(wasOldVersion) { rBackupStoreInfo.AdjustNumOldFiles(-1); } mapNewRefs->RemoveReference(ObjectID); return refs - 1; } // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) { // This entry is a patch from a newer entry. Just need to update the info on that entry. BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Change the info in the newer entry so that this no longer points to this entry pnewer->SetDependsOlder(0); } else if(pentry->GetDependsOlder() != 0) { BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder()); if(pentry->GetDependsNewer() == 0) { // There exists an older version which depends on this one. Need to combine the two over that one. // Adjust the other entry in the directory if(polder == 0 || polder->GetDependsNewer() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Change the info in the older entry so that this no longer points to this entry polder->SetDependsNewer(0); } else { // This entry is in the middle of a chain, and two patches need combining. // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID || polder == 0 || polder->GetDependsNewer() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Remove the middle entry from the linked list by simply using the values from this entry pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } // COMMON CODE to both cases // Generate the filename of the older version std::string objFilenameOlder; MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder); // Open it twice (it's the diff) std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); // Open this file std::string objFilename; MakeObjectFilename(ObjectID, objFilename); std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) { // There exists an older version which depends on this one. Need to combine the two over that one. BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry); } else { // This entry is in the middle of a chain, and two patches need combining. BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } // pentry no longer valid } // Delete it from the directory rDirectory.DeleteEntry(ObjectID); // Save directory back to disc // BLOCK { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); // Get the disc usage (must do this before commiting it) int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); // Adjust block counts if the directory itself changed in size int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); int64_t adjust = new_size - original_size; mBlocksUsedDelta += adjust; mBlocksInDirectoriesDelta += adjust; UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); padjustedEntry.reset(); // delete it now } // Drop reference count by one. Must now be zero, to delete the file. bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); ASSERT(!remaining_refs); // Delete from disc BOX_TRACE("Removing unreferenced object " << BOX_FORMAT_OBJECTID(ObjectID)); std::string objFilename; MakeObjectFilename(ObjectID, objFilename); RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); del.Delete(); // Adjust counts for the file ++mFilesDeleted; mBlocksUsedDelta -= deletedFileSizeInBlocks; if(wasDeleted) { mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; rBackupStoreInfo.AdjustNumDeletedFiles(-1); } if(wasOldVersion) { mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; rBackupStoreInfo.AdjustNumOldFiles(-1); } // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) { // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } return 0; }
// -------------------------------------------------------------------------- // // Function // Name: HousekeepStoreAccount::DeleteFiles() // Purpose: Delete the files targeted for deletion, returning // true if the operation was interrupted // Created: 15/12/03 // // -------------------------------------------------------------------------- bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { // Only delete files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) if(mDeletionSizeTarget <= 0) { // Not interrupted return false; } // Iterate through the set of potential deletions, until enough has been deleted. // (there is likely to be more in the set than should be actually deleted). for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i) { #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; // Check for having to stop if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked { // Need to abort now return true; } } #endif // Load up the directory it's in // Get the filename std::string dirFilename; BackupStoreDirectory dir; { MakeObjectFilename(i->mInDirectory, dirFilename); std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); } // Delete the file BackupStoreRefCountDatabase::refcount_t refs = DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, rBackupStoreInfo); if(refs == 0) { BOX_INFO("Housekeeping removed " << (i->mIsFlagDeleted ? "deleted" : "old") << " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); } else { BOX_TRACE("Housekeeping preserved " << (i->mIsFlagDeleted ? "deleted" : "old") << " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << " with " << refs << " references"); } // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the // space to slightly less than the soft limit, which will allow the backup // client to start uploading files again) if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget) { break; } } return false; }
void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo) { // Load up the directory to potentially delete std::string dirFilename; BackupStoreDirectory dir; int64_t dirSizeInBlocks = 0; // BLOCK { MakeObjectFilename(dirId, dirFilename); // Check it actually exists (just in case it gets // added twice to the list) if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) { // doesn't exist, next! return; } // load std::auto_ptr<RaidFileRead> dirStream( RaidFileRead::Open(mStoreDiscSet, dirFilename)); dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); } // Make sure this directory is actually empty if(dir.GetNumberOfEntries() != 0) { // Not actually empty, try next one return; } // Candidate for deletion... open containing directory std::string containingDirFilename; BackupStoreDirectory containingDir; int64_t containingDirSizeInBlocksOrig = 0; { MakeObjectFilename(dir.GetContainerID(), containingDirFilename); std::auto_ptr<RaidFileRead> containingDirStream( RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); } // Find the entry BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); // TODO FIXME invert test and reduce indentation if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted containingDir.DeleteEntry(dir.GetObjectID()); // Is the containing dir now a candidate for deletion? if(containingDir.GetNumberOfEntries() == 0) { rToExamine.push_back(containingDir.GetObjectID()); } // Write revised parent directory RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, mapNewRefs->GetRefCount(containingDir.GetObjectID())); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); // get the disc usage (must do this before commiting it) int64_t dirSize = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); UpdateDirectorySize(containingDir, dirSize); // adjust usage counts for this directory if(dirSize > 0) { int64_t adjust = dirSize - containingDirSizeInBlocksOrig; mBlocksUsedDelta += adjust; mBlocksInDirectoriesDelta += adjust; } if (mapNewRefs->RemoveReference(dir.GetObjectID())) { // Still referenced BOX_TRACE("Housekeeping spared empty deleted dir " << BOX_FORMAT_OBJECTID(dirId) << " due to " << mapNewRefs->GetRefCount(dir.GetObjectID()) << "remaining references"); return; } // Delete the directory itself BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); RaidFileWrite del(mStoreDiscSet, dirFilename, mapNewRefs->GetRefCount(dir.GetObjectID())); del.Delete(); // And adjust usage counts for the directory that's // just been deleted mBlocksUsedDelta -= dirSizeInBlocks; mBlocksInDirectoriesDelta -= dirSizeInBlocks; // Update count ++mEmptyDirectoriesDeleted; rBackupStoreInfo.AdjustNumDirectories(-1); } }