Ejemplo n.º 1
0
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
	int64_t DirectoryID, bool& rIsModified)
{
	int32_t IndexInDirBlock;
	IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock);
	ASSERT(piBlock != 0);

	uint8_t iflags = GetFlags(piBlock, IndexInDirBlock);

	// Is the type the same?
	if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir())
	{
		// Entry is of wrong type
		BOX_ERROR("Directory ID " <<
			BOX_FORMAT_OBJECTID(DirectoryID) <<
			" references object " <<
			BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
			" which has a different type than expected.");
		++mNumberErrorsFound;
		return false; // remove this entry
	}

	// Check that the entry is not already contained.
	if(iflags & Flags_IsContained)
	{
		BOX_ERROR("Directory ID " <<
			BOX_FORMAT_OBJECTID(DirectoryID) <<
			" references object " <<
			BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
			" which is already contained.");
		++mNumberErrorsFound;
		return false; // remove this entry
	}

	// Not already contained by another directory.
	// Don't set the flag until later, after we finish repairing
	// the directory and removing all bad entries.
	
	// Check that the container ID of the object is correct
	if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
	{
		// Needs fixing...
		if(iflags & Flags_IsDir)
		{
			// Add to will fix later list
			BOX_ERROR("Directory ID " <<
				BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
				<< " has wrong container ID.");
			mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
			++mNumberErrorsFound;
		}
		else
		{
			// This is OK for files, they might move
			BOX_INFO("File ID " <<
				BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
				<< " has different container ID, "
				"probably moved");
		}
		
		// Fix entry for now
		piBlock->mContainer[IndexInDirBlock] = DirectoryID;
	}

	// Check the object size
	if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
	{
		// Wrong size, correct it.
		BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) <<
			" entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
			" has wrong size " << rEntry.GetSizeInBlocks() <<
			", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]);

		rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);

		// Mark as changed
		rIsModified = true;
		++mNumberErrorsFound;
	}

	return true; // don't delete this entry
}
Ejemplo n.º 2
0
bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
{
	bool restart = true;
	bool isModified = false;

	while(restart)
	{
		std::vector<int64_t> toDelete;
		restart = false;

		// Check for validity
		if(dir.CheckAndFix())
		{
			// Wasn't quite right, and has been modified
			BOX_ERROR("Directory ID " <<
				BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
				" had invalid entries" <<
				(mFixErrors ? ", fixed" : ""));
			++mNumberErrorsFound;
			isModified = true;
		}

		// Go through, and check that every entry exists and is valid
		BackupStoreDirectory::Iterator i(dir);
		BackupStoreDirectory::Entry *en = 0;
		while((en = i.Next()) != 0)
		{
			// Lookup the item
			int32_t iIndex;
			IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
			bool badEntry = false;
			if(piBlock != 0)
			{
				badEntry = !CheckDirectoryEntry(*en,
					dir.GetObjectID(), isModified);
			}
			// Item can't be found. Is it a directory?
			else if(en->IsDir())
			{
				// Store the directory for later attention
				mDirsWhichContainLostDirs[en->GetObjectID()] =
					dir.GetObjectID();
			}
			else
			{
				// Just remove the entry
				badEntry = true;
				BOX_ERROR("Directory ID " <<
					BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
					" references object " <<
					BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
					" which does not exist.");
				++mNumberErrorsFound;
			}

			// Is this entry worth keeping?
			if(badEntry)
			{
				toDelete.push_back(en->GetObjectID());
			}
		}

		if(toDelete.size() > 0)
		{
			// Delete entries from directory
			for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
			{
				BOX_ERROR("Removing directory entry " <<
					BOX_FORMAT_OBJECTID(*d) << " from "
					"directory " <<
					BOX_FORMAT_OBJECTID(dir.GetObjectID()));
				++mNumberErrorsFound;
				dir.DeleteEntry(*d);
			}

			// Mark as modified
			restart = true;
			isModified = true;

			// Errors found
		}
	}

	return isModified;
}
Ejemplo n.º 3
0
// Count valid remaining entries and the number of blocks in them.
void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir)
{
	BackupStoreDirectory::Iterator i(dir);
	BackupStoreDirectory::Entry *en = 0;
	while((en = i.Next()) != 0)
	{
		int32_t iIndex;
		IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
		bool badEntry = false;
		bool wasAlreadyContained = false;

		ASSERT(piBlock != 0 ||
			mDirsWhichContainLostDirs.find(en->GetObjectID())
			!= mDirsWhichContainLostDirs.end());

		if (piBlock)
		{
			// Normally it would exist and this
			// check would not be necessary, but
			// we might have missing directories
			// that we will recreate later.
			// cf mDirsWhichContainLostDirs.
			uint8_t iflags = GetFlags(piBlock, iIndex);
			wasAlreadyContained = (iflags & Flags_IsContained);
			SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
		}

		if(wasAlreadyContained)
		{
			// don't double-count objects that are
			// contained by another directory as well.
		}
		else if(en->IsDir())
		{
			mNumDirectories++;
		}
		else if(!en->IsFile())
		{
			BOX_TRACE("Not counting object " <<
				BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
				" with flags " << en->GetFlags());
		}
		else // it's a file
		{
			// Add to sizes?
			// If piBlock was zero, then wasAlreadyContained
			// might be uninitialized; but we only process
			// files here, and if a file's piBlock was zero
			// then badEntry would be set above, so we
			// wouldn't be here.
			ASSERT(!badEntry)

			// It can be both old and deleted.
			// If neither, then it's current.
			if(en->IsDeleted())
			{
				mNumDeletedFiles++;
				mBlocksInDeletedFiles += en->GetSizeInBlocks();
			}

			if(en->IsOld())
			{
				mNumOldFiles++;
				mBlocksInOldFiles += en->GetSizeInBlocks();
			}

			if(!en->IsDeleted() && !en->IsOld())
			{
				mNumCurrentFiles++;
				mBlocksInCurrentFiles += en->GetSizeInBlocks();
			}
		}

		mapNewRefs->AddReference(en->GetObjectID());
	}
}
Ejemplo n.º 4
0
// --------------------------------------------------------------------------
//
// Function
//		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
//		Purpose: Private. Scan a directory for potentially deleteable
//			 items, and add them to the list. Returns true if the
//			 scan should continue.
//		Created: 11/12/03
//
// --------------------------------------------------------------------------
bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID,
	BackupStoreInfo& rBackupStoreInfo)
{
#ifndef WIN32
	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
	{
		mCountUntilNextInterprocessMsgCheck =
			POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;

		// Check for having to stop
		// Include account ID here as the specified account is locked
		if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
		{
			// Need to abort now
			return false;
		}
	}
#endif

	// Get the filename
	std::string objectFilename;
	MakeObjectFilename(ObjectID, objectFilename);

	// Open it.
	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
		objectFilename));

	// Add the size of the directory on disc to the size being calculated
	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
	mBlocksInDirectories += originalDirSizeInBlocks;
	mBlocksUsed += originalDirSizeInBlocks;

	// Read the directory in
	BackupStoreDirectory dir;
	BufferedStream buf(*dirStream);
	dir.ReadFromStream(buf, IOStream::TimeOutInfinite);
	dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks);
	dirStream->Close();

	// Is it empty?
	if(dir.GetNumberOfEntries() == 0)
	{
		// Add it to the list of directories to potentially delete
		mEmptyDirectories.push_back(dir.GetObjectID());
	}

	// Calculate reference counts first, before we start requesting
	// files to be deleted.
	// BLOCK
	{
		BackupStoreDirectory::Iterator i(dir);
		BackupStoreDirectory::Entry *en = 0;

		while((en = i.Next()) != 0)
		{
			// This directory references this object
			mapNewRefs->AddReference(en->GetObjectID());
		}
	}

	// BLOCK
	{
		// Remove any files which are marked for removal as soon
		// as they become old or deleted.
		bool deletedSomething = false;
		do
		{
			// Iterate through the directory
			deletedSomething = false;
			BackupStoreDirectory::Iterator i(dir);
			BackupStoreDirectory::Entry *en = 0;
			while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
			{
				int16_t enFlags = en->GetFlags();
				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
					&& (en->IsDeleted() || en->IsOld()))
				{
					// Delete this immediately.
					DeleteFile(ObjectID, en->GetObjectID(), dir,
						objectFilename, rBackupStoreInfo);
					
					// flag as having done something
					deletedSomething = true;

					// Must start the loop from the beginning again, as iterator is now
					// probably invalid.
					break;
				}
			}
		} while(deletedSomething);
	}

	// BLOCK
	{
		// Add files to the list of potential deletions

		// map to count the distance from the mark
		typedef std::pair<std::string, int32_t> version_t;
		std::map<version_t, int32_t> markVersionAges;
			// map of pair (filename, mark number) -> version age

		// NOTE: use a reverse iterator to allow the distance from mark stuff to work
		BackupStoreDirectory::ReverseIterator i(dir);
		BackupStoreDirectory::Entry *en = 0;

		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
		{
			// Update recalculated usage sizes
			int16_t enFlags = en->GetFlags();
			int64_t enSizeInBlocks = en->GetSizeInBlocks();
			mBlocksUsed += enSizeInBlocks;
			if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks;
			if(en->IsDeleted()) mBlocksInDeletedFiles += enSizeInBlocks;

			// Work out ages of this version from the last mark
			int32_t enVersionAge = 0;
			std::map<version_t, int32_t>::iterator enVersionAgeI(
				markVersionAges.find(
					version_t(en->GetName().GetEncodedFilename(),
						en->GetMarkNumber())));
			if(enVersionAgeI != markVersionAges.end())
			{
				enVersionAge = enVersionAgeI->second + 1;
				enVersionAgeI->second = enVersionAge;
			}
			else
			{
				markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge;
			}
			// enVersionAge is now the age of this version.

			// Potentially add it to the list if it's deleted, if it's an old version or deleted
			if(en->IsOld() || en->IsDeleted())
			{
				// Is deleted / old version.
				DelEn d;
				d.mObjectID = en->GetObjectID();
				d.mInDirectory = ObjectID;
				d.mSizeInBlocks = en->GetSizeInBlocks();
				d.mMarkNumber = en->GetMarkNumber();
				d.mVersionAgeWithinMark = enVersionAge;
				d.mIsFlagDeleted = en->IsDeleted();

				// Add it to the list
				mPotentialDeletions.insert(d);

				// Update various counts
				mPotentialDeletionsTotalSize += d.mSizeInBlocks;
				if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;

				// Too much in the list of potential deletions?
				// (check against the deletion target + the max size in deletions, so that we never delete things
				// and take the total size below the deletion size target)
				if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
				{
					int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
					bool recalcMaxSize = false;

					while(sizeToRemove > 0)
					{
						// Make iterator for the last element, while checking that there's something there in the first place.
						std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
						if(i != mPotentialDeletions.begin())
						{
							// Nothing left in set
							break;
						}
						// Make this into an iterator pointing to the last element in the set
						--i;

						// Delete this one?
						if(sizeToRemove > i->mSizeInBlocks)
						{
							sizeToRemove -= i->mSizeInBlocks;
							if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
							{
								// Will need to recalculate the maximum size now, because we've just deleted that element
								recalcMaxSize = true;
							}
							mPotentialDeletions.erase(i);
						}
						else
						{
							// Over the size to remove, so stop now
							break;
						}
					}

					if(recalcMaxSize)
					{
						// Because an object which was the maximum size recorded was deleted from the set
						// it's necessary to recalculate this maximum.
						mMaxSizeInPotentialDeletions = 0;
						std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
						for(; i != mPotentialDeletions.end(); ++i)
						{
							if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
							{
								mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
							}
						}
					}
				}
			}
		}
	}

	// Recurse into subdirectories
	{
		BackupStoreDirectory::Iterator i(dir);
		BackupStoreDirectory::Entry *en = 0;
		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
		{
			ASSERT(en->IsDir());
			
			if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo))
			{
				// Halting operation
				return false;
			}
		}
	}

	return true;
}