int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return if(!OpenAccount(ID, rootDir, discSetNum, user, NULL /* housekeeping locks the account itself */)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for housekeeping."); return 1; } HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); bool success = housekeeping.DoHousekeeping(); if(!success) { BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) << " for housekeeping: perhaps a client is " "still connected?"); return 1; } else { BOX_TRACE("Finished housekeeping on account " << BOX_FORMAT_ACCOUNT(ID)); return 0; } }
int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change name."); return 1; } // Load the info std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSetNum, false /* Read/Write */)); info->SetAccountName(rNewAccountName); // Save info->Save(); BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " name changed to " << rNewAccountName); return 0; }
int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet, bool ReturnNumErrorsFound) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; if(!OpenAccount(ID, rootDir, discSetNum, user, FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for checking."); return 1; } // Check it BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); check.Check(); if(ReturnNumErrorsFound) { return check.GetNumErrorsFound(); } else { return check.ErrorsFound() ? 1 : 0; } }
int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber, int32_t SoftLimit, int32_t HardLimit) { // Load in the account database std::auto_ptr<BackupStoreAccountDatabase> db( BackupStoreAccountDatabase::Read( mConfig.GetKeyValue("AccountDatabase"))); // Already exists? if(db->EntryExists(ID)) { BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << " already exists."); return 1; } // Get the user under which the daemon runs std::string username; { const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); if(rserverConfig.KeyExists("User")) { username = rserverConfig.GetKeyValue("User"); } } // Create it. BackupStoreAccounts acc(*db); acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); return 0; }
int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, const char *HardLimitStr) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change limits."); return 1; } // Load the info std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSetNum, false /* Read/Write */)); // Change the limits int blocksize = BlockSizeOfDiscSet(discSetNum); int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); CheckSoftHardLimits(softlimit, hardlimit); info->ChangeLimits(softlimit, hardlimit); // Save info->Save(); BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << " changed to " << softlimit << " soft, " << hardlimit << " hard."); return 0; }
// -------------------------------------------------------------------------- // // Function // Name: Random::Initialise() // Purpose: Add additional randomness to the standard library initialisation // Created: 18/6/04 // // -------------------------------------------------------------------------- void Random::Initialise() { #ifdef HAVE_RANDOM_DEVICE if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) { THROW_EXCEPTION(CipherException, RandomInitFailed) } #else BOX_ERROR("No random device -- additional seeding of random number " "generator not performed."); #endif }
// Start the first overlapped read void WinNamedPipeStream::StartFirstRead() { // create the Readable event mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); if (mReadableEvent == INVALID_HANDLE_VALUE) { BOX_ERROR("Failed to create the Readable event: " << GetErrorMessage(GetLastError())); Close(); THROW_EXCEPTION(CommonException, Internal) }
bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) { // Load in the account database std::auto_ptr<BackupStoreAccountDatabase> db( BackupStoreAccountDatabase::Read( mConfig.GetKeyValue("AccountDatabase"))); // Exists? if(!db->EntryExists(ID)) { BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << " does not exist."); return false; } // Get info from the database BackupStoreAccounts acc(*db); acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); // Get the user under which the daemon runs std::string username; { const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); if(rserverConfig.KeyExists("User")) { username = rserverConfig.GetKeyValue("User"); } } // Become the right user if(!username.empty()) { // Username specified, change... apUser.reset(new UnixUser(username)); apUser->ChangeProcessUser(true /* temporary */); // Change will be undone when apUser goes out of scope // in the caller. } if(pLock) { acc.LockAccount(ID, *pLock); } return true; }
int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to change enabled flag."); return 1; } // Load it in std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSetNum, false /* ReadOnly */)); info->SetAccountEnabled(enabled); info->Save(); return 0; }
int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return if(!OpenAccount(ID, rootDir, discSetNum, user, NULL /* no write lock needed for this read-only operation */)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " to display info."); return 1; } // Load it in std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSetNum, true /* ReadOnly */)); return BackupAccountControl::PrintAccountInfo(*info, BlockSizeOfDiscSet(discSetNum)); }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) // Purpose: Do check on file, return original container ID // if OK, or -1 on error // Created: 22/4/04 // // -------------------------------------------------------------------------- int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) { // Check that it's not the root directory ID. Having a file as // the root directory would be bad. if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) { // Get that dodgy thing deleted! BOX_ERROR("Have file as root directory. This is bad."); return -1; } // Check the format of the file, and obtain the container ID int64_t originalContainerID = -1; if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, &originalContainerID)) { // Didn't verify return -1; } return originalContainerID; }
void testservers_connection(SocketStream &rStream) { IOStreamGetLine getline(rStream); if(typeid(rStream) == typeid(SocketStreamTLS)) { // need to wait for some data before sending stuff, otherwise timeout test doesn't work std::string line; while(!getline.GetLine(line)) ; SocketStreamTLS &rtls = (SocketStreamTLS&)rStream; std::string line1("CONNECTED:"); line1 += rtls.GetPeerCommonName(); line1 += '\n'; testservers_pause_before_reply(); rStream.Write(line1.c_str(), line1.size()); } while(!getline.IsEOF()) { std::string line; while(!getline.GetLine(line)) ; if(line == "QUIT") { break; } if(line == "LARGEDATA") { { // Send lots of data char data[LARGE_DATA_BLOCK_SIZE]; for(unsigned int y = 0; y < sizeof(data); y++) { data[y] = y & 0xff; } for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s) { rStream.Write(data, sizeof(data), SHORT_TIMEOUT); } } { // Receive lots of data char buf[1024]; int total = 0; int r = 0; while(total < LARGE_DATA_SIZE && (r = rStream.Read(buf, sizeof(buf), SHORT_TIMEOUT)) != 0) { total += r; } TEST_THAT(total == LARGE_DATA_SIZE); if (total != LARGE_DATA_SIZE) { BOX_ERROR("Expected " << LARGE_DATA_SIZE << " bytes " << "but was " << total); return; } } { // Send lots of data again char data[LARGE_DATA_BLOCK_SIZE]; for(unsigned int y = 0; y < sizeof(data); y++) { data[y] = y & 0xff; } for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s) { rStream.Write(data, sizeof(data), SHORT_TIMEOUT); } } // next! continue; } std::string backwards; for(std::string::const_reverse_iterator i(line.end()); i != std::string::const_reverse_iterator(line.begin()); ++i) { backwards += (*i); } backwards += '\n'; testservers_pause_before_reply(); rStream.Write(backwards.c_str(), backwards.size()); } rStream.Shutdown(); rStream.Close(); }
int main(int argc, const char *argv[]) { int returnCode = 0; MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupctl.memleaks", "bbackupctl") MAINHELPER_START Logging::SetProgramName("bbackupctl"); // Filename for configuration file? std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; // See if there's another entry on the command line int c; std::string options("c:"); options += Logging::OptionParser::GetOptionString(); Logging::OptionParser LogLevel; while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1) { switch(c) { case 'c': // store argument configFilename = optarg; break; default: int ret = LogLevel.ProcessOption(c); if(ret != 0) { PrintUsageAndExit(ret); } } } // Adjust arguments argc -= optind; argv += optind; // Check there's a command if(argc != 1) { PrintUsageAndExit(2); } Logging::FilterConsole(LogLevel.GetCurrentLevel()); // Read in the configuration file BOX_INFO("Using configuration file " << configFilename); std::string errs; std::auto_ptr<Configuration> config( Configuration::LoadAndVerify (configFilename, &BackupDaemonConfigVerify, errs)); if(config.get() == 0 || !errs.empty()) { BOX_ERROR("Invalid configuration file: " << errs); return 1; } // Easier coding const Configuration &conf(*config); // Check there's a socket defined in the config file if(!conf.KeyExists("CommandSocket")) { BOX_ERROR("Daemon isn't using a control socket, " "could not execute command.\n" "Add a CommandSocket declaration to the " "bbackupd.conf file."); return 1; } // Connect to socket #ifndef WIN32 SocketStream connection; #else /* WIN32 */ WinNamedPipeStream connection; #endif /* ! WIN32 */ try { #ifdef WIN32 std::string socket = conf.GetKeyValue("CommandSocket"); connection.Connect(socket); #else connection.Open(Socket::TypeUNIX, conf.GetKeyValue("CommandSocket").c_str()); #endif } catch(...) { BOX_ERROR("Failed to connect to daemon control socket.\n" "Possible causes:\n" " * Daemon not running\n" " * Daemon busy syncing with store server\n" " * Another bbackupctl process is communicating with the daemon\n" " * Daemon is waiting to recover from an error" ); return 1; } // For receiving data IOStreamGetLine getLine(connection); // Wait for the configuration summary std::string configSummary; if(!getLine.GetLine(configSummary, false, PROTOCOL_DEFAULT_TIMEOUT)) { BOX_ERROR("Failed to receive configuration summary " "from daemon"); return 1; } // Was the connection rejected by the server? if(getLine.IsEOF()) { BOX_ERROR("Server rejected the connection. Are you running " "bbackupctl as the same user as the daemon?"); return 1; } // Decode it int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", &autoBackup, &updateStoreInterval, &minimumFileAge, &maxUploadWait) != 4) { BOX_ERROR("Config summary didn't decode."); return 1; } // Print summary? BOX_TRACE("Daemon configuration summary:\n" " AutomaticBackup = " << (autoBackup?"true":"false") << "\n" " UpdateStoreInterval = " << updateStoreInterval << " seconds\n" " MinimumFileAge = " << minimumFileAge << " seconds\n" " MaxUploadWait = " << maxUploadWait << " seconds"); std::string stateLine; if(!getLine.GetLine(stateLine, false, PROTOCOL_DEFAULT_TIMEOUT) || getLine.IsEOF()) { BOX_ERROR("Failed to receive state line from daemon"); return 1; } // Decode it int currentState; if(::sscanf(stateLine.c_str(), "state %d", ¤tState) != 1) { BOX_ERROR("Received invalid state line from daemon"); return 1; } BOX_TRACE("Current state: " << BackupDaemon::GetStateName(currentState)); Command command = Default; std::string commandName(argv[0]); if(commandName == "wait-for-sync") { command = WaitForSyncStart; } else if(commandName == "wait-for-end") { command = WaitForSyncEnd; } else if(commandName == "sync-and-wait") { command = SyncAndWaitForEnd; } else if(commandName == "status") { BOX_NOTICE("state " << BackupDaemon::GetStateName(currentState)); command = NoCommand; } switch(command) { case WaitForSyncStart: case WaitForSyncEnd: { // Check that it's in automatic mode, // because otherwise it'll never start if(!autoBackup) { BOX_ERROR("Daemon is not in automatic mode, " "sync will never start!"); return 1; } } break; case SyncAndWaitForEnd: { // send a sync command commandName = "force-sync"; std::string cmd = commandName + "\n"; connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); connection.WriteAllBuffered(); if(currentState != 0) { BOX_INFO("Waiting for current sync/error state " "to finish..."); } } break; default: { // Normal case, just send the command given, plus a // quit command. std::string cmd = commandName + "\n"; connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); } // fall through case NoCommand: { // Normal case, just send the command given plus a // quit command. std::string cmd = "quit\n"; connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); } } // Read the response std::string line; bool syncIsRunning = false; bool finished = false; while(command != NoCommand && !finished && !getLine.IsEOF() && getLine.GetLine(line, false, PROTOCOL_DEFAULT_TIMEOUT)) { BOX_TRACE("Received line: " << line); if(line.substr(0, 6) == "state ") { std::string state_str = line.substr(6); int state_num; if(sscanf(state_str.c_str(), "%d", &state_num) == 1) { BOX_INFO("Daemon state changed to: " << BackupDaemon::GetStateName(state_num)); } else { BOX_WARNING("Failed to parse line: " << line); } } switch(command) { case WaitForSyncStart: { // Need to wait for the state change... if(line == "start-sync") { // And we're done finished = true; } } break; case WaitForSyncEnd: case SyncAndWaitForEnd: { if(line == "start-sync") { BOX_TRACE("Sync started..."); syncIsRunning = true; } else if(line == "finish-sync") { if (syncIsRunning) { // And we're done BOX_TRACE("Sync finished."); finished = true; } else { BOX_TRACE("Previous sync finished."); } // daemon must still be busy } } break; default: { // Is this an OK or error line? if(line == "ok") { BOX_TRACE("Control command " "sent: " << commandName); finished = true; } else if(line == "error") { BOX_ERROR("Control command failed: " << commandName << ". Check " "command spelling."); returnCode = 1; finished = true; } } } } // Send a quit command to finish nicely connection.Write("quit\n", 5, PROTOCOL_DEFAULT_TIMEOUT); MAINHELPER_END #if defined WIN32 && ! defined BOX_RELEASE_BUILD closelog(); #endif return returnCode; }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &) // Purpose: Read in the contents of the directory, recurse to other levels, // return the maximum starting ID of any directory found. // Created: 21/4/04 // // -------------------------------------------------------------------------- int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName) { //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID); int64_t maxID = StartID; // Read in all the directories, and recurse downwards { // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); if(!rdiscSet.IsNonRaidSet()) { unsigned int numDiscs = rdiscSet.size(); for(unsigned int l = 0; l < numDiscs; ++l) { // build name std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); EMU_STRUCT_STAT st; if(EMU_STAT(dn.c_str(), &st) != 0 && errno == ENOENT) { if(mkdir(dn.c_str(), 0755) != 0) { THROW_SYS_FILE_ERROR("Failed to " "create missing RaidFile " "directory", dn, RaidFileException, OSError); } } } } std::vector<std::string> dirs; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, RaidFileRead::DirReadType_DirsOnly, dirs); for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i) { // Check to see if it's the right name int n = 0; if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) && n < (1<<STORE_ID_SEGMENT_LENGTH)) { // Next level down int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1, rDirName + DIRECTORY_SEPARATOR + *i); // Found a greater starting ID? if(mi > maxID) { maxID = mi; } } else { BOX_ERROR("Spurious or invalid directory " << rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); ++mNumberErrorsFound; } } } return maxID; }
int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation) { std::string rootDir; int discSetNum; std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; // Obtain a write lock, as the daemon user if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) { BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) << " for deletion."); return 1; } // Check user really wants to do this if(AskForConfirmation) { BOX_WARNING("Really delete account " << BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); char response[256]; if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) { BOX_NOTICE("Deletion cancelled."); return 0; } } // Back to original user, but write lock is maintained user.reset(); std::auto_ptr<BackupStoreAccountDatabase> db( BackupStoreAccountDatabase::Read( mConfig.GetKeyValue("AccountDatabase"))); // Delete from account database db->DeleteEntry(ID); // Write back to disc db->Write(); // Remove the store files... // First, become the user specified in the config file std::string username; { const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); if(rserverConfig.KeyExists("User")) { username = rserverConfig.GetKeyValue("User"); } } // Become the right user if(!username.empty()) { // Username specified, change... user.reset(new UnixUser(username)); user->ChangeProcessUser(true /* temporary */); // Change will be undone when user goes out of scope } // Secondly, work out which directories need wiping std::vector<std::string> toDelete; RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) { if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) { toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); } } // NamedLock will throw an exception if it can't delete the lockfile, // which it can't if it doesn't exist. Now that we've deleted the account, // nobody can open it anyway, so it's safe to unlock. writeLock.ReleaseLock(); int retcode = 0; // Thirdly, delete the directories... for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) { BOX_NOTICE("Deleting store directory " << (*d) << "..."); // Just use the rm command to delete the files std::string cmd("rm -rf "); cmd += *d; // Run command if(::system(cmd.c_str()) != 0) { BOX_ERROR("Failed to delete files in " << (*d) << ", delete them manually."); retcode = 1; } } // Success! return retcode; }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckObjectsDir(int64_t) // Purpose: Check all the files within this directory which has // the given starting ID. // Created: 22/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { // Make directory name -- first generate the filename of an entry in it std::string dirName; StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); // Check expectations ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); // Remove the filename from it dirName.resize(dirName.size() - 4); // four chars for "/o00" // Check directory exists if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) { BOX_WARNING("RaidFile dir " << dirName << " does not exist"); return; } // Read directory contents std::vector<std::string> files; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, RaidFileRead::DirReadType_FilesOnly, files); // Array of things present bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) { idsPresent[l] = false; } // Parse each entry, building up a list of object IDs which are present in the dir. // This is done so that whatever order is retured from the directory, objects are scanned // in order. // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted. for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i) { bool fileOK = true; int n = 0; if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) && n < (1<<STORE_ID_SEGMENT_LENGTH)) { // Filename is valid, mark as existing idsPresent[n] = true; } // No other files should be present in subdirectories else if(StartID != 0) { fileOK = false; } // info and refcount databases are OK in the root directory else if(*i == "info" || *i == "refcount.db" || *i == "refcount.rdb" || *i == "refcount.rdbX") { fileOK = true; } else { fileOK = false; } if(!fileOK) { // Unexpected or bad file, delete it BOX_ERROR("Spurious file " << dirName << DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; if(mFixErrors) { RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i); del.Delete(); } } } // Check all the objects found in this directory for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) { if(idsPresent[i]) { // Check the object is OK, and add entry char leaf[8]; ::snprintf(leaf, sizeof(leaf), DIRECTORY_SEPARATOR "o%02x", i); if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it BOX_ERROR("Corrupted file " << dirName << leaf << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; if(mFixErrors) { RaidFileWrite del(mDiscSetNumber, dirName + leaf); del.Delete(); } } } } }
BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; bool wasOldVersion = false; int64_t deletedFileSizeInBlocks = 0; // A pointer to an object which requires committing if the directory save goes OK std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { BackupStoreRefCountDatabase::refcount_t refs = mapNewRefs->GetRefCount(ObjectID); BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { BOX_ERROR("Housekeeping on account " << BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); mErrorCount++; return refs; } // Record the flags it's got set wasDeleted = pentry->IsDeleted(); wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { // Things changed since we were last around return refs; } // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); if(refs > 1) { // Not safe to merge patches if someone else has a // reference to this object, so just remove the // directory entry and return. rDirectory.DeleteEntry(ObjectID); if(wasDeleted) { rBackupStoreInfo.AdjustNumDeletedFiles(-1); } if(wasOldVersion) { rBackupStoreInfo.AdjustNumOldFiles(-1); } mapNewRefs->RemoveReference(ObjectID); return refs - 1; } // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) { // This entry is a patch from a newer entry. Just need to update the info on that entry. BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Change the info in the newer entry so that this no longer points to this entry pnewer->SetDependsOlder(0); } else if(pentry->GetDependsOlder() != 0) { BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder()); if(pentry->GetDependsNewer() == 0) { // There exists an older version which depends on this one. Need to combine the two over that one. // Adjust the other entry in the directory if(polder == 0 || polder->GetDependsNewer() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Change the info in the older entry so that this no longer points to this entry polder->SetDependsNewer(0); } else { // This entry is in the middle of a chain, and two patches need combining. // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID || polder == 0 || polder->GetDependsNewer() != ObjectID) { THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); } // Remove the middle entry from the linked list by simply using the values from this entry pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } // COMMON CODE to both cases // Generate the filename of the older version std::string objFilenameOlder; MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder); // Open it twice (it's the diff) std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); // Open this file std::string objFilename; MakeObjectFilename(ObjectID, objFilename); std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) { // There exists an older version which depends on this one. Need to combine the two over that one. BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry); } else { // This entry is in the middle of a chain, and two patches need combining. BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } // pentry no longer valid } // Delete it from the directory rDirectory.DeleteEntry(ObjectID); // Save directory back to disc // BLOCK { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); // Get the disc usage (must do this before commiting it) int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); // Adjust block counts if the directory itself changed in size int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); int64_t adjust = new_size - original_size; mBlocksUsedDelta += adjust; mBlocksInDirectoriesDelta += adjust; UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); padjustedEntry.reset(); // delete it now } // Drop reference count by one. Must now be zero, to delete the file. bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); ASSERT(!remaining_refs); // Delete from disc BOX_TRACE("Removing unreferenced object " << BOX_FORMAT_OBJECTID(ObjectID)); std::string objFilename; MakeObjectFilename(ObjectID, objFilename); RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); del.Delete(); // Adjust counts for the file ++mFilesDeleted; mBlocksUsedDelta -= deletedFileSizeInBlocks; if(wasDeleted) { mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; rBackupStoreInfo.AdjustNumDeletedFiles(-1); } if(wasOldVersion) { mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; rBackupStoreInfo.AdjustNumOldFiles(-1); } // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) { // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } return 0; }
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified) { int32_t IndexInDirBlock; IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock); ASSERT(piBlock != 0); uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); // Is the type the same? if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) { // Entry is of wrong type BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which has a different type than expected."); ++mNumberErrorsFound; return false; // remove this entry } // Check that the entry is not already contained. if(iflags & Flags_IsContained) { BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which is already contained."); ++mNumberErrorsFound; return false; // remove this entry } // Not already contained by another directory. // Don't set the flag until later, after we finish repairing // the directory and removing all bad entries. // Check that the container ID of the object is correct if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) { // Needs fixing... if(iflags & Flags_IsDir) { // Add to will fix later list BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has wrong container ID."); mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); ++mNumberErrorsFound; } else { // This is OK for files, they might move BOX_INFO("File ID " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has different container ID, " "probably moved"); } // Fix entry for now piBlock->mContainer[IndexInDirBlock] = DirectoryID; } // Check the object size if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) { // Wrong size, correct it. BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) << " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " has wrong size " << rEntry.GetSizeInBlocks() << ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]); rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); // Mark as changed rIsModified = true; ++mNumberErrorsFound; } return true; // don't delete this entry }
bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { bool restart = true; bool isModified = false; while(restart) { std::vector<int64_t> toDelete; restart = false; // Check for validity if(dir.CheckAndFix()) { // Wasn't quite right, and has been modified BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << " had invalid entries" << (mFixErrors ? ", fixed" : "")); ++mNumberErrorsFound; isModified = true; } // Go through, and check that every entry exists and is valid BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { // Lookup the item int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); bool badEntry = false; if(piBlock != 0) { badEntry = !CheckDirectoryEntry(*en, dir.GetObjectID(), isModified); } // Item can't be found. Is it a directory? else if(en->IsDir()) { // Store the directory for later attention mDirsWhichContainLostDirs[en->GetObjectID()] = dir.GetObjectID(); } else { // Just remove the entry badEntry = true; BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); ++mNumberErrorsFound; } // Is this entry worth keeping? if(badEntry) { toDelete.push_back(en->GetObjectID()); } } if(toDelete.size() > 0) { // Delete entries from directory for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) { BOX_ERROR("Removing directory entry " << BOX_FORMAT_OBJECTID(*d) << " from " "directory " << BOX_FORMAT_OBJECTID(dir.GetObjectID())); ++mNumberErrorsFound; dir.DeleteEntry(*d); } // Mark as modified restart = true; isModified = true; // Errors found } } return isModified; }
// -------------------------------------------------------------------------- // // Function // Name: BackupStoreCheck::CheckDirectories() // Purpose: Check the directories // Created: 22/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::CheckDirectories() { // Phase 1 did this: // Checked that all the directories are readable // Built a list of all directories and files which exist on the store // // This phase will check all the files in the directories, make // a note of all directories which are missing, and do initial fixing. // The root directory is not contained inside another directory, so // it has no directory entry to scan, but we have to count it // somewhere, so we'll count it here. mNumDirectories++; // Scan all objects. for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); if(flags & Flags_IsDir) { // Found a directory. Read it in. std::string filename; StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); BackupStoreDirectory dir; { std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); dir.ReadFromStream(*file, IOStream::TimeOutInfinite); } // Flag for modifications bool isModified = CheckDirectory(dir); // Check the directory again, now that entries have been removed if(dir.CheckAndFix()) { // Wasn't quite right, and has been modified BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was still bad after all checks"); ++mNumberErrorsFound; isModified = true; } else if(isModified) { BOX_INFO("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was OK after fixing"); } if(isModified && mFixErrors) { BOX_WARNING("Writing modified directory to disk: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); RaidFileWrite fixed(mDiscSetNumber, filename); fixed.Open(true /* allow overwriting */); dir.WriteToStream(fixed); fixed.Commit(true /* convert to raid representation now */); } CountDirectoryEntries(dir); } } } }