int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
if(!OpenAccount(ID, rootDir, discSetNum, user,
NULL /* housekeeping locks the account itself */))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " for housekeeping.");
return 1;
}
HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL);
bool success = housekeeping.DoHousekeeping();
if(!success)
{
BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID)
<< " for housekeeping: perhaps a client is "
"still connected?");
return 1;
}
else
{
BOX_TRACE("Finished housekeeping on account " <<
BOX_FORMAT_ACCOUNT(ID));
return 0;
}
}
int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
NamedLock writeLock;
if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " to change name.");
return 1;
}
// Load the info
std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID,
rootDir, discSetNum, false /* Read/Write */));
info->SetAccountName(rNewAccountName);
// Save
info->Save();
BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) <<
" name changed to " << rNewAccountName);
return 0;
}
int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet,
bool ReturnNumErrorsFound)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
NamedLock writeLock;
if(!OpenAccount(ID, rootDir, discSetNum, user,
FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " for checking.");
return 1;
}
// Check it
BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet);
check.Check();
if(ReturnNumErrorsFound)
{
return check.GetNumErrorsFound();
}
else
{
return check.ErrorsFound() ? 1 : 0;
}
}
int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber,
int32_t SoftLimit, int32_t HardLimit)
{
// Load in the account database
std::auto_ptr<BackupStoreAccountDatabase> db(
BackupStoreAccountDatabase::Read(
mConfig.GetKeyValue("AccountDatabase")));
// Already exists?
if(db->EntryExists(ID))
{
BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) <<
" already exists.");
return 1;
}
// Get the user under which the daemon runs
std::string username;
{
const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server"));
if(rserverConfig.KeyExists("User"))
{
username = rserverConfig.GetKeyValue("User");
}
}
// Create it.
BackupStoreAccounts acc(*db);
acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username);
BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created.");
return 0;
}
int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr,
const char *HardLimitStr)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
NamedLock writeLock;
if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " to change limits.");
return 1;
}
// Load the info
std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir,
discSetNum, false /* Read/Write */));
// Change the limits
int blocksize = BlockSizeOfDiscSet(discSetNum);
int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize);
int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize);
CheckSoftHardLimits(softlimit, hardlimit);
info->ChangeLimits(softlimit, hardlimit);
// Save
info->Save();
BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) <<
" changed to " << softlimit << " soft, " <<
hardlimit << " hard.");
return 0;
}
// --------------------------------------------------------------------------
//
// Function
// Name: Random::Initialise()
// Purpose: Add additional randomness to the standard library initialisation
// Created: 18/6/04
//
// --------------------------------------------------------------------------
void Random::Initialise()
{
#ifdef HAVE_RANDOM_DEVICE
if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024)
{
THROW_EXCEPTION(CipherException, RandomInitFailed)
}
#else
BOX_ERROR("No random device -- additional seeding of random number "
"generator not performed.");
#endif
}
// Start the first overlapped read
void WinNamedPipeStream::StartFirstRead()
{
// create the Readable event
mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (mReadableEvent == INVALID_HANDLE_VALUE)
{
BOX_ERROR("Failed to create the Readable event: " <<
GetErrorMessage(GetLastError()));
Close();
THROW_EXCEPTION(CommonException, Internal)
}
bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut,
int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock)
{
// Load in the account database
std::auto_ptr<BackupStoreAccountDatabase> db(
BackupStoreAccountDatabase::Read(
mConfig.GetKeyValue("AccountDatabase")));
// Exists?
if(!db->EntryExists(ID))
{
BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) <<
" does not exist.");
return false;
}
// Get info from the database
BackupStoreAccounts acc(*db);
acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut);
// Get the user under which the daemon runs
std::string username;
{
const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server"));
if(rserverConfig.KeyExists("User"))
{
username = rserverConfig.GetKeyValue("User");
}
}
// Become the right user
if(!username.empty())
{
// Username specified, change...
apUser.reset(new UnixUser(username));
apUser->ChangeProcessUser(true /* temporary */);
// Change will be undone when apUser goes out of scope
// in the caller.
}
if(pLock)
{
acc.LockAccount(ID, *pLock);
}
return true;
}
int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
NamedLock writeLock;
if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " to change enabled flag.");
return 1;
}
// Load it in
std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID,
rootDir, discSetNum, false /* ReadOnly */));
info->SetAccountEnabled(enabled);
info->Save();
return 0;
}
int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
if(!OpenAccount(ID, rootDir, discSetNum, user,
NULL /* no write lock needed for this read-only operation */))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " to display info.");
return 1;
}
// Load it in
std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID,
rootDir, discSetNum, true /* ReadOnly */));
return BackupAccountControl::PrintAccountInfo(*info,
BlockSizeOfDiscSet(discSetNum));
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &)
// Purpose: Do check on file, return original container ID
// if OK, or -1 on error
// Created: 22/4/04
//
// --------------------------------------------------------------------------
int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
{
// Check that it's not the root directory ID. Having a file as
// the root directory would be bad.
if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
{
// Get that dodgy thing deleted!
BOX_ERROR("Have file as root directory. This is bad.");
return -1;
}
// Check the format of the file, and obtain the container ID
int64_t originalContainerID = -1;
if(!BackupStoreFile::VerifyEncodedFileFormat(rStream,
0 /* don't want diffing from ID */,
&originalContainerID))
{
// Didn't verify
return -1;
}
return originalContainerID;
}
void testservers_connection(SocketStream &rStream)
{
IOStreamGetLine getline(rStream);
if(typeid(rStream) == typeid(SocketStreamTLS))
{
// need to wait for some data before sending stuff, otherwise timeout test doesn't work
std::string line;
while(!getline.GetLine(line))
;
SocketStreamTLS &rtls = (SocketStreamTLS&)rStream;
std::string line1("CONNECTED:");
line1 += rtls.GetPeerCommonName();
line1 += '\n';
testservers_pause_before_reply();
rStream.Write(line1.c_str(), line1.size());
}
while(!getline.IsEOF())
{
std::string line;
while(!getline.GetLine(line))
;
if(line == "QUIT")
{
break;
}
if(line == "LARGEDATA")
{
{
// Send lots of data
char data[LARGE_DATA_BLOCK_SIZE];
for(unsigned int y = 0; y < sizeof(data); y++)
{
data[y] = y & 0xff;
}
for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s)
{
rStream.Write(data, sizeof(data), SHORT_TIMEOUT);
}
}
{
// Receive lots of data
char buf[1024];
int total = 0;
int r = 0;
while(total < LARGE_DATA_SIZE &&
(r = rStream.Read(buf, sizeof(buf), SHORT_TIMEOUT)) != 0)
{
total += r;
}
TEST_THAT(total == LARGE_DATA_SIZE);
if (total != LARGE_DATA_SIZE)
{
BOX_ERROR("Expected " <<
LARGE_DATA_SIZE << " bytes " <<
"but was " << total);
return;
}
}
{
// Send lots of data again
char data[LARGE_DATA_BLOCK_SIZE];
for(unsigned int y = 0; y < sizeof(data); y++)
{
data[y] = y & 0xff;
}
for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s)
{
rStream.Write(data, sizeof(data), SHORT_TIMEOUT);
}
}
// next!
continue;
}
std::string backwards;
for(std::string::const_reverse_iterator i(line.end()); i != std::string::const_reverse_iterator(line.begin()); ++i)
{
backwards += (*i);
}
backwards += '\n';
testservers_pause_before_reply();
rStream.Write(backwards.c_str(), backwards.size());
}
rStream.Shutdown();
rStream.Close();
}
int main(int argc, const char *argv[])
{
int returnCode = 0;
MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupctl.memleaks",
"bbackupctl")
MAINHELPER_START
Logging::SetProgramName("bbackupctl");
// Filename for configuration file?
std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE;
// See if there's another entry on the command line
int c;
std::string options("c:");
options += Logging::OptionParser::GetOptionString();
Logging::OptionParser LogLevel;
while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1)
{
switch(c)
{
case 'c':
// store argument
configFilename = optarg;
break;
default:
int ret = LogLevel.ProcessOption(c);
if(ret != 0)
{
PrintUsageAndExit(ret);
}
}
}
// Adjust arguments
argc -= optind;
argv += optind;
// Check there's a command
if(argc != 1)
{
PrintUsageAndExit(2);
}
Logging::FilterConsole(LogLevel.GetCurrentLevel());
// Read in the configuration file
BOX_INFO("Using configuration file " << configFilename);
std::string errs;
std::auto_ptr<Configuration> config(
Configuration::LoadAndVerify
(configFilename, &BackupDaemonConfigVerify, errs));
if(config.get() == 0 || !errs.empty())
{
BOX_ERROR("Invalid configuration file: " << errs);
return 1;
}
// Easier coding
const Configuration &conf(*config);
// Check there's a socket defined in the config file
if(!conf.KeyExists("CommandSocket"))
{
BOX_ERROR("Daemon isn't using a control socket, "
"could not execute command.\n"
"Add a CommandSocket declaration to the "
"bbackupd.conf file.");
return 1;
}
// Connect to socket
#ifndef WIN32
SocketStream connection;
#else /* WIN32 */
WinNamedPipeStream connection;
#endif /* ! WIN32 */
try
{
#ifdef WIN32
std::string socket = conf.GetKeyValue("CommandSocket");
connection.Connect(socket);
#else
connection.Open(Socket::TypeUNIX, conf.GetKeyValue("CommandSocket").c_str());
#endif
}
catch(...)
{
BOX_ERROR("Failed to connect to daemon control socket.\n"
"Possible causes:\n"
" * Daemon not running\n"
" * Daemon busy syncing with store server\n"
" * Another bbackupctl process is communicating with the daemon\n"
" * Daemon is waiting to recover from an error"
);
return 1;
}
// For receiving data
IOStreamGetLine getLine(connection);
// Wait for the configuration summary
std::string configSummary;
if(!getLine.GetLine(configSummary, false, PROTOCOL_DEFAULT_TIMEOUT))
{
BOX_ERROR("Failed to receive configuration summary "
"from daemon");
return 1;
}
// Was the connection rejected by the server?
if(getLine.IsEOF())
{
BOX_ERROR("Server rejected the connection. Are you running "
"bbackupctl as the same user as the daemon?");
return 1;
}
// Decode it
int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait;
if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", &autoBackup,
&updateStoreInterval, &minimumFileAge, &maxUploadWait) != 4)
{
BOX_ERROR("Config summary didn't decode.");
return 1;
}
// Print summary?
BOX_TRACE("Daemon configuration summary:\n"
" AutomaticBackup = " <<
(autoBackup?"true":"false") << "\n"
" UpdateStoreInterval = " << updateStoreInterval <<
" seconds\n"
" MinimumFileAge = " << minimumFileAge << " seconds\n"
" MaxUploadWait = " << maxUploadWait << " seconds");
std::string stateLine;
if(!getLine.GetLine(stateLine, false, PROTOCOL_DEFAULT_TIMEOUT) || getLine.IsEOF())
{
BOX_ERROR("Failed to receive state line from daemon");
return 1;
}
// Decode it
int currentState;
if(::sscanf(stateLine.c_str(), "state %d", ¤tState) != 1)
{
BOX_ERROR("Received invalid state line from daemon");
return 1;
}
BOX_TRACE("Current state: " <<
BackupDaemon::GetStateName(currentState));
Command command = Default;
std::string commandName(argv[0]);
if(commandName == "wait-for-sync")
{
command = WaitForSyncStart;
}
else if(commandName == "wait-for-end")
{
command = WaitForSyncEnd;
}
else if(commandName == "sync-and-wait")
{
command = SyncAndWaitForEnd;
}
else if(commandName == "status")
{
BOX_NOTICE("state " <<
BackupDaemon::GetStateName(currentState));
command = NoCommand;
}
switch(command)
{
case WaitForSyncStart:
case WaitForSyncEnd:
{
// Check that it's in automatic mode,
// because otherwise it'll never start
if(!autoBackup)
{
BOX_ERROR("Daemon is not in automatic mode, "
"sync will never start!");
return 1;
}
}
break;
case SyncAndWaitForEnd:
{
// send a sync command
commandName = "force-sync";
std::string cmd = commandName + "\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
connection.WriteAllBuffered();
if(currentState != 0)
{
BOX_INFO("Waiting for current sync/error state "
"to finish...");
}
}
break;
default:
{
// Normal case, just send the command given, plus a
// quit command.
std::string cmd = commandName + "\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
}
// fall through
case NoCommand:
{
// Normal case, just send the command given plus a
// quit command.
std::string cmd = "quit\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
}
}
// Read the response
std::string line;
bool syncIsRunning = false;
bool finished = false;
while(command != NoCommand && !finished && !getLine.IsEOF() &&
getLine.GetLine(line, false, PROTOCOL_DEFAULT_TIMEOUT))
{
BOX_TRACE("Received line: " << line);
if(line.substr(0, 6) == "state ")
{
std::string state_str = line.substr(6);
int state_num;
if(sscanf(state_str.c_str(), "%d", &state_num) == 1)
{
BOX_INFO("Daemon state changed to: " <<
BackupDaemon::GetStateName(state_num));
}
else
{
BOX_WARNING("Failed to parse line: " << line);
}
}
switch(command)
{
case WaitForSyncStart:
{
// Need to wait for the state change...
if(line == "start-sync")
{
// And we're done
finished = true;
}
}
break;
case WaitForSyncEnd:
case SyncAndWaitForEnd:
{
if(line == "start-sync")
{
BOX_TRACE("Sync started...");
syncIsRunning = true;
}
else if(line == "finish-sync")
{
if (syncIsRunning)
{
// And we're done
BOX_TRACE("Sync finished.");
finished = true;
}
else
{
BOX_TRACE("Previous sync finished.");
}
// daemon must still be busy
}
}
break;
default:
{
// Is this an OK or error line?
if(line == "ok")
{
BOX_TRACE("Control command "
"sent: " <<
commandName);
finished = true;
}
else if(line == "error")
{
BOX_ERROR("Control command failed: " <<
commandName << ". Check "
"command spelling.");
returnCode = 1;
finished = true;
}
}
}
}
// Send a quit command to finish nicely
connection.Write("quit\n", 5, PROTOCOL_DEFAULT_TIMEOUT);
MAINHELPER_END
#if defined WIN32 && ! defined BOX_RELEASE_BUILD
closelog();
#endif
return returnCode;
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
// Purpose: Read in the contents of the directory, recurse to other levels,
// return the maximum starting ID of any directory found.
// Created: 21/4/04
//
// --------------------------------------------------------------------------
int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
{
//TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
int64_t maxID = StartID;
// Read in all the directories, and recurse downwards
{
// If any of the directories is missing, create it.
RaidFileController &rcontroller(RaidFileController::GetController());
RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
if(!rdiscSet.IsNonRaidSet())
{
unsigned int numDiscs = rdiscSet.size();
for(unsigned int l = 0; l < numDiscs; ++l)
{
// build name
std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
EMU_STRUCT_STAT st;
if(EMU_STAT(dn.c_str(), &st) != 0 &&
errno == ENOENT)
{
if(mkdir(dn.c_str(), 0755) != 0)
{
THROW_SYS_FILE_ERROR("Failed to "
"create missing RaidFile "
"directory", dn,
RaidFileException, OSError);
}
}
}
}
std::vector<std::string> dirs;
RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
RaidFileRead::DirReadType_DirsOnly, dirs);
for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
{
// Check to see if it's the right name
int n = 0;
if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
&& n < (1<<STORE_ID_SEGMENT_LENGTH))
{
// Next level down
int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
rDirName + DIRECTORY_SEPARATOR + *i);
// Found a greater starting ID?
if(mi > maxID)
{
maxID = mi;
}
}
else
{
BOX_ERROR("Spurious or invalid directory " <<
rDirName << DIRECTORY_SEPARATOR <<
(*i) << " found, " <<
(mFixErrors?"deleting":"delete manually"));
++mNumberErrorsFound;
}
}
}
return maxID;
}
int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
NamedLock writeLock;
// Obtain a write lock, as the daemon user
if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " for deletion.");
return 1;
}
// Check user really wants to do this
if(AskForConfirmation)
{
BOX_WARNING("Really delete account " <<
BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)");
char response[256];
if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0)
{
BOX_NOTICE("Deletion cancelled.");
return 0;
}
}
// Back to original user, but write lock is maintained
user.reset();
std::auto_ptr<BackupStoreAccountDatabase> db(
BackupStoreAccountDatabase::Read(
mConfig.GetKeyValue("AccountDatabase")));
// Delete from account database
db->DeleteEntry(ID);
// Write back to disc
db->Write();
// Remove the store files...
// First, become the user specified in the config file
std::string username;
{
const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server"));
if(rserverConfig.KeyExists("User"))
{
username = rserverConfig.GetKeyValue("User");
}
}
// Become the right user
if(!username.empty())
{
// Username specified, change...
user.reset(new UnixUser(username));
user->ChangeProcessUser(true /* temporary */);
// Change will be undone when user goes out of scope
}
// Secondly, work out which directories need wiping
std::vector<std::string> toDelete;
RaidFileController &rcontroller(RaidFileController::GetController());
RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum));
for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i)
{
if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end())
{
toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir);
}
}
// NamedLock will throw an exception if it can't delete the lockfile,
// which it can't if it doesn't exist. Now that we've deleted the account,
// nobody can open it anyway, so it's safe to unlock.
writeLock.ReleaseLock();
int retcode = 0;
// Thirdly, delete the directories...
for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
{
BOX_NOTICE("Deleting store directory " << (*d) << "...");
// Just use the rm command to delete the files
std::string cmd("rm -rf ");
cmd += *d;
// Run command
if(::system(cmd.c_str()) != 0)
{
BOX_ERROR("Failed to delete files in " << (*d) <<
", delete them manually.");
retcode = 1;
}
}
// Success!
return retcode;
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::CheckObjectsDir(int64_t)
// Purpose: Check all the files within this directory which has
// the given starting ID.
// Created: 22/4/04
//
// --------------------------------------------------------------------------
void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
{
// Make directory name -- first generate the filename of an entry in it
std::string dirName;
StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
// Check expectations
ASSERT(dirName.size() > 4 &&
dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR);
// Remove the filename from it
dirName.resize(dirName.size() - 4); // four chars for "/o00"
// Check directory exists
if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
{
BOX_WARNING("RaidFile dir " << dirName << " does not exist");
return;
}
// Read directory contents
std::vector<std::string> files;
RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
RaidFileRead::DirReadType_FilesOnly, files);
// Array of things present
bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
{
idsPresent[l] = false;
}
// Parse each entry, building up a list of object IDs which are present in the dir.
// This is done so that whatever order is retured from the directory, objects are scanned
// in order.
// Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
{
bool fileOK = true;
int n = 0;
if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
&& n < (1<<STORE_ID_SEGMENT_LENGTH))
{
// Filename is valid, mark as existing
idsPresent[n] = true;
}
// No other files should be present in subdirectories
else if(StartID != 0)
{
fileOK = false;
}
// info and refcount databases are OK in the root directory
else if(*i == "info" || *i == "refcount.db" ||
*i == "refcount.rdb" || *i == "refcount.rdbX")
{
fileOK = true;
}
else
{
fileOK = false;
}
if(!fileOK)
{
// Unexpected or bad file, delete it
BOX_ERROR("Spurious file " << dirName <<
DIRECTORY_SEPARATOR << (*i) << " found" <<
(mFixErrors?", deleting":""));
++mNumberErrorsFound;
if(mFixErrors)
{
RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
del.Delete();
}
}
}
// Check all the objects found in this directory
for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
{
if(idsPresent[i])
{
// Check the object is OK, and add entry
char leaf[8];
::snprintf(leaf, sizeof(leaf),
DIRECTORY_SEPARATOR "o%02x", i);
if(!CheckAndAddObject(StartID | i, dirName + leaf))
{
// File was bad, delete it
BOX_ERROR("Corrupted file " << dirName <<
leaf << " found" <<
(mFixErrors?", deleting":""));
++mNumberErrorsFound;
if(mFixErrors)
{
RaidFileWrite del(mDiscSetNumber, dirName + leaf);
del.Delete();
}
}
}
}
}
BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile(
int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory,
const std::string &rDirectoryFilename,
BackupStoreInfo& rBackupStoreInfo)
{
// Find the entry inside the directory
bool wasDeleted = false;
bool wasOldVersion = false;
int64_t deletedFileSizeInBlocks = 0;
// A pointer to an object which requires committing if the directory save goes OK
std::auto_ptr<RaidFileWrite> padjustedEntry;
// BLOCK
{
BackupStoreRefCountDatabase::refcount_t refs =
mapNewRefs->GetRefCount(ObjectID);
BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
if(pentry == 0)
{
BOX_ERROR("Housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID) << " "
"found error: object " <<
BOX_FORMAT_OBJECTID(ObjectID) << " "
"not found in dir " <<
BOX_FORMAT_OBJECTID(InDirectory) << ", "
"indicates logic error/corruption? Run "
"bbstoreaccounts check <accid> fix");
mErrorCount++;
return refs;
}
// Record the flags it's got set
wasDeleted = pentry->IsDeleted();
wasOldVersion = pentry->IsOld();
// Check this should be deleted
if(!wasDeleted && !wasOldVersion)
{
// Things changed since we were last around
return refs;
}
// Record size
deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
if(refs > 1)
{
// Not safe to merge patches if someone else has a
// reference to this object, so just remove the
// directory entry and return.
rDirectory.DeleteEntry(ObjectID);
if(wasDeleted)
{
rBackupStoreInfo.AdjustNumDeletedFiles(-1);
}
if(wasOldVersion)
{
rBackupStoreInfo.AdjustNumOldFiles(-1);
}
mapNewRefs->RemoveReference(ObjectID);
return refs - 1;
}
// If the entry is involved in a chain of patches, it needs to be handled
// a bit more carefully.
if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
{
// This entry is a patch from a newer entry. Just need to update the info on that entry.
BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Change the info in the newer entry so that this no longer points to this entry
pnewer->SetDependsOlder(0);
}
else if(pentry->GetDependsOlder() != 0)
{
BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
if(pentry->GetDependsNewer() == 0)
{
// There exists an older version which depends on this one. Need to combine the two over that one.
// Adjust the other entry in the directory
if(polder == 0 || polder->GetDependsNewer() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Change the info in the older entry so that this no longer points to this entry
polder->SetDependsNewer(0);
}
else
{
// This entry is in the middle of a chain, and two patches need combining.
// First, adjust the directory entries
BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
|| polder == 0 || polder->GetDependsNewer() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Remove the middle entry from the linked list by simply using the values from this entry
pnewer->SetDependsOlder(pentry->GetDependsOlder());
polder->SetDependsNewer(pentry->GetDependsNewer());
}
// COMMON CODE to both cases
// Generate the filename of the older version
std::string objFilenameOlder;
MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
// Open it twice (it's the diff)
std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
// Open this file
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
// And open a write file to overwrite the other directory entry
padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
objFilenameOlder, mapNewRefs->GetRefCount(ObjectID)));
padjustedEntry->Open(true /* allow overwriting */);
if(pentry->GetDependsNewer() == 0)
{
// There exists an older version which depends on this one. Need to combine the two over that one.
BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
}
else
{
// This entry is in the middle of a chain, and two patches need combining.
BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
}
// The file will be committed later when the directory is safely commited.
// Work out the adjusted size
int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
mBlocksUsedDelta += sizeDelta;
if(polder->IsDeleted())
{
mBlocksInDeletedFilesDelta += sizeDelta;
}
if(polder->IsOld())
{
mBlocksInOldFilesDelta += sizeDelta;
}
polder->SetSizeInBlocks(newSize);
}
// pentry no longer valid
}
// Delete it from the directory
rDirectory.DeleteEntry(ObjectID);
// Save directory back to disc
// BLOCK
{
RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
mapNewRefs->GetRefCount(InDirectory));
writeDir.Open(true /* allow overwriting */);
rDirectory.WriteToStream(writeDir);
// Get the disc usage (must do this before commiting it)
int64_t new_size = writeDir.GetDiscUsageInBlocks();
// Commit directory
writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
// Adjust block counts if the directory itself changed in size
int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks();
int64_t adjust = new_size - original_size;
mBlocksUsedDelta += adjust;
mBlocksInDirectoriesDelta += adjust;
UpdateDirectorySize(rDirectory, new_size);
}
// Commit any new adjusted entry
if(padjustedEntry.get() != 0)
{
padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
padjustedEntry.reset(); // delete it now
}
// Drop reference count by one. Must now be zero, to delete the file.
bool remaining_refs = mapNewRefs->RemoveReference(ObjectID);
ASSERT(!remaining_refs);
// Delete from disc
BOX_TRACE("Removing unreferenced object " <<
BOX_FORMAT_OBJECTID(ObjectID));
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID));
del.Delete();
// Adjust counts for the file
++mFilesDeleted;
mBlocksUsedDelta -= deletedFileSizeInBlocks;
if(wasDeleted)
{
mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
rBackupStoreInfo.AdjustNumDeletedFiles(-1);
}
if(wasOldVersion)
{
mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
rBackupStoreInfo.AdjustNumOldFiles(-1);
}
// Delete the directory?
// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
if(rDirectory.GetNumberOfEntries() == 0)
{
// Candidate for deletion
mEmptyDirectories.push_back(InDirectory);
}
return 0;
}
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
int64_t DirectoryID, bool& rIsModified)
{
int32_t IndexInDirBlock;
IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock);
ASSERT(piBlock != 0);
uint8_t iflags = GetFlags(piBlock, IndexInDirBlock);
// Is the type the same?
if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir())
{
// Entry is of wrong type
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(DirectoryID) <<
" references object " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
" which has a different type than expected.");
++mNumberErrorsFound;
return false; // remove this entry
}
// Check that the entry is not already contained.
if(iflags & Flags_IsContained)
{
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(DirectoryID) <<
" references object " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
" which is already contained.");
++mNumberErrorsFound;
return false; // remove this entry
}
// Not already contained by another directory.
// Don't set the flag until later, after we finish repairing
// the directory and removing all bad entries.
// Check that the container ID of the object is correct
if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
{
// Needs fixing...
if(iflags & Flags_IsDir)
{
// Add to will fix later list
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
<< " has wrong container ID.");
mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
++mNumberErrorsFound;
}
else
{
// This is OK for files, they might move
BOX_INFO("File ID " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
<< " has different container ID, "
"probably moved");
}
// Fix entry for now
piBlock->mContainer[IndexInDirBlock] = DirectoryID;
}
// Check the object size
if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
{
// Wrong size, correct it.
BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) <<
" entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
" has wrong size " << rEntry.GetSizeInBlocks() <<
", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
// Mark as changed
rIsModified = true;
++mNumberErrorsFound;
}
return true; // don't delete this entry
}
bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
{
bool restart = true;
bool isModified = false;
while(restart)
{
std::vector<int64_t> toDelete;
restart = false;
// Check for validity
if(dir.CheckAndFix())
{
// Wasn't quite right, and has been modified
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
" had invalid entries" <<
(mFixErrors ? ", fixed" : ""));
++mNumberErrorsFound;
isModified = true;
}
// Go through, and check that every entry exists and is valid
BackupStoreDirectory::Iterator i(dir);
BackupStoreDirectory::Entry *en = 0;
while((en = i.Next()) != 0)
{
// Lookup the item
int32_t iIndex;
IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
bool badEntry = false;
if(piBlock != 0)
{
badEntry = !CheckDirectoryEntry(*en,
dir.GetObjectID(), isModified);
}
// Item can't be found. Is it a directory?
else if(en->IsDir())
{
// Store the directory for later attention
mDirsWhichContainLostDirs[en->GetObjectID()] =
dir.GetObjectID();
}
else
{
// Just remove the entry
badEntry = true;
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
" references object " <<
BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
" which does not exist.");
++mNumberErrorsFound;
}
// Is this entry worth keeping?
if(badEntry)
{
toDelete.push_back(en->GetObjectID());
}
}
if(toDelete.size() > 0)
{
// Delete entries from directory
for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
{
BOX_ERROR("Removing directory entry " <<
BOX_FORMAT_OBJECTID(*d) << " from "
"directory " <<
BOX_FORMAT_OBJECTID(dir.GetObjectID()));
++mNumberErrorsFound;
dir.DeleteEntry(*d);
}
// Mark as modified
restart = true;
isModified = true;
// Errors found
}
}
return isModified;
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::CheckDirectories()
// Purpose: Check the directories
// Created: 22/4/04
//
// --------------------------------------------------------------------------
void BackupStoreCheck::CheckDirectories()
{
// Phase 1 did this:
// Checked that all the directories are readable
// Built a list of all directories and files which exist on the store
//
// This phase will check all the files in the directories, make
// a note of all directories which are missing, and do initial fixing.
// The root directory is not contained inside another directory, so
// it has no directory entry to scan, but we have to count it
// somewhere, so we'll count it here.
mNumDirectories++;
// Scan all objects.
for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
{
IDBlock *pblock = i->second;
int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
for(int e = 0; e < bentries; ++e)
{
uint8_t flags = GetFlags(pblock, e);
if(flags & Flags_IsDir)
{
// Found a directory. Read it in.
std::string filename;
StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
BackupStoreDirectory dir;
{
std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
}
// Flag for modifications
bool isModified = CheckDirectory(dir);
// Check the directory again, now that entries have been removed
if(dir.CheckAndFix())
{
// Wasn't quite right, and has been modified
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
" was still bad after all checks");
++mNumberErrorsFound;
isModified = true;
}
else if(isModified)
{
BOX_INFO("Directory ID " <<
BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
" was OK after fixing");
}
if(isModified && mFixErrors)
{
BOX_WARNING("Writing modified directory to disk: " <<
BOX_FORMAT_OBJECTID(pblock->mID[e]));
RaidFileWrite fixed(mDiscSetNumber, filename);
fixed.Open(true /* allow overwriting */);
dir.WriteToStream(fixed);
fixed.Commit(true /* convert to raid representation now */);
}
CountDirectoryEntries(dir);
}
}
}
}
