int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID)
{
std::string rootDir;
int discSetNum;
std::auto_ptr<UnixUser> user; // used to reset uid when we return
if(!OpenAccount(ID, rootDir, discSetNum, user,
NULL /* housekeeping locks the account itself */))
{
BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID)
<< " for housekeeping.");
return 1;
}
HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL);
bool success = housekeeping.DoHousekeeping();
if(!success)
{
BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID)
<< " for housekeeping: perhaps a client is "
"still connected?");
return 1;
}
else
{
BOX_TRACE("Finished housekeeping on account " <<
BOX_FORMAT_ACCOUNT(ID));
return 0;
}
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::CheckObjects()
// Purpose: Read in the contents of the directory, recurse to other levels,
// checking objects for sanity and readability
// Created: 21/4/04
//
// --------------------------------------------------------------------------
void BackupStoreCheck::CheckObjects()
{
// Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
int64_t maxDir = 0;
// Find the maximum directory starting ID
{
// Make sure the starting root dir doesn't end with '/'.
std::string start(mStoreRoot);
if(start.size() > 0 && (
start[start.size() - 1] == '/' ||
start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR))
{
start.resize(start.size() - 1);
}
maxDir = CheckObjectsScanDir(0, 1, start);
BOX_TRACE("Max dir starting ID is " <<
BOX_FORMAT_OBJECTID(maxDir));
}
// Then go through and scan all the objects within those directories
for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
{
CheckObjectsDir(d);
}
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreCheck::DumpObjectInfo()
// Purpose: Debug only. Trace out all object info.
// Created: 22/4/04
//
// --------------------------------------------------------------------------
void BackupStoreCheck::DumpObjectInfo()
{
for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
{
IDBlock *pblock = i->second;
int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
BOX_TRACE("BLOCK @ " << BOX_FORMAT_HEX32(pblock) <<
", " << bentries << " entries");
for(int e = 0; e < bentries; ++e)
{
uint8_t flags = GetFlags(pblock, e);
BOX_TRACE(std::hex <<
"id " << pblock->mID[e] <<
", c " << pblock->mContainer[e] <<
", " << ((flags & Flags_IsDir)?"dir":"file") <<
", " << ((flags & Flags_IsContained) ?
"contained":"unattached"));
}
}
}
// --------------------------------------------------------------------------
//
// Function
// Name: int GetNextWindowSize(int bytesChange,
// box_time_t timeElapsed, int rttEstimateMillis)
// Purpose: Calculate the next recommended window size, given the
// number of bytes sent since the previous recommendation,
// and the time elapsed.
// Created: 11/02/2012
//
// --------------------------------------------------------------------------
int TcpNice::GetNextWindowSize(int bytesChange, box_time_t timeElapsed,
int rttEstimateMicros)
{
int epsilon = (mAlphaStar * 1000000) / rttEstimateMicros;
// timeElapsed is in microseconds, so this will fail for T > 2000 seconds
int rateLastPeriod = ((uint64_t)bytesChange * 1000000 / timeElapsed);
int rawAdjustment = epsilon + rateLastPeriod -
mRateEstimateMovingAverage[0];
int gammaAdjustment = (rawAdjustment * mGammaPercent) / 100;
int newWindowSize = mLastWindowSize + gammaAdjustment;
int newRateEstimateMovingAverage =
(((100 - mDeltaPercent) * mRateEstimateMovingAverage[1]) / 100) +
((mDeltaPercent * rateLastPeriod) / 100);
/*
* b is the number of bytes sent during the previous control period
* T is the length (in us) of the previous control period
* rtt is the round trip time (in us) reported by the kernel on the socket
* e is epsilon, a parameter of the formula, calculated as alpha/rtt
* rb is the actual rate (goodput) over the previous period
* rbhat is the previous (last-but-one) EWMA rate estimate
* raw is the unscaled adjustment to the window size
* gamma is the scaled adjustment to the window size
* wb is the final window size
*/
BOX_TRACE("TcpNice: "
"b=" << bytesChange << ", "
"T=" << timeElapsed << ", "
"rtt=" << rttEstimateMicros << ", "
"e=" << epsilon << ", "
"rb=" << rateLastPeriod << ", "
"rbhat=" << newRateEstimateMovingAverage << ", "
"raw=" << rawAdjustment << ", "
"gamma=" << gammaAdjustment << ", "
"wb=" << newWindowSize);
mRateEstimateMovingAverage[0] = mRateEstimateMovingAverage[1];
mRateEstimateMovingAverage[1] = newRateEstimateMovingAverage;
mLastWindowSize = newWindowSize;
return newWindowSize;
}
void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror)
{
BOX_TRACE("Setup for error: " << filename <<
", after " << errorafter <<
", err " << errortoreturn <<
", syscall " << syscalltoerror);
intercept_count = 1;
intercept_filename = filename;
intercept_filedes = -1;
intercept_errorafter = errorafter;
intercept_syscall = syscalltoerror;
intercept_errno = errortoreturn;
intercept_filepos = 0;
intercept_delay_ms = 0;
}
void intercept_setup_delay(const char *filename, unsigned int delay_after,
int delay_ms, int syscall_to_delay, int num_delays)
{
BOX_TRACE("Setup for delay: " << filename <<
", after " << delay_after <<
", wait " << delay_ms << " ms" <<
", times " << num_delays <<
", syscall " << syscall_to_delay);
intercept_count = num_delays;
intercept_filename = filename;
intercept_filedes = -1;
intercept_errorafter = delay_after;
intercept_syscall = syscall_to_delay;
intercept_errno = 0;
intercept_filepos = 0;
intercept_delay_ms = delay_ms;
}
// --------------------------------------------------------------------------
//
// Function
// Name: ProtocolUncertainStream::Read(void *, int, int)
// Purpose: As interface.
// Created: 2003/12/05
//
// --------------------------------------------------------------------------
int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout)
{
// Finished?
if(mFinished)
{
return 0;
}
int read = 0;
while(read < NBytes)
{
// Anything we can get from the current block?
ASSERT(mBytesLeftInCurrentBlock >= 0);
if(mBytesLeftInCurrentBlock > 0)
{
// Yes, let's use some of these up
int toRead = (NBytes - read);
if(toRead > mBytesLeftInCurrentBlock)
{
// Adjust downwards to only read stuff out of the current block
toRead = mBytesLeftInCurrentBlock;
}
BOX_TRACE("Reading " << toRead << " bytes from stream");
// Read it
int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout);
// Give up now if it didn't return anything
if(r == 0)
{
BOX_TRACE("Read " << r << " bytes from "
"stream, returning");
return read;
}
// Adjust counts of bytes by the bytes recieved
read += r;
mBytesLeftInCurrentBlock -= r;
// stop now if the stream returned less than we asked for -- avoid blocking
if(r != toRead)
{
BOX_TRACE("Read " << r << " bytes from "
"stream, returning");
return read;
}
}
else
{
// Read the header byte to find out how much there is
// in the next block
uint8_t header;
if(mrSource.Read(&header, 1, Timeout) == 0)
{
// Didn't get the byte, return now
BOX_TRACE("Read 0 bytes of block header, "
"returning with " << read << " bytes "
"read this time");
return read;
}
// Interpret the byte...
if(header == Protocol::ProtocolStreamHeader_EndOfStream)
{
// All done.
mFinished = true;
BOX_TRACE("Stream finished, returning with " <<
read << " bytes read this time");
return read;
}
else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue)
{
// get size of the block from the Protocol's lovely list
mBytesLeftInCurrentBlock = Protocol::sProtocolStreamHeaderLengths[header];
}
else if(header == Protocol::ProtocolStreamHeader_SizeIs64k)
{
// 64k
mBytesLeftInCurrentBlock = (64*1024);
}
else
{
// Bad. It used the reserved values.
THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader)
}
BOX_TRACE("Read header byte " << (int)header << ", "
"next block has " <<
mBytesLeftInCurrentBlock << " bytes");
}
}
// Return the number read
return read;
}
// Count valid remaining entries and the number of blocks in them.
void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir)
{
BackupStoreDirectory::Iterator i(dir);
BackupStoreDirectory::Entry *en = 0;
while((en = i.Next()) != 0)
{
int32_t iIndex;
IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
bool badEntry = false;
bool wasAlreadyContained = false;
ASSERT(piBlock != 0 ||
mDirsWhichContainLostDirs.find(en->GetObjectID())
!= mDirsWhichContainLostDirs.end());
if (piBlock)
{
// Normally it would exist and this
// check would not be necessary, but
// we might have missing directories
// that we will recreate later.
// cf mDirsWhichContainLostDirs.
uint8_t iflags = GetFlags(piBlock, iIndex);
wasAlreadyContained = (iflags & Flags_IsContained);
SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
}
if(wasAlreadyContained)
{
// don't double-count objects that are
// contained by another directory as well.
}
else if(en->IsDir())
{
mNumDirectories++;
}
else if(!en->IsFile())
{
BOX_TRACE("Not counting object " <<
BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
" with flags " << en->GetFlags());
}
else // it's a file
{
// Add to sizes?
// If piBlock was zero, then wasAlreadyContained
// might be uninitialized; but we only process
// files here, and if a file's piBlock was zero
// then badEntry would be set above, so we
// wouldn't be here.
ASSERT(!badEntry)
// It can be both old and deleted.
// If neither, then it's current.
if(en->IsDeleted())
{
mNumDeletedFiles++;
mBlocksInDeletedFiles += en->GetSizeInBlocks();
}
if(en->IsOld())
{
mNumOldFiles++;
mBlocksInOldFiles += en->GetSizeInBlocks();
}
if(!en->IsDeleted() && !en->IsOld())
{
mNumCurrentFiles++;
mBlocksInCurrentFiles += en->GetSizeInBlocks();
}
}
mapNewRefs->AddReference(en->GetObjectID());
}
}
// Test stream based interface
int test_stream()
{
// Make a load of compressible data to compress
CollectInBufferStream source;
uint16_t data[1024];
for(int x = 0; x < 1024; ++x)
{
data[x] = x;
}
for(int x = 0; x < (32*1024); ++x)
{
source.Write(data, (x % 1024) * 2);
}
source.SetForReading();
// Straight compress from one stream to another
{
CollectInBufferStream *poutput = new CollectInBufferStream;
CompressStream compress(poutput, true /* take ownership */, false /* read */, true /* write */);
source.CopyStreamTo(compress);
compress.Close();
poutput->SetForReading();
// Check sizes
TEST_THAT(poutput->GetSize() < source.GetSize());
BOX_TRACE("compressed size = " << poutput->GetSize() <<
", source size = " << source.GetSize());
// Decompress the data
{
CollectInBufferStream decompressed;
CompressStream decompress(poutput, false /* don't take ownership */, true /* read */, false /* write */);
decompress.CopyStreamTo(decompressed);
decompress.Close();
TEST_THAT(decompressed.GetSize() == source.GetSize());
TEST_THAT(::memcmp(decompressed.GetBuffer(), source.GetBuffer(), decompressed.GetSize()) == 0);
}
// Don't delete poutput, let mem leak testing ensure it's deleted.
}
// Set source to the beginning
source.Seek(0, IOStream::SeekType_Absolute);
// Test where the same stream compresses and decompresses, should be fun!
{
CollectInBufferStream output;
CopyInToOutStream copyer;
CompressStream compress(©er, false /* no ownership */, true, true);
bool done = false;
int count = 0;
int written = 0;
while(!done)
{
++count;
bool do_sync = (count % 256) == 0;
uint8_t buffer[4096];
int r = source.Read(buffer, sizeof(buffer), IOStream::TimeOutInfinite);
if(r == 0)
{
done = true;
compress.Close();
}
else
{
compress.Write(buffer, r);
written += r;
if(do_sync)
{
compress.WriteAllBuffered();
}
}
int r2 = 0;
do
{
r2 = compress.Read(buffer, sizeof(buffer), IOStream::TimeOutInfinite);
if(r2 > 0)
{
output.Write(buffer, r2);
}
} while(r2 > 0);
if(do_sync && r != 0)
{
// Check that everything is synced
TEST_THAT(output.GetSize() == written);
TEST_THAT(::memcmp(output.GetBuffer(), source.GetBuffer(), output.GetSize()) == 0);
}
}
output.SetForReading();
// Test that it's the same
TEST_THAT(output.GetSize() == source.GetSize());
TEST_THAT(::memcmp(output.GetBuffer(), source.GetBuffer(), output.GetSize()) == 0);
}
return 0;
}
int main(int argc, const char *argv[])
{
int returnCode = 0;
MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupctl.memleaks",
"bbackupctl")
MAINHELPER_START
Logging::SetProgramName("bbackupctl");
// Filename for configuration file?
std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE;
// See if there's another entry on the command line
int c;
std::string options("c:");
options += Logging::OptionParser::GetOptionString();
Logging::OptionParser LogLevel;
while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1)
{
switch(c)
{
case 'c':
// store argument
configFilename = optarg;
break;
default:
int ret = LogLevel.ProcessOption(c);
if(ret != 0)
{
PrintUsageAndExit(ret);
}
}
}
// Adjust arguments
argc -= optind;
argv += optind;
// Check there's a command
if(argc != 1)
{
PrintUsageAndExit(2);
}
Logging::FilterConsole(LogLevel.GetCurrentLevel());
// Read in the configuration file
BOX_INFO("Using configuration file " << configFilename);
std::string errs;
std::auto_ptr<Configuration> config(
Configuration::LoadAndVerify
(configFilename, &BackupDaemonConfigVerify, errs));
if(config.get() == 0 || !errs.empty())
{
BOX_ERROR("Invalid configuration file: " << errs);
return 1;
}
// Easier coding
const Configuration &conf(*config);
// Check there's a socket defined in the config file
if(!conf.KeyExists("CommandSocket"))
{
BOX_ERROR("Daemon isn't using a control socket, "
"could not execute command.\n"
"Add a CommandSocket declaration to the "
"bbackupd.conf file.");
return 1;
}
// Connect to socket
#ifndef WIN32
SocketStream connection;
#else /* WIN32 */
WinNamedPipeStream connection;
#endif /* ! WIN32 */
try
{
#ifdef WIN32
std::string socket = conf.GetKeyValue("CommandSocket");
connection.Connect(socket);
#else
connection.Open(Socket::TypeUNIX, conf.GetKeyValue("CommandSocket").c_str());
#endif
}
catch(...)
{
BOX_ERROR("Failed to connect to daemon control socket.\n"
"Possible causes:\n"
" * Daemon not running\n"
" * Daemon busy syncing with store server\n"
" * Another bbackupctl process is communicating with the daemon\n"
" * Daemon is waiting to recover from an error"
);
return 1;
}
// For receiving data
IOStreamGetLine getLine(connection);
// Wait for the configuration summary
std::string configSummary;
if(!getLine.GetLine(configSummary, false, PROTOCOL_DEFAULT_TIMEOUT))
{
BOX_ERROR("Failed to receive configuration summary "
"from daemon");
return 1;
}
// Was the connection rejected by the server?
if(getLine.IsEOF())
{
BOX_ERROR("Server rejected the connection. Are you running "
"bbackupctl as the same user as the daemon?");
return 1;
}
// Decode it
int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait;
if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", &autoBackup,
&updateStoreInterval, &minimumFileAge, &maxUploadWait) != 4)
{
BOX_ERROR("Config summary didn't decode.");
return 1;
}
// Print summary?
BOX_TRACE("Daemon configuration summary:\n"
" AutomaticBackup = " <<
(autoBackup?"true":"false") << "\n"
" UpdateStoreInterval = " << updateStoreInterval <<
" seconds\n"
" MinimumFileAge = " << minimumFileAge << " seconds\n"
" MaxUploadWait = " << maxUploadWait << " seconds");
std::string stateLine;
if(!getLine.GetLine(stateLine, false, PROTOCOL_DEFAULT_TIMEOUT) || getLine.IsEOF())
{
BOX_ERROR("Failed to receive state line from daemon");
return 1;
}
// Decode it
int currentState;
if(::sscanf(stateLine.c_str(), "state %d", ¤tState) != 1)
{
BOX_ERROR("Received invalid state line from daemon");
return 1;
}
BOX_TRACE("Current state: " <<
BackupDaemon::GetStateName(currentState));
Command command = Default;
std::string commandName(argv[0]);
if(commandName == "wait-for-sync")
{
command = WaitForSyncStart;
}
else if(commandName == "wait-for-end")
{
command = WaitForSyncEnd;
}
else if(commandName == "sync-and-wait")
{
command = SyncAndWaitForEnd;
}
else if(commandName == "status")
{
BOX_NOTICE("state " <<
BackupDaemon::GetStateName(currentState));
command = NoCommand;
}
switch(command)
{
case WaitForSyncStart:
case WaitForSyncEnd:
{
// Check that it's in automatic mode,
// because otherwise it'll never start
if(!autoBackup)
{
BOX_ERROR("Daemon is not in automatic mode, "
"sync will never start!");
return 1;
}
}
break;
case SyncAndWaitForEnd:
{
// send a sync command
commandName = "force-sync";
std::string cmd = commandName + "\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
connection.WriteAllBuffered();
if(currentState != 0)
{
BOX_INFO("Waiting for current sync/error state "
"to finish...");
}
}
break;
default:
{
// Normal case, just send the command given, plus a
// quit command.
std::string cmd = commandName + "\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
}
// fall through
case NoCommand:
{
// Normal case, just send the command given plus a
// quit command.
std::string cmd = "quit\n";
connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT);
}
}
// Read the response
std::string line;
bool syncIsRunning = false;
bool finished = false;
while(command != NoCommand && !finished && !getLine.IsEOF() &&
getLine.GetLine(line, false, PROTOCOL_DEFAULT_TIMEOUT))
{
BOX_TRACE("Received line: " << line);
if(line.substr(0, 6) == "state ")
{
std::string state_str = line.substr(6);
int state_num;
if(sscanf(state_str.c_str(), "%d", &state_num) == 1)
{
BOX_INFO("Daemon state changed to: " <<
BackupDaemon::GetStateName(state_num));
}
else
{
BOX_WARNING("Failed to parse line: " << line);
}
}
switch(command)
{
case WaitForSyncStart:
{
// Need to wait for the state change...
if(line == "start-sync")
{
// And we're done
finished = true;
}
}
break;
case WaitForSyncEnd:
case SyncAndWaitForEnd:
{
if(line == "start-sync")
{
BOX_TRACE("Sync started...");
syncIsRunning = true;
}
else if(line == "finish-sync")
{
if (syncIsRunning)
{
// And we're done
BOX_TRACE("Sync finished.");
finished = true;
}
else
{
BOX_TRACE("Previous sync finished.");
}
// daemon must still be busy
}
}
break;
default:
{
// Is this an OK or error line?
if(line == "ok")
{
BOX_TRACE("Control command "
"sent: " <<
commandName);
finished = true;
}
else if(line == "error")
{
BOX_ERROR("Control command failed: " <<
commandName << ". Check "
"command spelling.");
returnCode = 1;
finished = true;
}
}
}
}
// Send a quit command to finish nicely
connection.Write("quit\n", 5, PROTOCOL_DEFAULT_TIMEOUT);
MAINHELPER_END
#if defined WIN32 && ! defined BOX_RELEASE_BUILD
closelog();
#endif
return returnCode;
}
// --------------------------------------------------------------------------
//
// Function
// Name: HTTPRequest::ParseHeaders(IOStreamGetLine &, int)
// Purpose: Private. Parse the headers of the request
// Created: 26/3/04
//
// --------------------------------------------------------------------------
void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout)
{
std::string header;
bool haveHeader = false;
while(true)
{
if(rGetLine.IsEOF())
{
// Header terminates unexpectedly
THROW_EXCEPTION(HTTPException, BadRequest)
}
std::string currentLine;
if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout))
{
// Timeout
THROW_EXCEPTION(HTTPException, RequestReadFailed)
}
// Is this a continuation of the previous line?
bool processHeader = haveHeader;
if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t'))
{
// A continuation, don't process anything yet
processHeader = false;
}
//TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str());
// Parse the header -- this will actually process the header
// from the previous run around the loop.
if(processHeader)
{
// Find where the : is in the line
const char *h = header.c_str();
int p = 0;
while(h[p] != '\0' && h[p] != ':')
{
++p;
}
// Skip white space
int dataStart = p + 1;
while(h[dataStart] == ' ' || h[dataStart] == '\t')
{
++dataStart;
}
std::string header_name(ToLowerCase(std::string(h,
p)));
if (header_name == "content-length")
{
// Decode number
long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK
if(len < 0) len = 0;
// Store
mContentLength = len;
}
else if (header_name == "content-type")
{
// Store rest of string as content type
mContentType = h + dataStart;
}
else if (header_name == "host")
{
// Store host header
mHostName = h + dataStart;
// Is there a port number to split off?
std::string::size_type colon = mHostName.find_first_of(':');
if(colon != std::string::npos)
{
// There's a port in the string... attempt to turn it into an int
mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10);
// Truncate the string to just the hostname
mHostName = mHostName.substr(0, colon);
BOX_TRACE("Host: header, hostname = " <<
"'" << mHostName << "', host "
"port = " << mHostPort);
}
}
else if (header_name == "cookie")
{
// Parse cookies
ParseCookies(header, dataStart);
}
else if (header_name == "connection")
{
// Connection header, what is required?
const char *v = h + dataStart;
if(::strcasecmp(v, "close") == 0)
{
mClientKeepAliveRequested = false;
}
else if(::strcasecmp(v, "keep-alive") == 0)
{
mClientKeepAliveRequested = true;
}
// else don't understand, just assume default for protocol version
}
else
{
mExtraHeaders.push_back(Header(header_name,
h + dataStart));
}
// Unset have header flag, as it's now been processed
haveHeader = false;
}
// Store the chunk of header the for next time round
if(haveHeader)
{
header += currentLine;
}
else
{
header = currentLine;
haveHeader = true;
}
// End of headers?
if(currentLine.empty())
{
// All done!
break;
}
}
}
// --------------------------------------------------------------------------
//
// Function
// Name: HTTPRequest::Receive(IOStreamGetLine &, int)
// Purpose: Read the request from an IOStreamGetLine (and
// attached stream).
// Returns false if there was no valid request,
// probably due to a kept-alive connection closing.
// Created: 26/3/04
//
// --------------------------------------------------------------------------
bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout)
{
// Check caller's logic
if(mMethod != Method_UNINITIALISED)
{
THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead);
}
// Read the first line, which is of a different format to the rest of the lines
std::string requestLine;
if(!rGetLine.GetLine(requestLine, false /* no preprocessing */, Timeout))
{
// Didn't get the request line, probably end of connection which had been kept alive
return false;
}
BOX_TRACE("Request line: " << requestLine);
// Check the method
size_t p = 0; // current position in string
p = requestLine.find(' '); // end of first word
if(p == std::string::npos)
{
// No terminating space, looks bad
p = requestLine.size();
}
else
{
mHttpVerb = requestLine.substr(0, p);
if (mHttpVerb == "GET")
{
mMethod = Method_GET;
}
else if (mHttpVerb == "HEAD")
{
mMethod = Method_HEAD;
}
else if (mHttpVerb == "POST")
{
mMethod = Method_POST;
}
else if (mHttpVerb == "PUT")
{
mMethod = Method_PUT;
}
else
{
mMethod = Method_UNKNOWN;
}
}
// Skip spaces to find URI
const char *requestLinePtr = requestLine.c_str();
while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ')
{
++p;
}
// Check there's a URI following...
if(requestLinePtr[p] == '\0')
{
// Didn't get the request line, probably end of connection which had been kept alive
return false;
}
// Read the URI, unescaping any %XX hex codes
while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0')
{
// End of URI, on to query string?
if(requestLinePtr[p] == '?')
{
// Put the rest into the query string, without escaping anything
++p;
while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0')
{
mQueryString += requestLinePtr[p];
++p;
}
break;
}
// Needs unescaping?
else if(requestLinePtr[p] == '+')
{
mRequestURI += ' ';
}
else if(requestLinePtr[p] == '%')
{
// Be tolerant about this... bad things are silently accepted,
// rather than throwing an error.
char code[4] = {0,0,0,0};
code[0] = requestLinePtr[++p];
if(code[0] != '\0')
{
code[1] = requestLinePtr[++p];
}
// Convert into a char code
long c = ::strtol(code, NULL, 16);
// Accept it?
if(c > 0 && c <= 255)
{
mRequestURI += (char)c;
}
}
else
{
// Simple copy of character
mRequestURI += requestLinePtr[p];
}
++p;
}
// End of URL?
if(requestLinePtr[p] == '\0')
{
// Assume HTTP 0.9
mHTTPVersion = HTTPVersion_0_9;
}
else
{
// Skip any more spaces
while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ')
{
++p;
}
// Check to see if there's the right string next...
if(::strncmp(requestLinePtr + p, "HTTP/", 5) == 0)
{
// Find the version numbers
int major, minor;
if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2)
{
THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest,
"Unable to parse HTTP version number: " <<
requestLinePtr);
}
// Store version
mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor;
}
else
{
// Not good -- wrong string found
THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest,
"Unable to parse HTTP request line: " <<
requestLinePtr);
}
}
BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" <<
mRequestURI << ", version=" << mHTTPVersion);
// If HTTP 1.1 or greater, assume keep-alive
if(mHTTPVersion >= HTTPVersion_1_1)
{
mClientKeepAliveRequested = true;
}
// Decode query string?
if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty())
{
HTTPQueryDecoder decoder(mQuery);
decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size());
decoder.Finish();
}
// Now parse the headers
ParseHeaders(rGetLine, Timeout);
std::string expected;
if(GetHeader("Expect", &expected))
{
if(expected == "100-continue")
{
mExpectContinue = true;
}
}
// Parse form data?
if(mMethod == Method_POST && mContentLength >= 0)
{
// Too long? Don't allow people to be nasty by sending lots of data
if(mContentLength > MAX_CONTENT_SIZE)
{
THROW_EXCEPTION(HTTPException, POSTContentTooLong);
}
// Some data in the request to follow, parsing it bit by bit
HTTPQueryDecoder decoder(mQuery);
// Don't forget any data left in the GetLine object
int fromBuffer = rGetLine.GetSizeOfBufferedData();
if(fromBuffer > mContentLength) fromBuffer = mContentLength;
if(fromBuffer > 0)
{
BOX_TRACE("Decoding " << fromBuffer << " bytes of "
"data from getline buffer");
decoder.DecodeChunk((const char *)rGetLine.GetBufferedData(), fromBuffer);
// And tell the getline object to ignore the data we just used
rGetLine.IgnoreBufferedData(fromBuffer);
}
// Then read any more data, as required
int bytesToGo = mContentLength - fromBuffer;
while(bytesToGo > 0)
{
char buf[4096];
int toRead = sizeof(buf);
if(toRead > bytesToGo) toRead = bytesToGo;
IOStream &rstream(rGetLine.GetUnderlyingStream());
int r = rstream.Read(buf, toRead, Timeout);
if(r == 0)
{
// Timeout, just error
THROW_EXCEPTION_MESSAGE(HTTPException, RequestReadFailed,
"Failed to read complete request with the timeout");
}
decoder.DecodeChunk(buf, r);
bytesToGo -= r;
}
// Finish off
decoder.Finish();
}
else if (mContentLength > 0)
{
IOStream::pos_type bytesToCopy = rGetLine.GetSizeOfBufferedData();
if (bytesToCopy > mContentLength)
{
bytesToCopy = mContentLength;
}
Write(rGetLine.GetBufferedData(), bytesToCopy);
SetForReading();
mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream());
}
return true;
}
// --------------------------------------------------------------------------
//
// Function
// Name: HousekeepStoreAccount::DoHousekeeping()
// Purpose: Perform the housekeeping
// Created: 11/12/03
//
// --------------------------------------------------------------------------
bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
{
BOX_TRACE("Starting housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID));
// Attempt to lock the account
std::string writeLockFilename;
StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
writeLockFilename);
NamedLock writeLock;
if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
0600 /* restrictive file permissions */))
{
if(KeepTryingForever)
{
BOX_INFO("Failed to lock account for housekeeping, "
"still trying...");
while(!writeLock.TryAndGetLock(writeLockFilename,
0600 /* restrictive file permissions */))
{
sleep(1);
}
}
else
{
// Couldn't lock the account -- just stop now
return false;
}
}
// Load the store info to find necessary info for the housekeeping
std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
mStoreRoot, mStoreDiscSet, false /* Read/Write */));
std::auto_ptr<BackupStoreInfo> pOldInfo(
BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet,
true /* Read Only */));
// If the account has a name, change the logging tag to include it
if(!(info->GetAccountName().empty()))
{
std::ostringstream tag;
tag << "hk=" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" <<
info->GetAccountName();
mTagWithClientID.Change(tag.str());
}
// Calculate how much should be deleted
mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
if(mDeletionSizeTarget < 0)
{
mDeletionSizeTarget = 0;
}
BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
mapNewRefs = BackupStoreRefCountDatabase::Create(account);
// Scan the directory for potential things to delete
// This will also remove eligible items marked with RemoveASAP
bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID,
*info);
if(!continueHousekeeping)
{
// The scan was incomplete, so the new block counts are
// incorrect, we can't rely on them. It's better to discard
// the new info and adjust the old one instead.
info = pOldInfo;
// We're about to reset counters and exit, so report what
// happened now.
BOX_INFO("Housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID) << " removed " <<
(0 - mBlocksUsedDelta) << " blocks (" <<
mFilesDeleted << " files, " <<
mEmptyDirectoriesDeleted << " dirs) and the directory "
"scan was interrupted");
}
// If housekeeping made any changes, such as deleting RemoveASAP files,
// the differences in block counts will be recorded in the deltas.
info->ChangeBlocksUsed(mBlocksUsedDelta);
info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
// Reset the delta counts for files, as they will include
// RemoveASAP flagged files deleted during the initial scan.
// keep removeASAPBlocksUsedDelta for reporting
int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
mBlocksUsedDelta = 0;
mBlocksInOldFilesDelta = 0;
mBlocksInDeletedFilesDelta = 0;
// If scan directory stopped for some reason, probably parent
// instructed to terminate, stop now.
//
// We can only update the refcount database if we successfully
// finished our scan of all directories, otherwise we don't actually
// know which of the new counts are valid and which aren't
// (we might not have seen second references to some objects, etc.).
if(!continueHousekeeping)
{
mapNewRefs->Discard();
info->Save();
return false;
}
// Report any UNexpected changes, and consider them to be errors.
// Do this before applying the expected changes below.
mErrorCount += info->ReportChangesTo(*pOldInfo);
info->Save();
// Try to load the old reference count database and check whether
// any counts have changed. We want to compare the mapNewRefs to
// apOldRefs before we delete any files, because that will also change
// the reference count in a way that's not an error.
try
{
std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
BackupStoreRefCountDatabase::Load(account, false);
mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs);
}
catch(BoxException &e)
{
BOX_WARNING("Reference count database was missing or "
"corrupted during housekeeping, cannot check it for "
"errors.");
mErrorCount++;
}
// Go and delete items from the accounts
bool deleteInterrupted = DeleteFiles(*info);
// If that wasn't interrupted, remove any empty directories which
// are also marked as deleted in their containing directory
if(!deleteInterrupted)
{
deleteInterrupted = DeleteEmptyDirectories(*info);
}
// Log deletion if anything was deleted
if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
{
BOX_INFO("Housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID) << " "
"removed " <<
(0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) <<
" blocks (" << mFilesDeleted << " files, " <<
mEmptyDirectoriesDeleted << " dirs)" <<
(deleteInterrupted?" and was interrupted":""));
}
// Make sure the delta's won't cause problems if the counts are
// really wrong, and it wasn't fixed because the store was
// updated during the scan.
if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
{
mBlocksUsedDelta = (0 - info->GetBlocksUsed());
}
if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles()))
{
mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles());
}
if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles()))
{
mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles());
}
if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories()))
{
mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
}
// Update the usage counts in the store
info->ChangeBlocksUsed(mBlocksUsedDelta);
info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
// Save the store info back
info->Save();
// force file to be saved and closed before releasing the lock below
mapNewRefs->Commit();
mapNewRefs.reset();
// Explicity release the lock (would happen automatically on
// going out of scope, included for code clarity)
writeLock.ReleaseLock();
BOX_TRACE("Finished housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID));
return true;
}
BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile(
int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory,
const std::string &rDirectoryFilename,
BackupStoreInfo& rBackupStoreInfo)
{
// Find the entry inside the directory
bool wasDeleted = false;
bool wasOldVersion = false;
int64_t deletedFileSizeInBlocks = 0;
// A pointer to an object which requires committing if the directory save goes OK
std::auto_ptr<RaidFileWrite> padjustedEntry;
// BLOCK
{
BackupStoreRefCountDatabase::refcount_t refs =
mapNewRefs->GetRefCount(ObjectID);
BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
if(pentry == 0)
{
BOX_ERROR("Housekeeping on account " <<
BOX_FORMAT_ACCOUNT(mAccountID) << " "
"found error: object " <<
BOX_FORMAT_OBJECTID(ObjectID) << " "
"not found in dir " <<
BOX_FORMAT_OBJECTID(InDirectory) << ", "
"indicates logic error/corruption? Run "
"bbstoreaccounts check <accid> fix");
mErrorCount++;
return refs;
}
// Record the flags it's got set
wasDeleted = pentry->IsDeleted();
wasOldVersion = pentry->IsOld();
// Check this should be deleted
if(!wasDeleted && !wasOldVersion)
{
// Things changed since we were last around
return refs;
}
// Record size
deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
if(refs > 1)
{
// Not safe to merge patches if someone else has a
// reference to this object, so just remove the
// directory entry and return.
rDirectory.DeleteEntry(ObjectID);
if(wasDeleted)
{
rBackupStoreInfo.AdjustNumDeletedFiles(-1);
}
if(wasOldVersion)
{
rBackupStoreInfo.AdjustNumOldFiles(-1);
}
mapNewRefs->RemoveReference(ObjectID);
return refs - 1;
}
// If the entry is involved in a chain of patches, it needs to be handled
// a bit more carefully.
if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
{
// This entry is a patch from a newer entry. Just need to update the info on that entry.
BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Change the info in the newer entry so that this no longer points to this entry
pnewer->SetDependsOlder(0);
}
else if(pentry->GetDependsOlder() != 0)
{
BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
if(pentry->GetDependsNewer() == 0)
{
// There exists an older version which depends on this one. Need to combine the two over that one.
// Adjust the other entry in the directory
if(polder == 0 || polder->GetDependsNewer() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Change the info in the older entry so that this no longer points to this entry
polder->SetDependsNewer(0);
}
else
{
// This entry is in the middle of a chain, and two patches need combining.
// First, adjust the directory entries
BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
|| polder == 0 || polder->GetDependsNewer() != ObjectID)
{
THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
}
// Remove the middle entry from the linked list by simply using the values from this entry
pnewer->SetDependsOlder(pentry->GetDependsOlder());
polder->SetDependsNewer(pentry->GetDependsNewer());
}
// COMMON CODE to both cases
// Generate the filename of the older version
std::string objFilenameOlder;
MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
// Open it twice (it's the diff)
std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
// Open this file
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
// And open a write file to overwrite the other directory entry
padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
objFilenameOlder, mapNewRefs->GetRefCount(ObjectID)));
padjustedEntry->Open(true /* allow overwriting */);
if(pentry->GetDependsNewer() == 0)
{
// There exists an older version which depends on this one. Need to combine the two over that one.
BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
}
else
{
// This entry is in the middle of a chain, and two patches need combining.
BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
}
// The file will be committed later when the directory is safely commited.
// Work out the adjusted size
int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
mBlocksUsedDelta += sizeDelta;
if(polder->IsDeleted())
{
mBlocksInDeletedFilesDelta += sizeDelta;
}
if(polder->IsOld())
{
mBlocksInOldFilesDelta += sizeDelta;
}
polder->SetSizeInBlocks(newSize);
}
// pentry no longer valid
}
// Delete it from the directory
rDirectory.DeleteEntry(ObjectID);
// Save directory back to disc
// BLOCK
{
RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
mapNewRefs->GetRefCount(InDirectory));
writeDir.Open(true /* allow overwriting */);
rDirectory.WriteToStream(writeDir);
// Get the disc usage (must do this before commiting it)
int64_t new_size = writeDir.GetDiscUsageInBlocks();
// Commit directory
writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
// Adjust block counts if the directory itself changed in size
int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks();
int64_t adjust = new_size - original_size;
mBlocksUsedDelta += adjust;
mBlocksInDirectoriesDelta += adjust;
UpdateDirectorySize(rDirectory, new_size);
}
// Commit any new adjusted entry
if(padjustedEntry.get() != 0)
{
padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
padjustedEntry.reset(); // delete it now
}
// Drop reference count by one. Must now be zero, to delete the file.
bool remaining_refs = mapNewRefs->RemoveReference(ObjectID);
ASSERT(!remaining_refs);
// Delete from disc
BOX_TRACE("Removing unreferenced object " <<
BOX_FORMAT_OBJECTID(ObjectID));
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID));
del.Delete();
// Adjust counts for the file
++mFilesDeleted;
mBlocksUsedDelta -= deletedFileSizeInBlocks;
if(wasDeleted)
{
mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
rBackupStoreInfo.AdjustNumDeletedFiles(-1);
}
if(wasOldVersion)
{
mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
rBackupStoreInfo.AdjustNumOldFiles(-1);
}
// Delete the directory?
// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
if(rDirectory.GetNumberOfEntries() == 0)
{
// Candidate for deletion
mEmptyDirectories.push_back(InDirectory);
}
return 0;
}
// --------------------------------------------------------------------------
//
// Function
// Name: HousekeepStoreAccount::DeleteFiles()
// Purpose: Delete the files targeted for deletion, returning
// true if the operation was interrupted
// Created: 15/12/03
//
// --------------------------------------------------------------------------
bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo)
{
// Only delete files if the deletion target is greater than zero
// (otherwise we delete one file each time round, which gradually deletes the old versions)
if(mDeletionSizeTarget <= 0)
{
// Not interrupted
return false;
}
// Iterate through the set of potential deletions, until enough has been deleted.
// (there is likely to be more in the set than should be actually deleted).
for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
{
#ifndef WIN32
if((--mCountUntilNextInterprocessMsgCheck) <= 0)
{
mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
// Check for having to stop
if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
{
// Need to abort now
return true;
}
}
#endif
// Load up the directory it's in
// Get the filename
std::string dirFilename;
BackupStoreDirectory dir;
{
MakeObjectFilename(i->mInDirectory, dirFilename);
std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks());
}
// Delete the file
BackupStoreRefCountDatabase::refcount_t refs =
DeleteFile(i->mInDirectory, i->mObjectID, dir,
dirFilename, rBackupStoreInfo);
if(refs == 0)
{
BOX_INFO("Housekeeping removed " <<
(i->mIsFlagDeleted ? "deleted" : "old") <<
" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
}
else
{
BOX_TRACE("Housekeeping preserved " <<
(i->mIsFlagDeleted ? "deleted" : "old") <<
" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
" in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) <<
" with " << refs << " references");
}
// Stop if the deletion target has been matched or exceeded
// (checking here rather than at the beginning will tend to reduce the
// space to slightly less than the soft limit, which will allow the backup
// client to start uploading files again)
if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
{
break;
}
}
return false;
}
void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo)
{
// Load up the directory to potentially delete
std::string dirFilename;
BackupStoreDirectory dir;
int64_t dirSizeInBlocks = 0;
// BLOCK
{
MakeObjectFilename(dirId, dirFilename);
// Check it actually exists (just in case it gets
// added twice to the list)
if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
{
// doesn't exist, next!
return;
}
// load
std::auto_ptr<RaidFileRead> dirStream(
RaidFileRead::Open(mStoreDiscSet, dirFilename));
dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
}
// Make sure this directory is actually empty
if(dir.GetNumberOfEntries() != 0)
{
// Not actually empty, try next one
return;
}
// Candidate for deletion... open containing directory
std::string containingDirFilename;
BackupStoreDirectory containingDir;
int64_t containingDirSizeInBlocksOrig = 0;
{
MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
std::auto_ptr<RaidFileRead> containingDirStream(
RaidFileRead::Open(mStoreDiscSet,
containingDirFilename));
containingDirSizeInBlocksOrig =
containingDirStream->GetDiscUsageInBlocks();
containingDir.ReadFromStream(*containingDirStream,
IOStream::TimeOutInfinite);
containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig);
}
// Find the entry
BackupStoreDirectory::Entry *pdirentry =
containingDir.FindEntryByID(dir.GetObjectID());
// TODO FIXME invert test and reduce indentation
if((pdirentry != 0) && pdirentry->IsDeleted())
{
// Should be deleted
containingDir.DeleteEntry(dir.GetObjectID());
// Is the containing dir now a candidate for deletion?
if(containingDir.GetNumberOfEntries() == 0)
{
rToExamine.push_back(containingDir.GetObjectID());
}
// Write revised parent directory
RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
mapNewRefs->GetRefCount(containingDir.GetObjectID()));
writeDir.Open(true /* allow overwriting */);
containingDir.WriteToStream(writeDir);
// get the disc usage (must do this before commiting it)
int64_t dirSize = writeDir.GetDiscUsageInBlocks();
// Commit directory
writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
UpdateDirectorySize(containingDir, dirSize);
// adjust usage counts for this directory
if(dirSize > 0)
{
int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
mBlocksUsedDelta += adjust;
mBlocksInDirectoriesDelta += adjust;
}
if (mapNewRefs->RemoveReference(dir.GetObjectID()))
{
// Still referenced
BOX_TRACE("Housekeeping spared empty deleted dir " <<
BOX_FORMAT_OBJECTID(dirId) << " due to " <<
mapNewRefs->GetRefCount(dir.GetObjectID()) <<
"remaining references");
return;
}
// Delete the directory itself
BOX_INFO("Housekeeping removing empty deleted dir " <<
BOX_FORMAT_OBJECTID(dirId));
RaidFileWrite del(mStoreDiscSet, dirFilename,
mapNewRefs->GetRefCount(dir.GetObjectID()));
del.Delete();
// And adjust usage counts for the directory that's
// just been deleted
mBlocksUsedDelta -= dirSizeInBlocks;
mBlocksInDirectoriesDelta -= dirSizeInBlocks;
// Update count
++mEmptyDirectoriesDeleted;
rBackupStoreInfo.AdjustNumDirectories(-1);
}
}
