// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
// Purpose: Move the file pointer in this stream to just before the block index.
// Assumes that the stream is at the beginning, seekable, and
// reading from the stream is OK.
// Created: 12/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
{
// Size of file
int64_t fileSize = rStream.BytesLeftToRead();
// Get header
file_StreamFormat hdr;
// Read the header
if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
{
// Couldn't read header
THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
}
// Check magic number
if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
#endif
)
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// Work out where the index is
int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
// Sanity check
if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// Seek to that position
rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
// Done. Stream now in right position (as long as the file is formatted correctly)
}
// --------------------------------------------------------------------------
//
// Function
// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &)
// Purpose: Read in an index, and decrypt, and store in the in memory block format.
// rCanDiffFromThis is set to false if the version of the from file is too old.
// Created: 12/1/04
//
// --------------------------------------------------------------------------
static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis)
{
// Reset
rNumBlocksOut = 0;
rCanDiffFromThis = false;
// Read header
file_BlockIndexHeader hdr;
if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
{
// Couldn't read header
THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
}
#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
// Check against backwards comptaibility stuff
if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
{
// Won't diff against old version
// Absorb rest of stream
char buffer[2048];
while(rBlockIndex.StreamDataLeft())
{
rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */);
}
// Tell caller
rCanDiffFromThis = false;
return;
}
#endif
// Check magic
if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1))
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// Check that we're not trying to diff against a file which references blocks from another file
if(((int64_t)box_ntoh64(hdr.mOtherFileID)) != 0)
{
THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
}
// Mark as an acceptable diff.
rCanDiffFromThis = true;
// Get basic information
int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
//TODO: Verify that these sizes look reasonable
// Allocate space for the index
BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks);
if(pindex == 0)
{
throw std::bad_alloc();
}
try
{
for(int64_t b = 0; b < numBlocks; ++b)
{
// Read an entry from the stream
file_BlockIndexEntry entry;
if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
{
// Couldn't read entry
THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
}
// Calculate IV for this entry
uint64_t iv = entryIVBase;
iv += b;
// Network byte order
iv = box_hton64(iv);
sBlowfishDecryptBlockEntry.SetIV(&iv);
// Decrypt the encrypted section
file_BlockIndexEntryEnc entryEnc;
int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
entry.mEnEnc, sizeof(entry.mEnEnc));
if(sectionSize != sizeof(entryEnc))
{
THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
}
// Check that we're not trying to diff against a file which references blocks from another file
if(((int64_t)box_ntoh64(entry.mEncodedSize)) <= 0)
{
THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
}
// Store all the required information
pindex[b].mpNextInHashList = 0; // hash list not set up yet
pindex[b].mSize = ntohl(entryEnc.mSize);
pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum);
::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum));
}
// Store index pointer for called
ASSERT(ppIndex != 0);
*ppIndex = pindex;
// Store number of blocks for caller
rNumBlocksOut = numBlocks;
}
catch(...)
{
// clean up and send the exception along its way
::free(pindex);
throw;
}
}
std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(IOStream& rStream,
const std::string FileName, bool ReadOnly)
{
// Read in format and version
int32_t magic;
if(!rStream.ReadFullBuffer(&magic, sizeof(magic), 0))
{
THROW_FILE_ERROR("Failed to read store info file: "
"short read of magic number", FileName,
BackupStoreException, CouldNotLoadStoreInfo);
}
bool v1 = false, v2 = false;
if(ntohl(magic) == INFO_MAGIC_VALUE_1)
{
v1 = true;
}
else if(ntohl(magic) == INFO_MAGIC_VALUE_2)
{
v2 = true;
}
else
{
THROW_FILE_ERROR("Failed to read store info file: "
"unknown magic " << BOX_FORMAT_HEX32(ntohl(magic)),
FileName, BackupStoreException, BadStoreInfoOnLoad);
}
// Make new object
std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
// Put in basic location info
info->mFilename = FileName;
info->mReadOnly = ReadOnly;
int64_t numDelObj = 0;
if (v1)
{
// Read in a header
info_StreamFormat_1 hdr;
rStream.Seek(0, IOStream::SeekType_Absolute);
if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr),
0 /* not interested in bytes read if this fails */))
{
THROW_FILE_ERROR("Failed to read store info header",
FileName, BackupStoreException, CouldNotLoadStoreInfo);
}
// Insert info from file
info->mAccountID = ntohl(hdr.mAccountID);
info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker);
info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed);
info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed);
info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles);
info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles);
info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories);
info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit);
info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit);
// Load up array of deleted objects
numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories);
}
else if(v2)
{
Archive archive(rStream, IOStream::TimeOutInfinite);
// Check it
archive.Read(info->mAccountID);
archive.Read(info->mAccountName);
archive.Read(info->mClientStoreMarker);
archive.Read(info->mLastObjectIDUsed);
archive.Read(info->mBlocksUsed);
archive.Read(info->mBlocksInCurrentFiles);
archive.Read(info->mBlocksInOldFiles);
archive.Read(info->mBlocksInDeletedFiles);
archive.Read(info->mBlocksInDirectories);
archive.Read(info->mBlocksSoftLimit);
archive.Read(info->mBlocksHardLimit);
archive.Read(info->mNumCurrentFiles);
archive.Read(info->mNumOldFiles);
archive.Read(info->mNumDeletedFiles);
archive.Read(info->mNumDirectories);
archive.Read(numDelObj);
}
// Then load the list of deleted directories
if(numDelObj > 0)
{
int64_t objs[NUM_DELETED_DIRS_BLOCK];
int64_t toload = numDelObj;
while(toload > 0)
{
// How many in this one?
int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
if(!rStream.ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
{
THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
}
// Add them
for(int t = 0; t < b; ++t)
{
info->mDeletedDirectories.push_back(box_ntoh64(objs[t]));
}
// Number loaded
toload -= b;
}
}
