void AssbinImporter::InternReadFile( const std::string& pFile, aiScene* pScene, IOSystem* pIOHandler )
{
IOStream * stream = pIOHandler->Open(pFile,"rb");
if (!stream)
return;
stream->Seek( 44, aiOrigin_CUR ); // signature
/*unsigned int versionMajor =*/ Read<unsigned int>(stream);
/*unsigned int versionMinor =*/ Read<unsigned int>(stream);
/*unsigned int versionRevision =*/ Read<unsigned int>(stream);
/*unsigned int compileFlags =*/ Read<unsigned int>(stream);
shortened = Read<uint16_t>(stream) > 0;
compressed = Read<uint16_t>(stream) > 0;
if (shortened)
throw DeadlyImportError( "Shortened binaries are not supported!" );
stream->Seek( 256, aiOrigin_CUR ); // original filename
stream->Seek( 128, aiOrigin_CUR ); // options
stream->Seek( 64, aiOrigin_CUR ); // padding
if (compressed)
{
uLongf uncompressedSize = Read<uint32_t>(stream);
uLongf compressedSize = stream->FileSize() - stream->Tell();
unsigned char * compressedData = new unsigned char[ compressedSize ];
stream->Read( compressedData, 1, compressedSize );
unsigned char * uncompressedData = new unsigned char[ uncompressedSize ];
uncompress( uncompressedData, &uncompressedSize, compressedData, compressedSize );
MemoryIOStream io( uncompressedData, uncompressedSize );
ReadBinaryScene(&io,pScene);
delete[] uncompressedData;
delete[] compressedData;
}
else
{
ReadBinaryScene(stream,pScene);
}
pIOHandler->Close(stream);
}
// --------------------------------------------------------------------------
//
// Function
// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
// Purpose: Move the file pointer in this stream to just before the block index.
// Assumes that the stream is at the beginning, seekable, and
// reading from the stream is OK.
// Created: 12/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
{
// Size of file
int64_t fileSize = rStream.BytesLeftToRead();
// Get header
file_StreamFormat hdr;
// Read the header
if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
{
// Couldn't read header
THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
}
// Check magic number
if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
#endif
)
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// Work out where the index is
int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
// Sanity check
if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// Seek to that position
rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
// Done. Stream now in right position (as long as the file is formatted correctly)
}
// --------------------------------------------------------------------------
//
// Function
// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
// Purpose: Find the matching blocks within the file.
// Created: 12/1/04
//
// --------------------------------------------------------------------------
static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
BlocksAvailableEntry *pIndex, int64_t NumBlocks,
int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer)
{
Timer maximumDiffingTime(0, "MaximumDiffingTime");
if(pDiffTimer && pDiffTimer->IsManaged())
{
maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime() *
MILLI_SEC_IN_SEC, "MaximumDiffingTime");
}
std::map<int64_t, int32_t> goodnessOfFit;
// Allocate the hash lookup table
BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
// Choose a size for the buffer, just a little bit more than the maximum block size
int32_t bufSize = Sizes[0];
for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
{
if(Sizes[z] > bufSize) bufSize = Sizes[z];
}
bufSize += 4;
ASSERT(bufSize > Sizes[0]);
ASSERT(bufSize > 0);
if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
{
THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
}
// TODO: Because we read in the file a scanned block size at a time,
// it is likely to be inefficient. Probably will be much better to
// calculate checksums for all block sizes in a single pass.
// Allocate the buffers.
uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
try
{
// Check buffer allocation
if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
{
// If a buffer got allocated, it will be cleaned up in the catch block
throw std::bad_alloc();
}
// Flag to abort the run, if too many blocks are found -- avoid using
// huge amounts of processor time when files contain many similar blocks.
bool abortSearch = false;
// Search for each block size in turn
// NOTE: Do the smallest size first, so that the scheme for adding
// entries in the found list works as expected and replaces smaller blocks
// with larger blocks when it finds matches at the same offset in the file.
for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
{
ASSERT(Sizes[s] <= bufSize);
BOX_TRACE("Diff pass " << s << ", for block size " <<
Sizes[s]);
// Check we haven't finished
if(Sizes[s] == 0)
{
// empty entry, try next size
continue;
}
// Set up the hash table entries
SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
// Shift file position to beginning
rFile.Seek(0, IOStream::SeekType_Absolute);
// Read first block
if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
{
// Size of file too short to match -- do next size
continue;
}
// Setup block pointers
uint8_t *beginnings = pbuffer0;
uint8_t *endings = pbuffer1;
int offset = 0;
// Calculate the first checksum, ready for rolling
RollingChecksum rolling(beginnings, Sizes[s]);
// Then roll, until the file is exhausted
int64_t fileBlockNumber = 0;
int64_t fileOffset = 0;
int rollOverInitialBytes = 0;
while(true)
{
if(maximumDiffingTime.HasExpired())
{
ASSERT(pDiffTimer != NULL);
BOX_INFO("MaximumDiffingTime reached - "
"suspending file diff");
abortSearch = true;
break;
}
if(pDiffTimer)
{
pDiffTimer->DoKeepAlive();
}
// Load in another block of data, and record how big it is
int bytesInEndings = rFile.Read(endings, Sizes[s]);
int tmp;
// Skip any bytes from a previous matched block
if(rollOverInitialBytes > 0 && offset < bytesInEndings)
{
int spaceLeft = bytesInEndings - offset;
int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes;
rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
offset += thisRoll;
fileOffset += thisRoll;
rollOverInitialBytes -= thisRoll;
if(rollOverInitialBytes)
{
goto refresh;
}
}
if(goodnessOfFit.count(fileOffset))
{
tmp = goodnessOfFit[fileOffset];
}
else
{
tmp = 0;
}
if(tmp >= Sizes[s])
{
// Skip over bigger ready-matched blocks completely
rollOverInitialBytes = tmp;
int spaceLeft = bytesInEndings - offset;
int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes;
rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
offset += thisRoll;
fileOffset += thisRoll;
rollOverInitialBytes -= thisRoll;
if(rollOverInitialBytes)
{
goto refresh;
}
}
while(offset < bytesInEndings)
{
// Is current checksum in hash list?
uint16_t hash = rolling.GetComponentForHashing();
if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s]))
{
if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
{
BOX_TRACE("Found block match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset);
goodnessOfFit[fileOffset] = Sizes[s];
// Block matched, roll the checksum forward to the next block without doing
// any more comparisons, because these are pointless (as any more matches will be ignored when
// the recipe is generated) and just take up valuable processor time. Edge cases are
// especially nasty, using huge amounts of time and memory.
int skip = Sizes[s];
if(offset < bytesInEndings && skip > 0)
{
int spaceLeft = bytesInEndings - offset;
int thisRoll = (skip > spaceLeft) ? spaceLeft : skip;
rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
offset += thisRoll;
fileOffset += thisRoll;
skip -= thisRoll;
}
// Not all the bytes necessary will have been skipped, so get them
// skipped after the next block is loaded.
rollOverInitialBytes = skip;
// End this loop, so the final byte isn't used again
break;
}
else
{
// Too many to log
// BOX_TRACE("False alarm match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset);
}
int64_t NumBlocksFound = static_cast<int64_t>(
rFoundBlocks.size());
int64_t MaxBlocksFound = NumBlocks *
BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE;
if(NumBlocksFound > MaxBlocksFound)
{
abortSearch = true;
break;
}
}
// Roll checksum forward
rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
// Increment offsets
++offset;
++fileOffset;
}
if(abortSearch) break;
refresh:
// Finished?
if(bytesInEndings != Sizes[s])
{
// No more data in file -- check the final block
// (Do a copy and paste of 5 lines of code instead of introducing a comparison for
// each byte of the file)
uint16_t hash = rolling.GetComponentForHashing();
if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s]))
{
if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
{
goodnessOfFit[fileOffset] = Sizes[s];
}
}
// finish
break;
}
// Switch buffers, reset offset
beginnings = endings;
endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer
offset = 0;
// And count the blocks which have been done
++fileBlockNumber;
}
if(abortSearch) break;
}
// Free buffers and hash table
::free(pbuffer1);
pbuffer1 = 0;
::free(pbuffer0);
pbuffer0 = 0;
::free(phashTable);
phashTable = 0;
}
catch(...)
{
// Cleanup and throw
if(pbuffer1 != 0) ::free(pbuffer1);
if(pbuffer0 != 0) ::free(pbuffer0);
if(phashTable != 0) ::free(phashTable);
throw;
}
#ifndef BOX_RELEASE_BUILD
if(BackupStoreFile::TraceDetailsOfDiffProcess)
{
// Trace out the found blocks in debug mode
BOX_TRACE("Diff: list of found blocks");
BOX_TRACE("======== ======== ======== ========");
BOX_TRACE(" Offset BlkIdx Size Movement");
for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i)
{
int64_t orgLoc = 0;
for(int64_t b = 0; b < i->second; ++b)
{
orgLoc += pIndex[b].mSize;
}
BOX_TRACE(std::setw(8) << i->first << " " <<
std::setw(8) << i->second << " " <<
std::setw(8) << pIndex[i->second].mSize <<
" " <<
std::setw(8) << (i->first - orgLoc));
}
BOX_TRACE("======== ======== ======== ========");
}
#endif
}
std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(IOStream& rStream,
const std::string FileName, bool ReadOnly)
{
// Read in format and version
int32_t magic;
if(!rStream.ReadFullBuffer(&magic, sizeof(magic), 0))
{
THROW_FILE_ERROR("Failed to read store info file: "
"short read of magic number", FileName,
BackupStoreException, CouldNotLoadStoreInfo);
}
bool v1 = false, v2 = false;
if(ntohl(magic) == INFO_MAGIC_VALUE_1)
{
v1 = true;
}
else if(ntohl(magic) == INFO_MAGIC_VALUE_2)
{
v2 = true;
}
else
{
THROW_FILE_ERROR("Failed to read store info file: "
"unknown magic " << BOX_FORMAT_HEX32(ntohl(magic)),
FileName, BackupStoreException, BadStoreInfoOnLoad);
}
// Make new object
std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
// Put in basic location info
info->mFilename = FileName;
info->mReadOnly = ReadOnly;
int64_t numDelObj = 0;
if (v1)
{
// Read in a header
info_StreamFormat_1 hdr;
rStream.Seek(0, IOStream::SeekType_Absolute);
if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr),
0 /* not interested in bytes read if this fails */))
{
THROW_FILE_ERROR("Failed to read store info header",
FileName, BackupStoreException, CouldNotLoadStoreInfo);
}
// Insert info from file
info->mAccountID = ntohl(hdr.mAccountID);
info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker);
info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed);
info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed);
info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles);
info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles);
info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories);
info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit);
info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit);
// Load up array of deleted objects
numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories);
}
else if(v2)
{
Archive archive(rStream, IOStream::TimeOutInfinite);
// Check it
archive.Read(info->mAccountID);
archive.Read(info->mAccountName);
archive.Read(info->mClientStoreMarker);
archive.Read(info->mLastObjectIDUsed);
archive.Read(info->mBlocksUsed);
archive.Read(info->mBlocksInCurrentFiles);
archive.Read(info->mBlocksInOldFiles);
archive.Read(info->mBlocksInDeletedFiles);
archive.Read(info->mBlocksInDirectories);
archive.Read(info->mBlocksSoftLimit);
archive.Read(info->mBlocksHardLimit);
archive.Read(info->mNumCurrentFiles);
archive.Read(info->mNumOldFiles);
archive.Read(info->mNumDeletedFiles);
archive.Read(info->mNumDirectories);
archive.Read(numDelObj);
}
// Then load the list of deleted directories
if(numDelObj > 0)
{
int64_t objs[NUM_DELETED_DIRS_BLOCK];
int64_t toload = numDelObj;
while(toload > 0)
{
// How many in this one?
int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
if(!rStream.ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
{
THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
}
// Add them
for(int t = 0; t < b; ++t)
{
info->mDeletedDirectories.push_back(box_ntoh64(objs[t]));
}
// Number loaded
toload -= b;
}
}
