bool FBuildPatchUtils::UncompressChunkFile( TArray< uint8 >& ChunkFileArray )
{
FMemoryReader ChunkArrayReader( ChunkFileArray );
// Read the header
FChunkHeader Header;
ChunkArrayReader << Header;
// Check header
const bool bValidHeader = Header.IsValidMagic();
const bool bSupportedFormat = Header.HashType == FChunkHeader::HASH_ROLLING && !( Header.StoredAs & FChunkHeader::STORED_ENCRYPTED );
if( bValidHeader && bSupportedFormat )
{
bool bSuccess = true;
// Uncompress if we need to
if( Header.StoredAs == FChunkHeader::STORED_COMPRESSED )
{
// Load the compressed chunk data
TArray< uint8 > CompressedData;
TArray< uint8 > UncompressedData;
CompressedData.Empty( Header.DataSize );
CompressedData.AddUninitialized( Header.DataSize );
UncompressedData.Empty( FBuildPatchData::ChunkDataSize );
UncompressedData.AddUninitialized( FBuildPatchData::ChunkDataSize );
ChunkArrayReader.Serialize( CompressedData.GetData(), Header.DataSize );
ChunkArrayReader.Close();
// Uncompress
bSuccess = FCompression::UncompressMemory(
static_cast< ECompressionFlags >( COMPRESS_ZLIB | COMPRESS_BiasMemory ),
UncompressedData.GetData(),
UncompressedData.Num(),
CompressedData.GetData(),
CompressedData.Num() );
// If successful, write back over the original array
if( bSuccess )
{
ChunkFileArray.Empty();
FMemoryWriter ChunkArrayWriter( ChunkFileArray );
Header.StoredAs = FChunkHeader::STORED_RAW;
Header.DataSize = FBuildPatchData::ChunkDataSize;
ChunkArrayWriter << Header;
ChunkArrayWriter.Serialize( UncompressedData.GetData(), UncompressedData.Num() );
ChunkArrayWriter.Close();
}
}
return bSuccess;
}
return false;
}
bool FBuildPatchUtils::VerifyChunkFile( FArchive& ChunkFileData, bool bQuickCheck )
{
const int64 FileSize = ChunkFileData.TotalSize();
bool bSuccess = ChunkFileData.IsLoading();
if ( !bSuccess )
{
GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile expected readonly archive" ) );
}
else
{
// Read the header
FChunkHeader Header;
ChunkFileData << Header;
// Check header magic
if ( !Header.IsValidMagic() )
{
bSuccess = false;
GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile corrupt header" ) );
}
// Check Header and data size
if ( bSuccess && ( Header.HeaderSize + Header.DataSize ) != FileSize )
{
bSuccess = false;
GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile header info does not match file size" ) );
}
if( bSuccess && !bQuickCheck )
{
// Hashes for checking data
FSHA1 SHAHasher;
FSHAHashData SHAHash;
uint64 CycPoly64Hash = 0;
// Load the data to check
uint8* FileReadBuffer = new uint8[ FileBufferSize ];
int64 DataOffset = 0;
switch ( Header.StoredAs )
{
case FChunkHeader::STORED_RAW:
while( !ChunkFileData.AtEnd() )
{
const int64 SizeLeft = FileSize - ChunkFileData.Tell();
const uint32 ReadLen = FMath::Min< int64 >( FileBufferSize, SizeLeft );
ChunkFileData.Serialize( FileReadBuffer, ReadLen );
switch ( Header.HashType )
{
case FChunkHeader::HASH_ROLLING:
CycPoly64Hash = FCycPoly64Hash::GetHashForDataSet(FileReadBuffer, ReadLen, CycPoly64Hash);
break;
case FChunkHeader::HASH_SHA1:
SHAHasher.Update( FileReadBuffer, ReadLen );
break;
default:
check( false ); // @TODO LSwift: Implement other storage methods!
bSuccess = false;
break;
}
DataOffset += ReadLen;
}
if( bSuccess )
{
switch ( Header.HashType )
{
case FChunkHeader::HASH_ROLLING:
bSuccess = Header.RollingHash == CycPoly64Hash;
break;
case FChunkHeader::HASH_SHA1:
SHAHasher.Final();
SHAHasher.GetHash( SHAHash.Hash );
bSuccess = SHAHash == Header.SHAHash;
break;
}
if (!bSuccess)
{
GLog->Logf(TEXT("BuildPatchServices: ERROR: VerifyChunkFile file hashcheck failed"));
}
}
break;
default:
GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile failed, unknown storage type" ) );
bSuccess = false;
break;
}
delete[] FileReadBuffer;
}
}
return bSuccess;
}
bool FBuildPatchChunkCache::ReadChunkFromDriveCache( const FGuid& ChunkGuid )
{
bool bSuccess = true;
// Get the chunk filename
const FString Filename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, ChunkGuid );
// Read the chunk
FArchive* FileReader = IFileManager::Get().CreateFileReader( *Filename );
bSuccess = FileReader != NULL;
if( bSuccess )
{
// Get file size
const int64 FileSize = FileReader->TotalSize();
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Read the header
*FileReader << *ChunkHeader;
// Check header magic
bSuccess = ChunkHeader->IsValidMagic();
if ( bSuccess )
{
// Check the right data size
bSuccess = ChunkHeader->DataSize == FBuildPatchData::ChunkDataSize;
if( bSuccess )
{
// Check Header and data size
bSuccess = ( ChunkHeader->HeaderSize + ChunkHeader->DataSize ) == FileSize;
if( bSuccess )
{
// Read the data
FileReader->Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Verify the data hash
FSHAHashData ShaHashCheck;
switch (ChunkHeader->HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = ChunkHeader->RollingHash == FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(ChunkData);
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(ChunkData, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkHeader->SHAHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Hash Check Failed" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache chunk failed hash check %s" ), *ChunkGuid.ToString() );
}
else
{
// Count loads
NumDriveCacheChunkLoads.Increment();
GLog->Logf( TEXT( "FBuildPatchChunkCache: ReadChunkFromDriveCache loaded chunk %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Incorrect File Size" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache header info does not match file size %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Datasize/Hashtype Mismatch" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache mismatch datasize/hashtype combination %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Corrupt Header" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache corrupt header %s" ), *ChunkGuid.ToString() );
}
// Release data
NewChunkFile->ReleaseDataLock();
// Add the newly filled data to the cache if successful
if( bSuccess )
{
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// If there was a problem, remove from cache and reservation
else
{
ChunkCache.Remove( ChunkGuid );
}
// Close the file
FileReader->Close();
delete FileReader;
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, FPlatformMisc::GetLastError(), TEXT( "DriveCache" ), TEXT( "Open File Fail" ) );
GLog->Logf( TEXT( "BuildPatchServices: ERROR: GetChunkData could not open chunk file %s" ), *ChunkGuid.ToString() );
}
return bSuccess;
}
