bool FBuildPatchUtils::UncompressChunkFile( TArray< uint8 >& ChunkFileArray ) { FMemoryReader ChunkArrayReader( ChunkFileArray ); // Read the header FChunkHeader Header; ChunkArrayReader << Header; // Check header const bool bValidHeader = Header.IsValidMagic(); const bool bSupportedFormat = Header.HashType == FChunkHeader::HASH_ROLLING && !( Header.StoredAs & FChunkHeader::STORED_ENCRYPTED ); if( bValidHeader && bSupportedFormat ) { bool bSuccess = true; // Uncompress if we need to if( Header.StoredAs == FChunkHeader::STORED_COMPRESSED ) { // Load the compressed chunk data TArray< uint8 > CompressedData; TArray< uint8 > UncompressedData; CompressedData.Empty( Header.DataSize ); CompressedData.AddUninitialized( Header.DataSize ); UncompressedData.Empty( FBuildPatchData::ChunkDataSize ); UncompressedData.AddUninitialized( FBuildPatchData::ChunkDataSize ); ChunkArrayReader.Serialize( CompressedData.GetData(), Header.DataSize ); ChunkArrayReader.Close(); // Uncompress bSuccess = FCompression::UncompressMemory( static_cast< ECompressionFlags >( COMPRESS_ZLIB | COMPRESS_BiasMemory ), UncompressedData.GetData(), UncompressedData.Num(), CompressedData.GetData(), CompressedData.Num() ); // If successful, write back over the original array if( bSuccess ) { ChunkFileArray.Empty(); FMemoryWriter ChunkArrayWriter( ChunkFileArray ); Header.StoredAs = FChunkHeader::STORED_RAW; Header.DataSize = FBuildPatchData::ChunkDataSize; ChunkArrayWriter << Header; ChunkArrayWriter.Serialize( UncompressedData.GetData(), UncompressedData.Num() ); ChunkArrayWriter.Close(); } } return bSuccess; } return false; }
bool FBuildPatchUtils::VerifyChunkFile( FArchive& ChunkFileData, bool bQuickCheck ) { const int64 FileSize = ChunkFileData.TotalSize(); bool bSuccess = ChunkFileData.IsLoading(); if ( !bSuccess ) { GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile expected readonly archive" ) ); } else { // Read the header FChunkHeader Header; ChunkFileData << Header; // Check header magic if ( !Header.IsValidMagic() ) { bSuccess = false; GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile corrupt header" ) ); } // Check Header and data size if ( bSuccess && ( Header.HeaderSize + Header.DataSize ) != FileSize ) { bSuccess = false; GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile header info does not match file size" ) ); } if( bSuccess && !bQuickCheck ) { // Hashes for checking data FSHA1 SHAHasher; FSHAHashData SHAHash; uint64 CycPoly64Hash = 0; // Load the data to check uint8* FileReadBuffer = new uint8[ FileBufferSize ]; int64 DataOffset = 0; switch ( Header.StoredAs ) { case FChunkHeader::STORED_RAW: while( !ChunkFileData.AtEnd() ) { const int64 SizeLeft = FileSize - ChunkFileData.Tell(); const uint32 ReadLen = FMath::Min< int64 >( FileBufferSize, SizeLeft ); ChunkFileData.Serialize( FileReadBuffer, ReadLen ); switch ( Header.HashType ) { case FChunkHeader::HASH_ROLLING: CycPoly64Hash = FCycPoly64Hash::GetHashForDataSet(FileReadBuffer, ReadLen, CycPoly64Hash); break; case FChunkHeader::HASH_SHA1: SHAHasher.Update( FileReadBuffer, ReadLen ); break; default: check( false ); // @TODO LSwift: Implement other storage methods! bSuccess = false; break; } DataOffset += ReadLen; } if( bSuccess ) { switch ( Header.HashType ) { case FChunkHeader::HASH_ROLLING: bSuccess = Header.RollingHash == CycPoly64Hash; break; case FChunkHeader::HASH_SHA1: SHAHasher.Final(); SHAHasher.GetHash( SHAHash.Hash ); bSuccess = SHAHash == Header.SHAHash; break; } if (!bSuccess) { GLog->Logf(TEXT("BuildPatchServices: ERROR: VerifyChunkFile file hashcheck failed")); } } break; default: GLog->Logf( TEXT( "BuildPatchServices: ERROR: VerifyChunkFile failed, unknown storage type" ) ); bSuccess = false; break; } delete[] FileReadBuffer; } } return bSuccess; }
bool FBuildPatchChunkCache::ReadChunkFromDriveCache( const FGuid& ChunkGuid ) { bool bSuccess = true; // Get the chunk filename const FString Filename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, ChunkGuid ); // Read the chunk FArchive* FileReader = IFileManager::Get().CreateFileReader( *Filename ); bSuccess = FileReader != NULL; if( bSuccess ) { // Get file size const int64 FileSize = FileReader->TotalSize(); // Create the ChunkFile data structure FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true ); // Lock data FChunkHeader* ChunkHeader; uint8* ChunkData; NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader ); // Read the header *FileReader << *ChunkHeader; // Check header magic bSuccess = ChunkHeader->IsValidMagic(); if ( bSuccess ) { // Check the right data size bSuccess = ChunkHeader->DataSize == FBuildPatchData::ChunkDataSize; if( bSuccess ) { // Check Header and data size bSuccess = ( ChunkHeader->HeaderSize + ChunkHeader->DataSize ) == FileSize; if( bSuccess ) { // Read the data FileReader->Serialize( ChunkData, FBuildPatchData::ChunkDataSize ); // Verify the data hash FSHAHashData ShaHashCheck; switch (ChunkHeader->HashType) { case FChunkHeader::HASH_ROLLING: bSuccess = ChunkHeader->RollingHash == FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(ChunkData); break; case FChunkHeader::HASH_SHA1: FSHA1::HashBuffer(ChunkData, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash); bSuccess = ShaHashCheck == ChunkHeader->SHAHash; break; default: bSuccess = false; } if( !bSuccess ) { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Hash Check Failed" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache chunk failed hash check %s" ), *ChunkGuid.ToString() ); } else { // Count loads NumDriveCacheChunkLoads.Increment(); GLog->Logf( TEXT( "FBuildPatchChunkCache: ReadChunkFromDriveCache loaded chunk %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Incorrect File Size" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache header info does not match file size %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Datasize/Hashtype Mismatch" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache mismatch datasize/hashtype combination %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Corrupt Header" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache corrupt header %s" ), *ChunkGuid.ToString() ); } // Release data NewChunkFile->ReleaseDataLock(); // Add the newly filled data to the cache if successful if( bSuccess ) { ChunkCache.Add( ChunkGuid, NewChunkFile ); } // If there was a problem, remove from cache and reservation else { ChunkCache.Remove( ChunkGuid ); } // Close the file FileReader->Close(); delete FileReader; } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, FPlatformMisc::GetLastError(), TEXT( "DriveCache" ), TEXT( "Open File Fail" ) ); GLog->Logf( TEXT( "BuildPatchServices: ERROR: GetChunkData could not open chunk file %s" ), *ChunkGuid.ToString() ); } return bSuccess; }