bool FBuildPatchFileConstructor::InsertChunkData(const FChunkPartData& ChunkPart, FArchive& DestinationFile, FSHA1& HashState)
{
uint8* Data;
uint8* DataStart;
FChunkFile* ChunkFile = FBuildPatchChunkCache::Get().GetChunkFile( ChunkPart.Guid );
if( ChunkFile != NULL && !FBuildPatchInstallError::HasFatalError() )
{
ChunkFile->GetDataLock( &Data, NULL );
DataStart = &Data[ ChunkPart.Offset ];
HashState.Update( DataStart, ChunkPart.Size );
DestinationFile.Serialize( DataStart, ChunkPart.Size );
ChunkFile->Dereference();
ChunkFile->ReleaseDataLock();
return true;
}
return false;
}
void FBuildPatchChunkCache::AddDataToCache( const FGuid& ChunkGuid, const TArray< uint8 >& ChunkDataFile )
{
// Must never double acquire
check( ChunkCache.Contains( ChunkGuid ) == false );
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), false );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Copy the data
FMemoryReader MemReader( ChunkDataFile );
MemReader << *ChunkHeader;
MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Release data
NewChunkFile->ReleaseDataLock();
// Add it to our cache.
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
void FBuildPatchChunkCache::ReserveChunkInventorySlotForce( const FGuid& ChunkGuid )
{
// If already reserved, return immediate
if( ChunkCache.HasReservation( ChunkGuid ) || ChunkCache.Contains( ChunkGuid ) )
{
return;
}
// Begin by checking if any slots can be freed
ChunkCache.PurgeUnreferenced();
// Try to add the reservation
bool bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid );
// If we couldn't reserve, we need to boot out a chunk for this required one
if( bReservationAccepted == false )
{
// We create a unique ref array from the use order so that chunks not needed
// for longer times end up nearer the bottom of the array
TArray< FGuid > ChunkPriorityList;
ChunkInfoLock.Lock();
for( int32 ChunkUseOrderStackIdx = ChunkUseOrderStack.Num() - 1; ChunkUseOrderStackIdx >= 0 ; --ChunkUseOrderStackIdx )
{
ChunkPriorityList.AddUnique( ChunkUseOrderStack[ ChunkUseOrderStackIdx ] );
}
ChunkInfoLock.Unlock();
// Starting at the bottom of the list, we look for a chunk that is contained in the cache
for( int32 ChunkPriorityListIdx = ChunkPriorityList.Num() - 1; ChunkPriorityListIdx >= 0 && !bReservationAccepted; --ChunkPriorityListIdx )
{
const FGuid& LowPriChunk = ChunkPriorityList[ ChunkPriorityListIdx ];
BuildProgress->WaitWhilePaused();
// Check if there were any errors while paused, like canceling
if( FBuildPatchInstallError::HasFatalError() )
{
return;
}
if( ChunkCache.Contains( LowPriChunk ) )
{
GWarn->Logf( TEXT( "FBuildPatchChunkCache: Booting chunk %s" ), *LowPriChunk.ToString() );
// Save chunk to disk so we don't have to download again
bool bSuccess = true;
const FString NewChunkFilename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, LowPriChunk );
FChunkFile* LowPriChunkFile = ChunkCache.Get( LowPriChunk );
FChunkHeader* LowPriChunkHeader;
uint8* LowPriChunkData;
LowPriChunkFile->GetDataLock( &LowPriChunkData, &LowPriChunkHeader );
FArchive* FileOut = IFileManager::Get().CreateFileWriter( *NewChunkFilename );
bSuccess = FileOut != NULL;
const int32 LastError = FPlatformMisc::GetLastError();
if( bSuccess )
{
// Setup Header
*FileOut << *LowPriChunkHeader;
LowPriChunkHeader->HeaderSize = FileOut->Tell();
LowPriChunkHeader->StoredAs = FChunkHeader::STORED_RAW;
LowPriChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting
// Write out file
FileOut->Seek( 0 );
*FileOut << *LowPriChunkHeader;
FileOut->Serialize( LowPriChunkData, FBuildPatchData::ChunkDataSize );
FileOut->Close();
delete FileOut;
}
LowPriChunkFile->ReleaseDataLock();
// Setup new chunk origin
if( bSuccess )
{
ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Harddisk;
}
else
{
// Queue download if save failed
ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Download;
FBuildPatchDownloader::Get().AddChunkToDownload( LowPriChunk );
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, NewChunkFilename, LastError, TEXT( "ChunkBooting" ), TEXT( "Chunk Save Failed" ) );
}
// Boot this chunk
ChunkCache.Remove( LowPriChunk );
// Try get the reservation again!
bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid );
// Count the boot
NumChunksCacheBooted.Increment();
}
}
// We must have been able to make room
check( bReservationAccepted );
}
}
bool FBuildPatchChunkCache::RecycleChunkFromBuild( const FGuid& ChunkGuid )
{
// Must never double acquire
check( ChunkCache.Contains( ChunkGuid ) == false );
// Debug leaving any files open
bool bSuccess = true;
// Get the app manifest that this chunk can be sourced from
FBuildPatchAppManifestPtr ChunkSourceAppManifest = InstallationInfo.GetManifestContainingChunk(ChunkGuid);
if (!ChunkSourceAppManifest.IsValid())
{
return false;
}
// Get the install directory for this manifest
const FString ChunkSourceInstallDir = InstallationInfo.GetManifestInstallDir(ChunkSourceAppManifest);
if(ChunkSourceInstallDir.Len() <= 0)
{
return false;
}
// We need to generate an inventory of all chunk parts in this build that refer to the chunk that we require
TMap< FGuid, TArray< FFileChunkPart > > ChunkPartInventory;
TArray< FGuid > Array;
Array.Add( ChunkGuid );
ChunkSourceAppManifest->EnumerateChunkPartInventory(Array, ChunkPartInventory);
// Attempt construction of the chunk from the parts
FArchive* BuildFileIn = NULL;
FString BuildFileOpened;
int64 BuildFileInSize = 0;
// We must have a hash for this chunk or else we cant verify it
uint8 HashType = 0;
uint64 ChunkHash = 0;
FSHAHashData ChunkShaHash;
if (InstallManifet->GetChunkShaHash(ChunkGuid, ChunkShaHash))
{
HashType = FChunkHeader::HASH_SHA1;
}
else if (ChunkSourceAppManifest->GetChunkHash(ChunkGuid, ChunkHash))
{
HashType = FChunkHeader::HASH_ROLLING;
}
TArray< FFileChunkPart >* FileChunkPartsPtr = ChunkPartInventory.Find( ChunkGuid );
bSuccess = (FileChunkPartsPtr != NULL && HashType != 0);
if( bSuccess )
{
const TArray< FFileChunkPart >& FileChunkParts = *FileChunkPartsPtr;
TArray< uint8 > TempArray;
TempArray.AddUninitialized( FBuildPatchData::ChunkDataSize );
uint8* TempChunkConstruction = TempArray.GetData();
FMemory::Memzero( TempChunkConstruction, FBuildPatchData::ChunkDataSize );
bSuccess = FileChunkParts.Num() > 0;
for( auto FileChunkPartIt = FileChunkParts.CreateConstIterator(); FileChunkPartIt && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++FileChunkPartIt )
{
const FFileChunkPart& FileChunkPart = *FileChunkPartIt;
FString FullFilename = ChunkSourceInstallDir / FileChunkPart.Filename;
// Close current build file ?
if( BuildFileIn != NULL && BuildFileOpened != FullFilename )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
BuildFileOpened = TEXT( "" );
BuildFileInSize = 0;
}
// Open build file ?
if( BuildFileIn == NULL )
{
BuildFileIn = IFileManager::Get().CreateFileReader( *FullFilename );
bSuccess = BuildFileIn != NULL;
if( !bSuccess )
{
BuildFileOpened = TEXT( "" );
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, FPlatformMisc::GetLastError(), TEXT( "ChunkRecycle" ), TEXT( "Source File Missing" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Failed to load source file for chunk. %s" ), *FullFilename );
}
else
{
BuildFileOpened = FullFilename;
BuildFileInSize = BuildFileIn->TotalSize();
}
}
// Grab the section of the chunk
if( BuildFileIn != NULL )
{
// Make sure we don't attempt to read off the end of the file
const int64 LastRequiredByte = FileChunkPart.FileOffset + FileChunkPart.ChunkPart.Size;
if( BuildFileInSize >= LastRequiredByte )
{
BuildFileIn->Seek( FileChunkPart.FileOffset );
BuildFileIn->Serialize( TempChunkConstruction + FileChunkPart.ChunkPart.Offset, FileChunkPart.ChunkPart.Size );
}
else
{
bSuccess = false;
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Source File Too Small" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Source file too small for chunk position. %s" ), *FullFilename );
}
}
}
// Check no other fatal errors were registered in the meantime
bSuccess = bSuccess && !FBuildPatchInstallError::HasFatalError();
// Check chunk hash
if( bSuccess )
{
FSHAHashData ShaHashCheck;
switch (HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(TempChunkConstruction) == ChunkHash;
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(TempChunkConstruction, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkShaHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, TEXT( "" ), INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Chunk Hash Fail" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Hash check failed for recycled chunk %s" ), *ChunkGuid.ToString() );
}
}
// Save the chunk to cache if all went well
if( bSuccess )
{
// It was added asynchronously!!
check( ChunkCache.Contains( ChunkGuid ) == false );
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Copy the data
FMemoryReader MemReader( TempArray );
MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Setup the header
ChunkHeader->Guid = ChunkGuid;
ChunkHeader->StoredAs = FChunkHeader::STORED_RAW;
ChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting
ChunkHeader->HashType = HashType;
ChunkHeader->RollingHash = ChunkHash;
ChunkHeader->SHAHash = ChunkShaHash;
// Release data
NewChunkFile->ReleaseDataLock();
// Count chunk
NumChunksRecycled.Increment();
// Add it to our cache.
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// Close any open file
if( BuildFileIn != NULL )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
}
}
return bSuccess;
}
bool FBuildPatchChunkCache::ReadChunkFromDriveCache( const FGuid& ChunkGuid )
{
bool bSuccess = true;
// Get the chunk filename
const FString Filename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, ChunkGuid );
// Read the chunk
FArchive* FileReader = IFileManager::Get().CreateFileReader( *Filename );
bSuccess = FileReader != NULL;
if( bSuccess )
{
// Get file size
const int64 FileSize = FileReader->TotalSize();
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Read the header
*FileReader << *ChunkHeader;
// Check header magic
bSuccess = ChunkHeader->IsValidMagic();
if ( bSuccess )
{
// Check the right data size
bSuccess = ChunkHeader->DataSize == FBuildPatchData::ChunkDataSize;
if( bSuccess )
{
// Check Header and data size
bSuccess = ( ChunkHeader->HeaderSize + ChunkHeader->DataSize ) == FileSize;
if( bSuccess )
{
// Read the data
FileReader->Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Verify the data hash
FSHAHashData ShaHashCheck;
switch (ChunkHeader->HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = ChunkHeader->RollingHash == FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(ChunkData);
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(ChunkData, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkHeader->SHAHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Hash Check Failed" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache chunk failed hash check %s" ), *ChunkGuid.ToString() );
}
else
{
// Count loads
NumDriveCacheChunkLoads.Increment();
GLog->Logf( TEXT( "FBuildPatchChunkCache: ReadChunkFromDriveCache loaded chunk %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Incorrect File Size" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache header info does not match file size %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Datasize/Hashtype Mismatch" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache mismatch datasize/hashtype combination %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Corrupt Header" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache corrupt header %s" ), *ChunkGuid.ToString() );
}
// Release data
NewChunkFile->ReleaseDataLock();
// Add the newly filled data to the cache if successful
if( bSuccess )
{
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// If there was a problem, remove from cache and reservation
else
{
ChunkCache.Remove( ChunkGuid );
}
// Close the file
FileReader->Close();
delete FileReader;
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, FPlatformMisc::GetLastError(), TEXT( "DriveCache" ), TEXT( "Open File Fail" ) );
GLog->Logf( TEXT( "BuildPatchServices: ERROR: GetChunkData could not open chunk file %s" ), *ChunkGuid.ToString() );
}
return bSuccess;
}
