bool FBuildPatchFileConstructor::InsertChunkData(const FChunkPartData& ChunkPart, FArchive& DestinationFile, FSHA1& HashState) { uint8* Data; uint8* DataStart; FChunkFile* ChunkFile = FBuildPatchChunkCache::Get().GetChunkFile( ChunkPart.Guid ); if( ChunkFile != NULL && !FBuildPatchInstallError::HasFatalError() ) { ChunkFile->GetDataLock( &Data, NULL ); DataStart = &Data[ ChunkPart.Offset ]; HashState.Update( DataStart, ChunkPart.Size ); DestinationFile.Serialize( DataStart, ChunkPart.Size ); ChunkFile->Dereference(); ChunkFile->ReleaseDataLock(); return true; } return false; }
void FBuildPatchChunkCache::AddDataToCache( const FGuid& ChunkGuid, const TArray< uint8 >& ChunkDataFile ) { // Must never double acquire check( ChunkCache.Contains( ChunkGuid ) == false ); // Create the ChunkFile data structure FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), false ); // Lock data FChunkHeader* ChunkHeader; uint8* ChunkData; NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader ); // Copy the data FMemoryReader MemReader( ChunkDataFile ); MemReader << *ChunkHeader; MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize ); // Release data NewChunkFile->ReleaseDataLock(); // Add it to our cache. ChunkCache.Add( ChunkGuid, NewChunkFile ); }
void FBuildPatchChunkCache::ReserveChunkInventorySlotForce( const FGuid& ChunkGuid ) { // If already reserved, return immediate if( ChunkCache.HasReservation( ChunkGuid ) || ChunkCache.Contains( ChunkGuid ) ) { return; } // Begin by checking if any slots can be freed ChunkCache.PurgeUnreferenced(); // Try to add the reservation bool bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid ); // If we couldn't reserve, we need to boot out a chunk for this required one if( bReservationAccepted == false ) { // We create a unique ref array from the use order so that chunks not needed // for longer times end up nearer the bottom of the array TArray< FGuid > ChunkPriorityList; ChunkInfoLock.Lock(); for( int32 ChunkUseOrderStackIdx = ChunkUseOrderStack.Num() - 1; ChunkUseOrderStackIdx >= 0 ; --ChunkUseOrderStackIdx ) { ChunkPriorityList.AddUnique( ChunkUseOrderStack[ ChunkUseOrderStackIdx ] ); } ChunkInfoLock.Unlock(); // Starting at the bottom of the list, we look for a chunk that is contained in the cache for( int32 ChunkPriorityListIdx = ChunkPriorityList.Num() - 1; ChunkPriorityListIdx >= 0 && !bReservationAccepted; --ChunkPriorityListIdx ) { const FGuid& LowPriChunk = ChunkPriorityList[ ChunkPriorityListIdx ]; BuildProgress->WaitWhilePaused(); // Check if there were any errors while paused, like canceling if( FBuildPatchInstallError::HasFatalError() ) { return; } if( ChunkCache.Contains( LowPriChunk ) ) { GWarn->Logf( TEXT( "FBuildPatchChunkCache: Booting chunk %s" ), *LowPriChunk.ToString() ); // Save chunk to disk so we don't have to download again bool bSuccess = true; const FString NewChunkFilename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, LowPriChunk ); FChunkFile* LowPriChunkFile = ChunkCache.Get( LowPriChunk ); FChunkHeader* LowPriChunkHeader; uint8* LowPriChunkData; LowPriChunkFile->GetDataLock( &LowPriChunkData, &LowPriChunkHeader ); FArchive* FileOut = IFileManager::Get().CreateFileWriter( *NewChunkFilename ); bSuccess = FileOut != NULL; const int32 LastError = FPlatformMisc::GetLastError(); if( bSuccess ) { // Setup Header *FileOut << *LowPriChunkHeader; LowPriChunkHeader->HeaderSize = FileOut->Tell(); LowPriChunkHeader->StoredAs = FChunkHeader::STORED_RAW; LowPriChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting // Write out file FileOut->Seek( 0 ); *FileOut << *LowPriChunkHeader; FileOut->Serialize( LowPriChunkData, FBuildPatchData::ChunkDataSize ); FileOut->Close(); delete FileOut; } LowPriChunkFile->ReleaseDataLock(); // Setup new chunk origin if( bSuccess ) { ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Harddisk; } else { // Queue download if save failed ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Download; FBuildPatchDownloader::Get().AddChunkToDownload( LowPriChunk ); FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, NewChunkFilename, LastError, TEXT( "ChunkBooting" ), TEXT( "Chunk Save Failed" ) ); } // Boot this chunk ChunkCache.Remove( LowPriChunk ); // Try get the reservation again! bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid ); // Count the boot NumChunksCacheBooted.Increment(); } } // We must have been able to make room check( bReservationAccepted ); } }
bool FBuildPatchChunkCache::RecycleChunkFromBuild( const FGuid& ChunkGuid ) { // Must never double acquire check( ChunkCache.Contains( ChunkGuid ) == false ); // Debug leaving any files open bool bSuccess = true; // Get the app manifest that this chunk can be sourced from FBuildPatchAppManifestPtr ChunkSourceAppManifest = InstallationInfo.GetManifestContainingChunk(ChunkGuid); if (!ChunkSourceAppManifest.IsValid()) { return false; } // Get the install directory for this manifest const FString ChunkSourceInstallDir = InstallationInfo.GetManifestInstallDir(ChunkSourceAppManifest); if(ChunkSourceInstallDir.Len() <= 0) { return false; } // We need to generate an inventory of all chunk parts in this build that refer to the chunk that we require TMap< FGuid, TArray< FFileChunkPart > > ChunkPartInventory; TArray< FGuid > Array; Array.Add( ChunkGuid ); ChunkSourceAppManifest->EnumerateChunkPartInventory(Array, ChunkPartInventory); // Attempt construction of the chunk from the parts FArchive* BuildFileIn = NULL; FString BuildFileOpened; int64 BuildFileInSize = 0; // We must have a hash for this chunk or else we cant verify it uint8 HashType = 0; uint64 ChunkHash = 0; FSHAHashData ChunkShaHash; if (InstallManifet->GetChunkShaHash(ChunkGuid, ChunkShaHash)) { HashType = FChunkHeader::HASH_SHA1; } else if (ChunkSourceAppManifest->GetChunkHash(ChunkGuid, ChunkHash)) { HashType = FChunkHeader::HASH_ROLLING; } TArray< FFileChunkPart >* FileChunkPartsPtr = ChunkPartInventory.Find( ChunkGuid ); bSuccess = (FileChunkPartsPtr != NULL && HashType != 0); if( bSuccess ) { const TArray< FFileChunkPart >& FileChunkParts = *FileChunkPartsPtr; TArray< uint8 > TempArray; TempArray.AddUninitialized( FBuildPatchData::ChunkDataSize ); uint8* TempChunkConstruction = TempArray.GetData(); FMemory::Memzero( TempChunkConstruction, FBuildPatchData::ChunkDataSize ); bSuccess = FileChunkParts.Num() > 0; for( auto FileChunkPartIt = FileChunkParts.CreateConstIterator(); FileChunkPartIt && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++FileChunkPartIt ) { const FFileChunkPart& FileChunkPart = *FileChunkPartIt; FString FullFilename = ChunkSourceInstallDir / FileChunkPart.Filename; // Close current build file ? if( BuildFileIn != NULL && BuildFileOpened != FullFilename ) { BuildFileIn->Close(); delete BuildFileIn; BuildFileIn = NULL; BuildFileOpened = TEXT( "" ); BuildFileInSize = 0; } // Open build file ? if( BuildFileIn == NULL ) { BuildFileIn = IFileManager::Get().CreateFileReader( *FullFilename ); bSuccess = BuildFileIn != NULL; if( !bSuccess ) { BuildFileOpened = TEXT( "" ); FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, FPlatformMisc::GetLastError(), TEXT( "ChunkRecycle" ), TEXT( "Source File Missing" ) ); GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Failed to load source file for chunk. %s" ), *FullFilename ); } else { BuildFileOpened = FullFilename; BuildFileInSize = BuildFileIn->TotalSize(); } } // Grab the section of the chunk if( BuildFileIn != NULL ) { // Make sure we don't attempt to read off the end of the file const int64 LastRequiredByte = FileChunkPart.FileOffset + FileChunkPart.ChunkPart.Size; if( BuildFileInSize >= LastRequiredByte ) { BuildFileIn->Seek( FileChunkPart.FileOffset ); BuildFileIn->Serialize( TempChunkConstruction + FileChunkPart.ChunkPart.Offset, FileChunkPart.ChunkPart.Size ); } else { bSuccess = false; FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Source File Too Small" ) ); GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Source file too small for chunk position. %s" ), *FullFilename ); } } } // Check no other fatal errors were registered in the meantime bSuccess = bSuccess && !FBuildPatchInstallError::HasFatalError(); // Check chunk hash if( bSuccess ) { FSHAHashData ShaHashCheck; switch (HashType) { case FChunkHeader::HASH_ROLLING: bSuccess = FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(TempChunkConstruction) == ChunkHash; break; case FChunkHeader::HASH_SHA1: FSHA1::HashBuffer(TempChunkConstruction, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash); bSuccess = ShaHashCheck == ChunkShaHash; break; default: bSuccess = false; } if( !bSuccess ) { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, TEXT( "" ), INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Chunk Hash Fail" ) ); GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Hash check failed for recycled chunk %s" ), *ChunkGuid.ToString() ); } } // Save the chunk to cache if all went well if( bSuccess ) { // It was added asynchronously!! check( ChunkCache.Contains( ChunkGuid ) == false ); // Create the ChunkFile data structure FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true ); // Lock data FChunkHeader* ChunkHeader; uint8* ChunkData; NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader ); // Copy the data FMemoryReader MemReader( TempArray ); MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize ); // Setup the header ChunkHeader->Guid = ChunkGuid; ChunkHeader->StoredAs = FChunkHeader::STORED_RAW; ChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting ChunkHeader->HashType = HashType; ChunkHeader->RollingHash = ChunkHash; ChunkHeader->SHAHash = ChunkShaHash; // Release data NewChunkFile->ReleaseDataLock(); // Count chunk NumChunksRecycled.Increment(); // Add it to our cache. ChunkCache.Add( ChunkGuid, NewChunkFile ); } // Close any open file if( BuildFileIn != NULL ) { BuildFileIn->Close(); delete BuildFileIn; BuildFileIn = NULL; } } return bSuccess; }
bool FBuildPatchChunkCache::ReadChunkFromDriveCache( const FGuid& ChunkGuid ) { bool bSuccess = true; // Get the chunk filename const FString Filename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, ChunkGuid ); // Read the chunk FArchive* FileReader = IFileManager::Get().CreateFileReader( *Filename ); bSuccess = FileReader != NULL; if( bSuccess ) { // Get file size const int64 FileSize = FileReader->TotalSize(); // Create the ChunkFile data structure FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true ); // Lock data FChunkHeader* ChunkHeader; uint8* ChunkData; NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader ); // Read the header *FileReader << *ChunkHeader; // Check header magic bSuccess = ChunkHeader->IsValidMagic(); if ( bSuccess ) { // Check the right data size bSuccess = ChunkHeader->DataSize == FBuildPatchData::ChunkDataSize; if( bSuccess ) { // Check Header and data size bSuccess = ( ChunkHeader->HeaderSize + ChunkHeader->DataSize ) == FileSize; if( bSuccess ) { // Read the data FileReader->Serialize( ChunkData, FBuildPatchData::ChunkDataSize ); // Verify the data hash FSHAHashData ShaHashCheck; switch (ChunkHeader->HashType) { case FChunkHeader::HASH_ROLLING: bSuccess = ChunkHeader->RollingHash == FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(ChunkData); break; case FChunkHeader::HASH_SHA1: FSHA1::HashBuffer(ChunkData, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash); bSuccess = ShaHashCheck == ChunkHeader->SHAHash; break; default: bSuccess = false; } if( !bSuccess ) { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Hash Check Failed" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache chunk failed hash check %s" ), *ChunkGuid.ToString() ); } else { // Count loads NumDriveCacheChunkLoads.Increment(); GLog->Logf( TEXT( "FBuildPatchChunkCache: ReadChunkFromDriveCache loaded chunk %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Incorrect File Size" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache header info does not match file size %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Datasize/Hashtype Mismatch" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache mismatch datasize/hashtype combination %s" ), *ChunkGuid.ToString() ); } } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Corrupt Header" ) ); GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache corrupt header %s" ), *ChunkGuid.ToString() ); } // Release data NewChunkFile->ReleaseDataLock(); // Add the newly filled data to the cache if successful if( bSuccess ) { ChunkCache.Add( ChunkGuid, NewChunkFile ); } // If there was a problem, remove from cache and reservation else { ChunkCache.Remove( ChunkGuid ); } // Close the file FileReader->Close(); delete FileReader; } else { FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, FPlatformMisc::GetLastError(), TEXT( "DriveCache" ), TEXT( "Open File Fail" ) ); GLog->Logf( TEXT( "BuildPatchServices: ERROR: GetChunkData could not open chunk file %s" ), *ChunkGuid.ToString() ); } return bSuccess; }