void FBuildPatchAppManifest::GetOutdatedFiles(FBuildPatchAppManifestPtr OldManifest, FBuildPatchAppManifestRef NewManifest, const FString& InstallDirectory, TArray< FString >& OutDatedFiles)
{
if (!OldManifest.IsValid())
{
// All files are outdated if no OldManifest
NewManifest->FileManifestLookup.GetKeys(OutDatedFiles);
}
else
{
// Enumerate files in the NewManifest file list, that do not exist, or have different hashes in the OldManifest
// to be files no longer required by the build
for (const auto& NewFile : NewManifest->Data->FileManifestList)
{
const int64 ExistingFileSize = IFileManager::Get().FileSize(*(InstallDirectory / NewFile.Filename));
// Check changed
if (IsFileOutdated(OldManifest.ToSharedRef(), NewManifest, NewFile.Filename))
{
OutDatedFiles.Add(NewFile.Filename);
}
// Double check an unchanged file is not missing (size will be -1) or is incorrect size
else if (ExistingFileSize != NewFile.GetFileSize())
{
OutDatedFiles.Add(NewFile.Filename);
}
}
}
}
IBuildInstallerPtr FBuildPatchServicesModule::StartBuildInstall( IBuildManifestPtr CurrentManifest, IBuildManifestPtr InstallManifest, const FString& InstallDirectory, FBuildPatchBoolManifestDelegate OnCompleteDelegate )
{
// Using a local bool for this check will improve the assert message that gets displayed
const bool bIsCalledFromMainThread = IsInGameThread();
check( bIsCalledFromMainThread );
// Cast manifest parameters
FBuildPatchAppManifestPtr CurrentManifestInternal = StaticCastSharedPtr< FBuildPatchAppManifest >( CurrentManifest );
FBuildPatchAppManifestPtr InstallManifestInternal = StaticCastSharedPtr< FBuildPatchAppManifest >( InstallManifest );
if( !InstallManifestInternal.IsValid() )
{
// We must have an install manifest to continue
return NULL;
}
// Make directory
IFileManager::Get().MakeDirectory( *InstallDirectory, true );
if( !IFileManager::Get().DirectoryExists( *InstallDirectory ) )
{
return NULL;
}
// Make sure the http wrapper is already created
FBuildPatchHTTP::Initialize();
// Run the install thread
BuildPatchInstallers.Add( MakeShareable( new FBuildPatchInstaller( OnCompleteDelegate, CurrentManifestInternal, InstallManifestInternal.ToSharedRef(), InstallDirectory, GetStagingDirectory(), InstallationInfo, false ) ) );
return BuildPatchInstallers.Top();
}
/* FBuildPatchChunkCache implementation
*****************************************************************************/
FBuildPatchChunkCache::FBuildPatchChunkCache( const FBuildPatchAppManifestRef& InInstallManifet, const FBuildPatchAppManifestPtr& InCurrentManifest, const FString& InChunkCacheStage, const FString& InCurrentInstallDir, FBuildPatchProgress* InBuildProgress, TArray<FString>& FilesToConstruct, FBuildPatchInstallationInfo& InstallationInfoRef )
: ChunkCacheStage(InChunkCacheStage)
, CurrentInstallDir( InCurrentInstallDir )
, InstallManifet( InInstallManifet )
, CurrentManifest( InCurrentManifest )
, BuildProgress( InBuildProgress )
, bEnableChunkRecycle( InCurrentManifest.IsValid() )
, bDownloadsStarted( 0 )
, SkippedChunkDownloadSize( 0 )
, InstallationInfo( InstallationInfoRef )
{
// Setup chunk information
InstallManifet->GetChunksRequiredForFiles( FilesToConstruct, FullChunkRefList, false );
for( int32 FullChunkRefListIdx = FullChunkRefList.Num() - 1; FullChunkRefListIdx >= 0; --FullChunkRefListIdx )
{
const FGuid& ChunkGuid = FullChunkRefList[ FullChunkRefListIdx ];
ChunkOrigins.Add( ChunkGuid, EChunkOrigin::Download );
if( ChunkUseOrderStack.Num() == 0 || ChunkUseOrderStack.Last() != ChunkGuid )
{
ChunkUseOrderStack.Push( ChunkGuid );
}
}
// Get unique list of downloads
InstallManifet->GetChunksRequiredForFiles( FilesToConstruct, ChunksToDownload, true );
// Check for chunks that can be constructed locally
TArray< FGuid > ChunksToConstruct;
InstallationInfo.EnumerateProducibleChunks(FullChunkRefList, ChunksToConstruct);
for( auto ChunksToConstructIt = ChunksToConstruct.CreateConstIterator(); ChunksToConstructIt; ++ChunksToConstructIt )
{
const FGuid& ChunkGuid = *ChunksToConstructIt;
ChunkOrigins.Add( ChunkGuid, EChunkOrigin::Recycle );
ChunksToDownload.Remove( ChunkGuid );
}
// Init chunk stats
NumFilesToConstruct = FilesToConstruct.Num();
NumChunksToDownload = ChunksToDownload.Num();
NumChunksToRecycle = ChunksToConstruct.Num();
NumRequiredChunks = NumChunksToDownload + NumChunksToRecycle;
TotalChunkDownloadSize = InstallManifet->GetDataSize(ChunksToDownload);
NumChunksRecycled.Reset();
NumChunksCacheBooted.Reset();
NumDriveCacheChunkLoads.Reset();
NumRecycleFailures.Reset();
NumDriveCacheLoadFailures.Reset();
}
bool FBuildPatchChunkCache::RecycleChunkFromBuild( const FGuid& ChunkGuid )
{
// Must never double acquire
check( ChunkCache.Contains( ChunkGuid ) == false );
// Debug leaving any files open
bool bSuccess = true;
// Get the app manifest that this chunk can be sourced from
FBuildPatchAppManifestPtr ChunkSourceAppManifest = InstallationInfo.GetManifestContainingChunk(ChunkGuid);
if (!ChunkSourceAppManifest.IsValid())
{
return false;
}
// Get the install directory for this manifest
const FString ChunkSourceInstallDir = InstallationInfo.GetManifestInstallDir(ChunkSourceAppManifest);
if(ChunkSourceInstallDir.Len() <= 0)
{
return false;
}
// We need to generate an inventory of all chunk parts in this build that refer to the chunk that we require
TMap< FGuid, TArray< FFileChunkPart > > ChunkPartInventory;
TArray< FGuid > Array;
Array.Add( ChunkGuid );
ChunkSourceAppManifest->EnumerateChunkPartInventory(Array, ChunkPartInventory);
// Attempt construction of the chunk from the parts
FArchive* BuildFileIn = NULL;
FString BuildFileOpened;
int64 BuildFileInSize = 0;
// We must have a hash for this chunk or else we cant verify it
uint8 HashType = 0;
uint64 ChunkHash = 0;
FSHAHashData ChunkShaHash;
if (InstallManifet->GetChunkShaHash(ChunkGuid, ChunkShaHash))
{
HashType = FChunkHeader::HASH_SHA1;
}
else if (ChunkSourceAppManifest->GetChunkHash(ChunkGuid, ChunkHash))
{
HashType = FChunkHeader::HASH_ROLLING;
}
TArray< FFileChunkPart >* FileChunkPartsPtr = ChunkPartInventory.Find( ChunkGuid );
bSuccess = (FileChunkPartsPtr != NULL && HashType != 0);
if( bSuccess )
{
const TArray< FFileChunkPart >& FileChunkParts = *FileChunkPartsPtr;
TArray< uint8 > TempArray;
TempArray.AddUninitialized( FBuildPatchData::ChunkDataSize );
uint8* TempChunkConstruction = TempArray.GetData();
FMemory::Memzero( TempChunkConstruction, FBuildPatchData::ChunkDataSize );
bSuccess = FileChunkParts.Num() > 0;
for( auto FileChunkPartIt = FileChunkParts.CreateConstIterator(); FileChunkPartIt && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++FileChunkPartIt )
{
const FFileChunkPart& FileChunkPart = *FileChunkPartIt;
FString FullFilename = ChunkSourceInstallDir / FileChunkPart.Filename;
// Close current build file ?
if( BuildFileIn != NULL && BuildFileOpened != FullFilename )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
BuildFileOpened = TEXT( "" );
BuildFileInSize = 0;
}
// Open build file ?
if( BuildFileIn == NULL )
{
BuildFileIn = IFileManager::Get().CreateFileReader( *FullFilename );
bSuccess = BuildFileIn != NULL;
if( !bSuccess )
{
BuildFileOpened = TEXT( "" );
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, FPlatformMisc::GetLastError(), TEXT( "ChunkRecycle" ), TEXT( "Source File Missing" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Failed to load source file for chunk. %s" ), *FullFilename );
}
else
{
BuildFileOpened = FullFilename;
BuildFileInSize = BuildFileIn->TotalSize();
}
}
// Grab the section of the chunk
if( BuildFileIn != NULL )
{
// Make sure we don't attempt to read off the end of the file
const int64 LastRequiredByte = FileChunkPart.FileOffset + FileChunkPart.ChunkPart.Size;
if( BuildFileInSize >= LastRequiredByte )
{
BuildFileIn->Seek( FileChunkPart.FileOffset );
BuildFileIn->Serialize( TempChunkConstruction + FileChunkPart.ChunkPart.Offset, FileChunkPart.ChunkPart.Size );
}
else
{
bSuccess = false;
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Source File Too Small" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Source file too small for chunk position. %s" ), *FullFilename );
}
}
}
// Check no other fatal errors were registered in the meantime
bSuccess = bSuccess && !FBuildPatchInstallError::HasFatalError();
// Check chunk hash
if( bSuccess )
{
FSHAHashData ShaHashCheck;
switch (HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(TempChunkConstruction) == ChunkHash;
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(TempChunkConstruction, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkShaHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, TEXT( "" ), INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Chunk Hash Fail" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Hash check failed for recycled chunk %s" ), *ChunkGuid.ToString() );
}
}
// Save the chunk to cache if all went well
if( bSuccess )
{
// It was added asynchronously!!
check( ChunkCache.Contains( ChunkGuid ) == false );
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Copy the data
FMemoryReader MemReader( TempArray );
MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Setup the header
ChunkHeader->Guid = ChunkGuid;
ChunkHeader->StoredAs = FChunkHeader::STORED_RAW;
ChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting
ChunkHeader->HashType = HashType;
ChunkHeader->RollingHash = ChunkHash;
ChunkHeader->SHAHash = ChunkShaHash;
// Release data
NewChunkFile->ReleaseDataLock();
// Count chunk
NumChunksRecycled.Increment();
// Add it to our cache.
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// Close any open file
if( BuildFileIn != NULL )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
}
}
return bSuccess;
}
