QString InsertTmpFile(const QString &fileurl) {
QString filename = TmpFilePath(fileurl);
QFile file(fileurl);
file.open(QIODevice::ReadOnly);
QString tmpPath = InsertFileData(filename, file.readAll());
file.close();
return tmpPath;
}
QString InsertTmpFileList(const QString &subdir, const QStringList &fileurlList) {
QString tmpDir = TmpFilePath(subdir);
QDir dir(tmpDir);
dir.mkdir(".");
foreach (QString fileurl, fileurlList) {
QString filename = tmpDir + "/" + fileurl.split("/").last();
QFile file(fileurl);
file.open(QIODevice::ReadOnly);
InsertFileData(filename, file.readAll());
file.close();
}
bool FBuildPatchFileConstructor::ConstructFileFromChunks( const FString& Filename, bool bResumeExisting )
{
const bool bIsFileData = BuildManifest->IsFileDataManifest();
bResumeExisting = bResumeExisting && !bIsFileData;
bool bSuccess = true;
FString ErrorString;
FString NewFilename = StagingDirectory / Filename;
// Calculate the hash as we write the data
FSHA1 HashState;
FSHAHashData HashValue;
// First make sure we can get the file manifest
const FFileManifestData* FileManifest = BuildManifest->GetFileManifest(Filename);
bSuccess = FileManifest != nullptr;
if( bSuccess )
{
if( !FileManifest->SymlinkTarget.IsEmpty() )
{
#if PLATFORM_MAC
bSuccess = symlink(TCHAR_TO_UTF8(*FileManifest->SymlinkTarget), TCHAR_TO_UTF8(*NewFilename)) == 0;
#else
const bool bSymlinkNotImplemented = false;
check(bSymlinkNotImplemented);
bSuccess = false;
#endif
return bSuccess;
}
// Check for resuming of existing file
int64 StartPosition = 0;
int32 StartChunkPart = 0;
if( bResumeExisting )
{
// We have to read in the existing file so that the hash check can still be done.
FArchive* NewFileReader = IFileManager::Get().CreateFileReader( *NewFilename );
if( NewFileReader != NULL )
{
// Read buffer
uint8* ReadBuffer = new uint8[ FBuildPatchData::ChunkDataSize ];
// Reuse a certain amount of the file
StartPosition = FMath::Max<int64>( 0, NewFileReader->TotalSize() - NUM_BYTES_RESUME_IGNORE );
// We'll also find the correct chunkpart to start writing from
int64 ByteCounter = 0;
for( int32 ChunkPartIdx = StartChunkPart; ChunkPartIdx < FileManifest->FileChunkParts.Num() && !FBuildPatchInstallError::HasFatalError(); ++ChunkPartIdx )
{
const FChunkPartData& ChunkPart = FileManifest->FileChunkParts[ ChunkPartIdx ];
const int64 NextBytePosition = ByteCounter + ChunkPart.Size;
if( NextBytePosition <= StartPosition )
{
// Read data for hash check
NewFileReader->Serialize( ReadBuffer, ChunkPart.Size );
HashState.Update( ReadBuffer, ChunkPart.Size );
// Count bytes read from file
ByteCounter = NextBytePosition;
// Set to resume from next chunk part
StartChunkPart = ChunkPartIdx + 1;
// Inform the chunk cache of the chunk part skip
FBuildPatchChunkCache::Get().SkipChunkPart( ChunkPart );
// Wait if paused
BuildProgress->WaitWhilePaused();
}
else
{
// No more parts on disk
break;
}
}
// Set start position to the byte we got up to
StartPosition = ByteCounter;
// Clean read buffer
delete[] ReadBuffer;
// Close file
NewFileReader->Close();
delete NewFileReader;
}
}
// Now we can make sure the chunk cache knows to start downloading chunks
if( !bIsFileData && !bIsDownloadStarted && !FBuildPatchInstallError::HasFatalError() )
{
bIsDownloadStarted = true;
FBuildPatchChunkCache::Get().BeginDownloads();
}
// Attempt to create the file
FArchive* NewFile = IFileManager::Get().CreateFileWriter( *NewFilename, bResumeExisting ? EFileWrite::FILEWRITE_Append : 0 );
bSuccess = NewFile != NULL;
if( bSuccess )
{
// Whenever we start writing again, there's no more resuming to be done
BuildProgress->SetStateProgress( EBuildPatchProgress::Resuming, 1.0f );
// Seek to file write position
NewFile->Seek( StartPosition );
// For each chunk, load it, and place it's data into the file
for( int32 ChunkPartIdx = StartChunkPart; ChunkPartIdx < FileManifest->FileChunkParts.Num() && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++ChunkPartIdx )
{
const FChunkPartData& ChunkPart = FileManifest->FileChunkParts[ChunkPartIdx];
if( bIsFileData )
{
bSuccess = InsertFileData( ChunkPart, *NewFile, HashState );
}
else
{
bSuccess = InsertChunkData( ChunkPart, *NewFile, HashState );
}
if( bSuccess )
{
CountBytesProcessed( ChunkPart.Size );
// Wait if paused
BuildProgress->WaitWhilePaused();
}
else
{
ErrorString = TEXT( "Failed to construct file " );
ErrorString += Filename;
ErrorString += TEXT( " because of chunk " );
ErrorString += ChunkPart.Guid.ToString();
GWarn->Logf( TEXT( "BuildPatchFileConstructor: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
// Close the file writer
NewFile->Close();
delete NewFile;
}
else
{
FBuildPatchAnalytics::RecordConstructionError( Filename, FPlatformMisc::GetLastError(), TEXT( "Could Not Create File" ) );
ErrorString = TEXT( "Could not create new file " );
ErrorString += Filename;
GWarn->Logf( TEXT( "BuildPatchFileConstructor: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
else
{
FBuildPatchAnalytics::RecordConstructionError( Filename, INDEX_NONE, TEXT( "Missing File Manifest" ) );
ErrorString = TEXT( "Build manifest does not contain a file manifest for " );
ErrorString += Filename;
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
// Verify the hash for the file that we created
if( bSuccess )
{
HashState.Final();
HashState.GetHash( HashValue.Hash );
bSuccess = HashValue == FileManifest->FileHash;
if( !bSuccess )
{
FBuildPatchAnalytics::RecordConstructionError( Filename, INDEX_NONE, TEXT( "Serialised Verify Fail" ) );
ErrorString = TEXT( "Verify failed after constructing file " );
ErrorString += Filename;
GWarn->Logf( TEXT( "BuildDataGenerator: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
#if PLATFORM_MAC
if( bSuccess && FileManifest->bIsUnixExecutable )
{
// Enable executable permission bit
struct stat FileInfo;
if (stat(TCHAR_TO_UTF8(*NewFilename), &FileInfo) == 0)
{
bSuccess = chmod(TCHAR_TO_UTF8(*NewFilename), FileInfo.st_mode | S_IXUSR | S_IXGRP | S_IXOTH) == 0;
}
}
#endif
// Delete the staging file if unsuccessful by means of construction fail (i.e. keep if canceled or download issue)
if( !bSuccess && FBuildPatchInstallError::GetErrorState() == EBuildPatchInstallError::FileConstructionFail )
{
IFileManager::Get().Delete( *NewFilename, false, true );
}
return bSuccess;
}
