/**
* Load a text file to an FString.
* Supports all combination of ANSI/Unicode files and platforms.
* @param Result string representation of the loaded file
* @param Filename name of the file to load
* @param VerifyFlags flags controlling the hash verification behavior ( see EHashOptions )
*/
bool FFileHelper::LoadFileToString( FString& Result, const TCHAR* Filename, uint32 VerifyFlags )
{
FArchive* Reader = IFileManager::Get().CreateFileReader( Filename );
if( !Reader )
{
return 0;
}
int32 Size = Reader->TotalSize();
uint8* Ch = (uint8*)FMemory::Malloc(Size);
Reader->Serialize( Ch, Size );
bool Success = Reader->Close();
delete Reader;
BufferToString( Result, Ch, Size );
// handle SHA verify of the file
if( (VerifyFlags & EHashOptions::EnableVerify) && ( (VerifyFlags & EHashOptions::ErrorMissingHash) || FSHA1::GetFileSHAHash(Filename, NULL) ) )
{
// kick off SHA verify task. this frees the buffer on close
FBufferReaderWithSHA Ar( Ch, Size, true, Filename, false, true );
}
else
{
// free manually since not running SHA task
FMemory::Free(Ch);
}
return Success;
}
void SVisualLogger::HandleSaveCommandExecute()
{
TArray<TSharedPtr<class STimeline> > OutTimelines;
MainView->GetTimelines(OutTimelines, true);
if (OutTimelines.Num() == 0)
{
MainView->GetTimelines(OutTimelines);
}
if (OutTimelines.Num())
{
// Prompt the user for the filenames
TArray<FString> SaveFilenames;
IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get();
bool bSaved = false;
if (DesktopPlatform)
{
void* ParentWindowWindowHandle = NULL;
IMainFrameModule& MainFrameModule = FModuleManager::LoadModuleChecked<IMainFrameModule>(TEXT("MainFrame"));
const TSharedPtr<SWindow>& MainFrameParentWindow = MainFrameModule.GetParentWindow();
if (MainFrameParentWindow.IsValid() && MainFrameParentWindow->GetNativeWindow().IsValid())
{
ParentWindowWindowHandle = MainFrameParentWindow->GetNativeWindow()->GetOSWindowHandle();
}
const FString DefaultBrowsePath = FString::Printf(TEXT("%slogs/"), *FPaths::GameSavedDir());
bSaved = DesktopPlatform->SaveFileDialog(
ParentWindowWindowHandle,
LOCTEXT("NewProjectBrowseTitle", "Choose a project location").ToString(),
DefaultBrowsePath,
TEXT(""),
LogVisualizer::SaveFileTypes,
EFileDialogFlags::None,
SaveFilenames
);
}
if (bSaved)
{
if (SaveFilenames.Num() > 0)
{
TArray<FVisualLogDevice::FVisualLogEntryItem> FrameCache;
for (auto CurrentItem : OutTimelines)
{
FrameCache.Append(CurrentItem->GetEntries());
}
if (FrameCache.Num())
{
FArchive* FileArchive = IFileManager::Get().CreateFileWriter(*SaveFilenames[0]);
FVisualLoggerHelpers::Serialize(*FileArchive, FrameCache);
FileArchive->Close();
delete FileArchive;
FileArchive = NULL;
}
}
}
}
}
bool UParticleSystemAuditCommandlet::DumpSimpleSet(TSet<FString>& InSet, const TCHAR* InShortFilename, const TCHAR* InObjectClassName)
{
if (InSet.Num() > 0)
{
check(InShortFilename != NULL);
check(InObjectClassName != NULL);
FArchive* OutputStream = GetOutputFile(InShortFilename);
if (OutputStream != NULL)
{
UE_LOG(LogParticleSystemAuditCommandlet, Log, TEXT("Dumping '%s' results..."), InShortFilename);
OutputStream->Logf(TEXT("%s,..."), InObjectClassName);
for (TSet<FString>::TIterator DumpIt(InSet); DumpIt; ++DumpIt)
{
FString ObjName = *DumpIt;
OutputStream->Logf(TEXT("%s"), *ObjName);
}
OutputStream->Close();
delete OutputStream;
}
else
{
return false;
}
}
return true;
}
bool FBuildPatchAppManifest::SaveToFile(const FString& Filename, bool bUseBinary)
{
bool bSuccess = false;
FArchive* FileOut = IFileManager::Get().CreateFileWriter(*Filename);
if (FileOut)
{
if (bUseBinary)
{
Data->ManifestFileVersion = EBuildPatchAppManifestVersion::GetLatestVersion();
FManifestWriter ManifestData;
Serialize(ManifestData);
ManifestData.Finalize();
if (!ManifestData.IsError())
{
int32 DataSize = ManifestData.TotalSize();
TArray<uint8> TempCompressed;
TempCompressed.AddUninitialized(DataSize);
int32 CompressedSize = DataSize;
bool bDataIsCompressed = FCompression::CompressMemory(
static_cast<ECompressionFlags>(COMPRESS_ZLIB | COMPRESS_BiasMemory),
TempCompressed.GetData(),
CompressedSize,
ManifestData.GetBytes().GetData(),
DataSize);
TempCompressed.SetNum(CompressedSize);
TArray<uint8>& FileData = bDataIsCompressed ? TempCompressed : ManifestData.GetBytes();
FManifestFileHeader Header;
*FileOut << Header;
Header.HeaderSize = FileOut->Tell();
Header.StoredAs = bDataIsCompressed ? EManifestFileHeader::STORED_COMPRESSED : EManifestFileHeader::STORED_RAW;
Header.DataSize = DataSize;
Header.CompressedSize = bDataIsCompressed ? CompressedSize : 0;
FSHA1::HashBuffer(FileData.GetData(), FileData.Num(), Header.SHAHash.Hash);
FileOut->Seek(0);
*FileOut << Header;
FileOut->Serialize(FileData.GetData(), FileData.Num());
bSuccess = !FileOut->IsError();
}
}
else
{
Data->ManifestFileVersion = EBuildPatchAppManifestVersion::GetLatestJsonVersion();
FString JSONOutput;
SerializeToJSON(JSONOutput);
FTCHARToUTF8 JsonUTF8(*JSONOutput);
FileOut->Serialize((UTF8CHAR*)JsonUTF8.Get(), JsonUTF8.Length() * sizeof(UTF8CHAR));
}
FileOut->Close();
delete FileOut;
FileOut = nullptr;
}
return bSuccess;
}
/**
* Write all the data mined from the minidump to a text file
*/
void FLinuxCrashContext::GenerateReport(const FString & DiagnosticsPath) const
{
FArchive* ReportFile = IFileManager::Get().CreateFileWriter(*DiagnosticsPath);
if (ReportFile != NULL)
{
FString Line;
WriteLine(ReportFile, "Generating report for minidump");
WriteLine(ReportFile);
Line = FString::Printf(TEXT("Application version %d.%d.%d.0" ), FEngineVersion::Current().GetMajor(), FEngineVersion::Current().GetMinor(), FEngineVersion::Current().GetPatch());
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
Line = FString::Printf(TEXT(" ... built from changelist %d"), FEngineVersion::Current().GetChangelist());
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
WriteLine(ReportFile);
utsname UnixName;
if (uname(&UnixName) == 0)
{
Line = FString::Printf(TEXT( "OS version %s %s (network name: %s)" ), ANSI_TO_TCHAR(UnixName.sysname), ANSI_TO_TCHAR(UnixName.release), ANSI_TO_TCHAR(UnixName.nodename));
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
Line = FString::Printf( TEXT( "Running %d %s processors (%d logical cores)" ), FPlatformMisc::NumberOfCores(), ANSI_TO_TCHAR(UnixName.machine), FPlatformMisc::NumberOfCoresIncludingHyperthreads());
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
}
else
{
Line = FString::Printf(TEXT("OS version could not be determined (%d, %s)"), errno, ANSI_TO_TCHAR(strerror(errno)));
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
Line = FString::Printf( TEXT( "Running %d unknown processors" ), FPlatformMisc::NumberOfCores());
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
}
Line = FString::Printf(TEXT("Exception was \"%s\""), SignalDescription);
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
WriteLine(ReportFile);
WriteLine(ReportFile, "<SOURCE START>");
WriteLine(ReportFile, "<SOURCE END>");
WriteLine(ReportFile);
WriteLine(ReportFile, "<CALLSTACK START>");
WriteLine(ReportFile, MinidumpCallstackInfo);
WriteLine(ReportFile, "<CALLSTACK END>");
WriteLine(ReportFile);
WriteLine(ReportFile, "0 loaded modules");
WriteLine(ReportFile);
Line = FString::Printf(TEXT("Report end!"));
WriteLine(ReportFile, TCHAR_TO_UTF8(*Line));
ReportFile->Close();
delete ReportFile;
}
}
void SVisualLogger::HandleLoadCommandExecute()
{
FArchive Ar;
TArray<FVisualLogDevice::FVisualLogEntryItem> RecordedLogs;
TArray<FString> OpenFilenames;
IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get();
bool bOpened = false;
if (DesktopPlatform)
{
void* ParentWindowWindowHandle = NULL;
IMainFrameModule& MainFrameModule = FModuleManager::LoadModuleChecked<IMainFrameModule>(TEXT("MainFrame"));
const TSharedPtr<SWindow>& MainFrameParentWindow = MainFrameModule.GetParentWindow();
if (MainFrameParentWindow.IsValid() && MainFrameParentWindow->GetNativeWindow().IsValid())
{
ParentWindowWindowHandle = MainFrameParentWindow->GetNativeWindow()->GetOSWindowHandle();
}
const FString DefaultBrowsePath = FString::Printf(TEXT("%slogs/"), *FPaths::GameSavedDir());
bOpened = DesktopPlatform->OpenFileDialog(
ParentWindowWindowHandle,
LOCTEXT("OpenProjectBrowseTitle", "Open Project").ToString(),
DefaultBrowsePath,
TEXT(""),
LogVisualizer::LoadFileTypes,
EFileDialogFlags::None,
OpenFilenames
);
}
if (bOpened && OpenFilenames.Num() > 0)
{
OnNewWorld(nullptr);
for (int FilenameIndex = 0; FilenameIndex < OpenFilenames.Num(); ++FilenameIndex)
{
FString CurrentFileName = OpenFilenames[FilenameIndex];
const bool bIsBinaryFile = CurrentFileName.Find(TEXT(".bvlog")) != INDEX_NONE;
if (bIsBinaryFile)
{
FArchive* FileAr = IFileManager::Get().CreateFileReader(*CurrentFileName);
FVisualLoggerHelpers::Serialize(*FileAr, RecordedLogs);
FileAr->Close();
delete FileAr;
FileAr = NULL;
for (FVisualLogDevice::FVisualLogEntryItem& CurrentItem : RecordedLogs)
{
OnNewLogEntry(CurrentItem);
}
}
}
}
}
bool UJavascriptLibrary::WriteFile(UObject* Object, FString Filename)
{
FArchive* Writer = IFileManager::Get().CreateFileWriter(*Filename);
if (!Writer)
{
return false;
}
Writer->Serialize(FArrayBufferAccessor::GetData(), FArrayBufferAccessor::GetSize());
return Writer->Close();
}
uint8 FBuildPatchUtils::VerifyFile(const FString& FileToVerify, const FSHAHashData& Hash1, const FSHAHashData& Hash2, FBuildPatchFloatDelegate ProgressDelegate, FBuildPatchBoolRetDelegate ShouldPauseDelegate, double& TimeSpentPaused)
{
uint8 ReturnValue = 0;
FArchive* FileReader = IFileManager::Get().CreateFileReader(*FileToVerify);
ProgressDelegate.ExecuteIfBound(0.0f);
if (FileReader != NULL)
{
FSHA1 HashState;
FSHAHashData HashValue;
const int64 FileSize = FileReader->TotalSize();
uint8* FileReadBuffer = new uint8[FileBufferSize];
while (!FileReader->AtEnd() && !FBuildPatchInstallError::HasFatalError())
{
// Pause if necessary
const double PrePauseTime = FPlatformTime::Seconds();
double PostPauseTime = PrePauseTime;
bool bShouldPause = ShouldPauseDelegate.IsBound() && ShouldPauseDelegate.Execute();
while (bShouldPause && !FBuildPatchInstallError::HasFatalError())
{
FPlatformProcess::Sleep(0.1f);
bShouldPause = ShouldPauseDelegate.Execute();
PostPauseTime = FPlatformTime::Seconds();
}
// Count up pause time
TimeSpentPaused += PostPauseTime - PrePauseTime;
// Read file and update hash state
const int64 SizeLeft = FileSize - FileReader->Tell();
const uint32 ReadLen = FMath::Min< int64 >(FileBufferSize, SizeLeft);
FileReader->Serialize(FileReadBuffer, ReadLen);
HashState.Update(FileReadBuffer, ReadLen);
const double FileSizeTemp = FileSize;
const float Progress = 1.0f - ((SizeLeft - ReadLen) / FileSizeTemp);
ProgressDelegate.ExecuteIfBound(Progress);
}
delete[] FileReadBuffer;
HashState.Final();
HashState.GetHash(HashValue.Hash);
ReturnValue = (HashValue == Hash1) ? 1 : (HashValue == Hash2) ? 2 : 0;
if (ReturnValue == 0)
{
GLog->Logf(TEXT("BuildDataGenerator: Verify failed on %s"), *FPaths::GetCleanFilename(FileToVerify));
}
FileReader->Close();
delete FileReader;
}
else
{
GLog->Logf(TEXT("BuildDataGenerator: ERROR VerifyFile cannot open %s"), *FileToVerify);
}
ProgressDelegate.ExecuteIfBound(1.0f);
return ReturnValue;
}
/**
* Creates (fake so far) minidump
*/
void GenerateMinidump(const FString & Path)
{
FArchive* ReportFile = IFileManager::Get().CreateFileWriter(*Path);
if (ReportFile != NULL)
{
// write BOM
static uint32 Garbage = 0xDEADBEEF;
ReportFile->Serialize(&Garbage, sizeof(Garbage));
ReportFile->Close();
delete ReportFile;
}
}
void G_UnSnapshotLevel (bool hubLoad)
{
if (level.info->snapshot == NULL)
return;
if (level.info->isValid())
{
SaveVersion = level.info->snapshotVer;
level.info->snapshot->Reopen ();
FArchive arc (*level.info->snapshot);
if (hubLoad)
arc.SetHubTravel ();
G_SerializeLevel (arc, hubLoad);
arc.Close ();
level.FromSnapshot = true;
TThinkerIterator<APlayerPawn> it;
APlayerPawn *pawn, *next;
next = it.Next();
while ((pawn = next) != 0)
{
next = it.Next();
if (pawn->player == NULL || pawn->player->mo == NULL || !playeringame[pawn->player - players])
{
int i;
// If this isn't the unmorphed original copy of a player, destroy it, because it's extra.
for (i = 0; i < MAXPLAYERS; ++i)
{
if (playeringame[i] && players[i].morphTics && players[i].mo->tracer == pawn)
{
break;
}
}
if (i == MAXPLAYERS)
{
pawn->Destroy ();
}
}
}
}
// No reason to keep the snapshot around once the level's been entered.
level.info->ClearSnapshot();
if (hubLoad)
{
// Unlock ACS global strings that were locked when the snapshot was made.
FBehavior::StaticUnlockLevelVarStrings();
}
}
void FDesktopPlatformLinux::EnumerateEngineInstallations(TMap<FString, FString> &OutInstallations)
{
EnumerateLauncherEngineInstallations(OutInstallations);
FString UProjectPath = FString(FPlatformProcess::ApplicationSettingsDir()) / "Unreal.uproject";
FArchive* File = IFileManager::Get().CreateFileWriter(*UProjectPath, FILEWRITE_EvenIfReadOnly);
if (File)
{
File->Close();
delete File;
}
else
{
SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "Unable to write to Settings Directory", TCHAR_TO_UTF8(*UProjectPath), NULL);
}
FConfigFile ConfigFile;
FString ConfigPath = FString(FPlatformProcess::ApplicationSettingsDir()) / FString(TEXT("UnrealEngine")) / FString(TEXT("Install.ini"));
ConfigFile.Read(ConfigPath);
FConfigSection &Section = ConfigFile.FindOrAdd(TEXT("Installations"));
// @todo: currently we can enumerate only this installation
FString EngineDir = FPaths::EngineDir();
FString EngineId;
const FName* Key = Section.FindKey(EngineDir);
if (Key)
{
EngineId = Key->ToString();
}
else
{
if (!OutInstallations.FindKey(EngineDir))
{
EngineId = FGuid::NewGuid().ToString(EGuidFormats::DigitsWithHyphens);
Section.AddUnique(*EngineId, EngineDir);
ConfigFile.Dirty = true;
}
}
if (!EngineId.IsEmpty() && !OutInstallations.Find(EngineId))
{
OutInstallations.Add(EngineId, EngineDir);
}
ConfigFile.Write(ConfigPath);
IFileManager::Get().Delete(*UProjectPath);
}
bool FRuntimeAssetCacheBackend::PutCachedData(const FName Bucket, const TCHAR* CacheKey, TArray<uint8>& InData, FCacheEntryMetadata* Metadata)
{
bool bResult = false;
FArchive* Ar = CreateWriteArchive(Bucket, CacheKey);
if (!Ar)
{
return bResult;
}
*Ar << *Metadata;
Ar->Serialize(InData.GetData(), InData.Num());
bResult = Ar->Close();
delete Ar;
return bResult;
}
bool UJavascriptLibrary::ReadFile(UObject* Object, FString Filename)
{
FArchive* Reader = IFileManager::Get().CreateFileReader(*Filename);
if (!Reader)
{
return false;
}
int32 Size = Reader->TotalSize();
if (Size != FArrayBufferAccessor::GetSize())
{
return false;
}
Reader->Serialize(FArrayBufferAccessor::GetData(), Size);
return Reader->Close();
}
bool FTextLocalizationManager::FLocalizationEntryTracker::ReadFromFile(const FString& FilePath)
{
FArchive* Reader = IFileManager::Get().CreateFileReader( *FilePath );
if( !Reader )
{
return false;
}
Reader->SetForceUnicode(true);
ReadFromArchive(*Reader, FilePath);
bool Success = Reader->Close();
delete Reader;
return Success;
}
/**
* Load a binary file to a dynamic array.
*/
bool FFileHelper::LoadFileToArray( TArray<uint8>& Result, const TCHAR* Filename, uint32 Flags )
{
FArchive* Reader = IFileManager::Get().CreateFileReader( Filename, Flags );
if( !Reader )
{
if (!(Flags & FILEREAD_Silent))
{
UE_LOG(LogStreaming,Warning,TEXT("Failed to read file '%s' error."),Filename);
}
return 0;
}
Result.Reset();
Result.AddUninitialized( Reader->TotalSize() );
Reader->Serialize(Result.GetData(), Result.Num());
bool Success = Reader->Close();
delete Reader;
return Success;
}
bool FPerfCounters::ReportUnplayableCondition(const FString& ConditionDescription)
{
FString UnplayableConditionFile(FPaths::Combine(*FPaths::GameSavedDir(), *FString::Printf(TEXT("UnplayableConditionForPid_%d.txt"), FPlatformProcess::GetCurrentProcessId())));
FArchive* ReportFile = IFileManager::Get().CreateFileWriter(*UnplayableConditionFile);
if (UNLIKELY(ReportFile == nullptr))
{
return false;
}
// include description for debugging
FTCHARToUTF8 Converter(*FString::Printf(TEXT("Unplayable condition encountered: %s\n"), *ConditionDescription));
ReportFile->Serialize(reinterpret_cast<void *>(const_cast<char *>(Converter.Get())), Converter.Length());
ReportFile->Close();
delete ReportFile;
return true;
}
FCacheEntryMetadata* FRuntimeAssetCacheBackend::GetCachedData(const FName Bucket, const TCHAR* CacheKey, TArray<uint8>& OutData)
{
FCacheEntryMetadata* Result = nullptr;
FArchive* Ar = CreateReadArchive(Bucket, CacheKey);
if (!Ar)
{
return Result;
}
Result = PreloadMetadata(Ar);
int64 TotalSize = Ar->TotalSize();
int64 CurrentPosition = Ar->Tell();
int64 NumberOfBytesToSerialize = TotalSize - CurrentPosition;
OutData.Reset();
OutData.AddUninitialized(NumberOfBytesToSerialize);
Ar->Serialize(OutData.GetData(), NumberOfBytesToSerialize);
Ar->Close();
delete Ar;
return Result;
}
virtual void UpdateADBPath() override
{
FScopeLock PathUpdateLock(&ADBPathCheckLock);
TCHAR AndroidDirectory[32768] = { 0 };
FPlatformMisc::GetEnvironmentVariable(TEXT("ANDROID_HOME"), AndroidDirectory, 32768);
FString ADBPath;
#if PLATFORM_MAC
if (AndroidDirectory[0] == 0)
{
// didn't find ANDROID_HOME, so parse the .bash_profile file on MAC
FArchive* FileReader = IFileManager::Get().CreateFileReader(*FString([@"~/.bash_profile" stringByExpandingTildeInPath]));
if (FileReader)
{
const int64 FileSize = FileReader->TotalSize();
ANSICHAR* AnsiContents = (ANSICHAR*)FMemory::Malloc(FileSize);
FileReader->Serialize(AnsiContents, FileSize);
FileReader->Close();
delete FileReader;
TArray<FString> Lines;
FString(ANSI_TO_TCHAR(AnsiContents)).ParseIntoArrayLines(&Lines);
FMemory::Free(AnsiContents);
for (int32 Index = 0; Index < Lines.Num(); Index++)
{
if (AndroidDirectory[0] == 0 && Lines[Index].StartsWith(TEXT("export ANDROID_HOME=")))
{
FString Directory;
Lines[Index].Split(TEXT("="), NULL, &Directory);
Directory = Directory.Replace(TEXT("\""), TEXT(""));
FCString::Strcpy(AndroidDirectory, *Directory);
setenv("ANDROID_HOME", TCHAR_TO_ANSI(AndroidDirectory), 1);
}
}
}
}
int32 UFixupRedirectsCommandlet::Main( const FString& Params )
{
// Retrieve list of all packages in .ini paths.
TArray<FString> PackageList;
FEditorFileUtils::FindAllPackageFiles(PackageList);
if( !PackageList.Num() )
{
return 0;
}
// process the commandline
FString Token;
const TCHAR* CommandLine = *Params;
bool bIsQuietMode = false;
bool bIsTestOnly = false;
bool bShouldRestoreProgress= false;
bool bIsSCCDisabled = false;
bool bUpdateEnginePackages = false;
bool bDeleteRedirects = true;
bool bDeleteOnlyReferencedRedirects = false;
bool bAutoSubmit = false;
bool bSandboxed = IFileManager::Get().IsSandboxEnabled();
while (FParse::Token(CommandLine, Token, 1))
{
if (Token == TEXT("-nowarn"))
{
bIsQuietMode = true;
}
else if (Token == TEXT("-testonly"))
{
bIsTestOnly = true;
}
else if (Token == TEXT("-restore"))
{
bShouldRestoreProgress = true;
}
else if (Token == TEXT("-NoAutoCheckout"))
{
bIsSCCDisabled = true;
}
else if (Token == TEXT("-AutoSubmit"))
{
bAutoSubmit = true;
}
else if (Token == TEXT("-UpdateEnginePackages"))
{
bUpdateEnginePackages = true;
}
else if (Token == TEXT("-NoDelete"))
{
bDeleteRedirects = false;
}
else if (Token == TEXT("-NoDeleteUnreferenced"))
{
bDeleteOnlyReferencedRedirects = true;
}
else if ( Token.Left(1) != TEXT("-") )
{
FPackageName::SearchForPackageOnDisk(Token, &GRedirectCollector.FileToFixup);
}
}
if (bSandboxed)
{
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Running in a sandbox."));
bIsSCCDisabled = true;
bAutoSubmit = false;
bDeleteRedirects = false;
}
bool bShouldSkipErrors = false;
// Setup file Output log in the Saved Directory
const FString ProjectDir = FPaths::GetPath(FPaths::GetProjectFilePath());
const FString LogFilename = FPaths::Combine(*ProjectDir, TEXT("Saved"), TEXT("FixupRedirect"), TEXT("FixupRedirect.log"));
FArchive* LogOutputFile = IFileManager::Get().CreateFileWriter(*LogFilename);
LogOutputFile->Logf(TEXT("[Redirects]"));
/////////////////////////////////////////////////////////////////////
// Display any script code referenced redirects
/////////////////////////////////////////////////////////////////////
TArray<FString> UpdatePackages;
TArray<FString> RedirectorsThatCantBeCleaned;
if (!bShouldRestoreProgress)
{
// load all string asset reference targets, and add fake redirectors for them
GRedirectCollector.ResolveStringAssetReference();
for (int32 RedirIndex = 0; RedirIndex < GRedirectCollector.Redirections.Num(); RedirIndex++)
{
FRedirection& Redir = GRedirectCollector.Redirections[RedirIndex];
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Boot redirector can't be cleaned: %s(%s) -> %s(%s)"), *Redir.RedirectorName, *Redir.RedirectorPackageFilename, *Redir.DestinationObjectName, *Redir.PackageFilename);
RedirectorsThatCantBeCleaned.AddUnique(Redir.RedirectorName);
}
/////////////////////////////////////////////////////////////////////
// Build a list of packages that need to be updated
/////////////////////////////////////////////////////////////////////
SET_WARN_COLOR(COLOR_WHITE);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Generating list of tasks:"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("-------------------------"));
CLEAR_WARN_COLOR();
int32 GCIndex = 0;
// process script code first pass, then everything else
for( int32 PackageIndex = 0; PackageIndex < PackageList.Num(); PackageIndex++ )
{
const FString& Filename = PackageList[PackageIndex];
// already loaded code, skip them, and skip autosave packages
if (Filename.StartsWith(AutoSaveUtils::GetAutoSaveDir(), ESearchCase::IgnoreCase))
{
continue;
}
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Looking for redirects in %s..."), *Filename);
// Assert if package couldn't be opened so we have no chance of messing up saving later packages.
UPackage* Package = Cast<UPackage>(LoadPackage(NULL, *Filename, 0));
// load all string asset reference targets, and add fake redirectors for them
GRedirectCollector.ResolveStringAssetReference();
if (!Package)
{
SET_WARN_COLOR(COLOR_RED);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... failed to open %s"), *Filename);
CLEAR_WARN_COLOR();
// if we are not in quiet mode, and the user wants to continue, go on
const FText Message = FText::Format( NSLOCTEXT("Unreal", "PackageFailedToOpen", "The package '{0}' failed to open. Should this commandlet continue operation, ignoring further load errors? Continuing may have adverse effects (object references may be lost)."), FText::FromString( Filename ) );
if (bShouldSkipErrors || (!bIsQuietMode && GWarn->YesNof( Message )))
{
bShouldSkipErrors = true;
continue;
}
else
{
return 1;
}
}
// all notices here about redirects get this color
SET_WARN_COLOR(COLOR_DARK_GREEN);
// look for any redirects that were followed that are referenced by this package
// (may have been loaded earlier if this package was loaded by script code)
// any redirectors referenced by script code can't be cleaned up
for (int32 RedirIndex = 0; RedirIndex < GRedirectCollector.Redirections.Num(); RedirIndex++)
{
FRedirection& Redir = GRedirectCollector.Redirections[RedirIndex];
if (Redir.PackageFilename == Filename)
{
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... references %s [-> %s]"), *Redir.RedirectorName, *Redir.DestinationObjectName);
// put the source and dest packages in the list to be updated
UpdatePackages.AddUnique(Redir.PackageFilename);
UpdatePackages.AddUnique(Redir.RedirectorPackageFilename);
}
}
// clear the flag for needing to collect garbage
bool bNeedToCollectGarbage = false;
// if this package has any redirectors, make sure we update it
if (!GRedirectCollector.FileToFixup.Len() || GRedirectCollector.FileToFixup == FPackageName::FilenameToLongPackageName(Filename))
{
TArray<UObject *> ObjectsInOuter;
GetObjectsWithOuter(Package, ObjectsInOuter);
for( int32 Index = 0; Index < ObjectsInOuter.Num(); Index++ )
{
UObject* Obj = ObjectsInOuter[Index];
UObjectRedirector* Redir = Cast<UObjectRedirector>(Obj);
if (Redir)
{
// make sure this package is in the list of packages to update
UpdatePackages.AddUnique(Filename);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... has redirect %s [-> %s]"), *Redir->GetPathName(), *Redir->DestinationObject->GetFullName());
LogOutputFile->Logf(TEXT("%s = %s"), *Redir->GetPathName(), *Redir->DestinationObject->GetFullName());
// make sure we GC if we found a redirector
bNeedToCollectGarbage = true;
}
}
}
CLEAR_WARN_COLOR();
// collect garbage every N packages, or if there was any redirectors in the package
// (to make sure that redirectors get reloaded properly and followed by the callback)
// also, GC after loading a map package, to make sure it's unloaded so that we can track
// other packages loading a map package as a dependency
if (((++GCIndex) % 50) == 0 || bNeedToCollectGarbage || Package->ContainsMap())
{
// collect garbage to close the package
CollectGarbage(RF_Native);
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("GC..."));
// reset our counter
GCIndex = 0;
}
}
// save off restore point
FArchive* Ar = IFileManager::Get().CreateFileWriter(*(FPaths::GameSavedDir() + TEXT("Fixup.bin")));
*Ar << GRedirectCollector.Redirections;
*Ar << UpdatePackages;
*Ar << RedirectorsThatCantBeCleaned;
delete Ar;
}
else
{
// load restore point
FArchive* Ar = IFileManager::Get().CreateFileReader(*(FPaths::GameSavedDir() + TEXT("Fixup.bin")));
if( Ar != NULL )
{
*Ar << GRedirectCollector.Redirections;
*Ar << UpdatePackages;
*Ar << RedirectorsThatCantBeCleaned;
delete Ar;
}
}
// unregister the callback so we stop getting redirections added
FCoreUObjectDelegates::RedirectorFollowed.RemoveAll(&GRedirectCollector);
/////////////////////////////////////////////////////////////////////
// Explain to user what is happening
/////////////////////////////////////////////////////////////////////
SET_WARN_COLOR(COLOR_YELLOW);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Files that need to fixed up:"));
SET_WARN_COLOR(COLOR_DARK_YELLOW);
// print out the working set of packages
for (int32 PackageIndex = 0; PackageIndex < UpdatePackages.Num(); PackageIndex++)
{
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" %s"), *UpdatePackages[PackageIndex]);
}
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
CLEAR_WARN_COLOR();
// if we are only testing, just just quiet before actually doing anything
if (bIsTestOnly)
{
LogOutputFile->Close();
delete LogOutputFile;
LogOutputFile = NULL;
return 0;
}
/////////////////////////////////////////////////////////////////////
// Find redirectors that are referenced by packages we cant check out
/////////////////////////////////////////////////////////////////////
bool bEngineRedirectorCantBeCleaned = false;
ISourceControlProvider& SourceControlProvider = ISourceControlModule::Get().GetProvider();
// initialize source control if it hasn't been initialized yet
if(!bIsSCCDisabled)
{
SourceControlProvider.Init();
//Make sure we can check out all packages - update their state first
SourceControlProvider.Execute(ISourceControlOperation::Create<FUpdateStatus>(), UpdatePackages);
}
for( int32 PackageIndex = 0; PackageIndex < UpdatePackages.Num(); PackageIndex++ )
{
const FString& Filename = UpdatePackages[PackageIndex];
bool bCanEdit = true;
if(!bIsSCCDisabled)
{
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(Filename, EStateCacheUsage::ForceUpdate);
if(SourceControlState.IsValid() && !SourceControlState->CanEdit())
{
bCanEdit = false;
}
}
else if(IFileManager::Get().IsReadOnly(*Filename))
{
bCanEdit = false;
}
if(!bCanEdit)
{
const bool bAllowCheckout = bUpdateEnginePackages || !Filename.StartsWith( FPaths::EngineDir() );
if (!bIsSCCDisabled && bAllowCheckout)
{
FString PackageName(FPackageName::FilenameToLongPackageName(Filename));
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(Filename, EStateCacheUsage::ForceUpdate);
if (SourceControlState.IsValid() && SourceControlState->CanCheckout())
{
SourceControlProvider.Execute(ISourceControlOperation::Create<FCheckOut>(), SourceControlHelpers::PackageFilename(PackageName));
FSourceControlStatePtr NewSourceControlState = SourceControlProvider.GetState(Filename, EStateCacheUsage::ForceUpdate);
bCanEdit = NewSourceControlState.IsValid() && NewSourceControlState->CanEdit();
}
}
// if the checkout failed for any reason, we can't clean it up
if (bAllowCheckout && !bCanEdit)
{
SET_WARN_COLOR(COLOR_RED);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Couldn't check out/edit %s..."), *Filename);
CLEAR_WARN_COLOR();
// load all string asset reference targets, and add fake redirectors for them
GRedirectCollector.ResolveStringAssetReference();
// any redirectors that are pointed to by this read-only package can't be fixed up
for (int32 RedirIndex = 0; RedirIndex < GRedirectCollector.Redirections.Num(); RedirIndex++)
{
FRedirection& Redir = GRedirectCollector.Redirections[RedirIndex];
// any redirectors pointed to by this file we don't clean
if (Redir.PackageFilename == Filename)
{
RedirectorsThatCantBeCleaned.AddUnique(Redir.RedirectorName);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... can't fixup references to %s"), *Redir.RedirectorName);
// if this redirector is in an Engine package, we need to alert the user, because
// Engine packages are shared between games, so if one game still needs the redirector
// to be valid, then we can't check in Engine packages after another game has
// run that could
if (!bEngineRedirectorCantBeCleaned)
{
FString PackagePath;
FPackageName::DoesPackageExist(Redir.RedirectorPackageFilename, NULL, &PackagePath);
if (PackagePath.StartsWith(FPaths::EngineDir()))
{
bEngineRedirectorCantBeCleaned = true;
}
}
}
// any redirectors in this file can't be deleted
else if (Redir.RedirectorPackageFilename == Filename)
{
RedirectorsThatCantBeCleaned.AddUnique(Redir.RedirectorName);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... can't delete %s"), *Redir.RedirectorName);
}
}
}
else
{
SET_WARN_COLOR(COLOR_DARK_GREEN);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Checked out %s..."), *Filename);
CLEAR_WARN_COLOR();
}
}
}
// warn about an engine package that can't be cleaned up (may want to revert engine packages)
if (bEngineRedirectorCantBeCleaned)
{
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Engine package redirectors are referenced by packages that can't be cleaned. If you have multiple games, it's recommended you revert any checked out Engine packages."));
bUpdateEnginePackages=false;
}
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
/////////////////////////////////////////////////////////////////////
// Load and save packages to save out the proper fixed up references
/////////////////////////////////////////////////////////////////////
SET_WARN_COLOR(COLOR_WHITE);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Fixing references to point to proper objects:"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("---------------------------------------------"));
CLEAR_WARN_COLOR();
// keep a list of all packages that have ObjectRedirects in them. This is not needed if we aren't deleting redirects
TArray<bool> PackagesWithRedirects;
// Delete these packages entirely because they are empty
TArray<bool> EmptyPackagesToDelete;
if ( bDeleteRedirects )
{
PackagesWithRedirects.AddZeroed(UpdatePackages.Num());
EmptyPackagesToDelete.AddZeroed(PackageList.Num());
}
bool bSCCEnabled = ISourceControlModule::Get().IsEnabled();
// Iterate over all packages, loading and saving to remove all references to ObjectRedirectors (can't delete the Redirects yet)
for( int32 PackageIndex = 0; PackageIndex < UpdatePackages.Num(); PackageIndex++ )
{
const FString& Filename = UpdatePackages[PackageIndex];
// we can't do anything with packages that are read-only or cant be edited due to source control (we don't want to fixup the redirects)
if(bSCCEnabled)
{
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(SourceControlHelpers::PackageFilename(Filename), EStateCacheUsage::ForceUpdate);
if(SourceControlState.IsValid() && !SourceControlState->CanEdit())
{
continue;
}
}
else if(IFileManager::Get().IsReadOnly(*Filename))
{
continue;
}
// Assert if package couldn't be opened so we have no chance of messing up saving later packages.
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Cleaning %s"), *Filename);
UPackage* Package = LoadPackage( NULL, *Filename, LOAD_NoVerify );
// load all string asset reference targets, and add fake redirectors for them
GRedirectCollector.ResolveStringAssetReference();
// if it failed to open, we have already dealt with quitting if we are going to, so just skip it
if (!Package)
{
continue;
}
if ( bDeleteOnlyReferencedRedirects && bDeleteRedirects )
{
TArray<UObject *> ObjectsInOuter;
GetObjectsWithOuter(Package, ObjectsInOuter);
for( int32 Index = 0; Index < ObjectsInOuter.Num(); Index++ )
{
UObject* Obj = ObjectsInOuter[Index];
UObjectRedirector* Redir = Cast<UObjectRedirector>(Obj);
if (Redir)
{
PackagesWithRedirects[PackageIndex] = 1;
break;
}
}
}
// Don't save packages in the engine directory if we did not opt to update engine packages
if( bUpdateEnginePackages || !Filename.StartsWith( FPaths::EngineDir() ) )
{
// save out the package
UWorld* World = UWorld::FindWorldInPackage(Package);
GEditor->SavePackage( Package, World, World ? RF_NoFlags : RF_Standalone, *Filename, GWarn );
}
// collect garbage to close the package
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Collecting Garbage..."));
CollectGarbage(RF_Native);
}
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
if ( bDeleteRedirects )
{
/////////////////////////////////////////////////////////////////////
// Delete all redirects that are no longer referenced
/////////////////////////////////////////////////////////////////////
SET_WARN_COLOR(COLOR_WHITE);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Deleting redirector objects:"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("----------------------------"));
CLEAR_WARN_COLOR();
const TArray<FString>& PackagesToCleaseOfRedirects = bDeleteOnlyReferencedRedirects ? UpdatePackages : PackageList;
int32 GCIndex = 0;
// Again, iterate over all packages, loading and saving to and this time deleting all
for( int32 PackageIndex = 0; PackageIndex < PackagesToCleaseOfRedirects.Num(); PackageIndex++ )
{
const FString& Filename = PackagesToCleaseOfRedirects[PackageIndex];
if (bDeleteOnlyReferencedRedirects && PackagesWithRedirects[PackageIndex] == 0)
{
continue;
}
// The file should have already been checked out/be writable
// If it is read only that means the checkout failed or we are not using source control and we can not write this file.
if(bSCCEnabled)
{
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(SourceControlHelpers::PackageFilename(Filename), EStateCacheUsage::ForceUpdate);
if(SourceControlState.IsValid() && !SourceControlState->CanEdit())
{
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Skipping file: source control says we cant edit the file %s..."), *Filename);
continue;
}
}
else if( IFileManager::Get().IsReadOnly( *Filename ))
{
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Skipping read-only file %s..."), *Filename);
continue;
}
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Wiping %s..."), *Filename);
UPackage* Package = LoadPackage( NULL, *Filename, 0 );
// load all string asset reference targets, and add fake redirectors for them
GRedirectCollector.ResolveStringAssetReference();
// if it failed to open, we have already dealt with quitting if we are going to, so just skip it
if (!Package)
{
continue;
}
// assume that all redirs in this file are still-referenced
bool bIsDirty = false;
// see if this package is completely empty other than purged redirectors and metadata
bool bIsEmpty = true;
// delete all ObjectRedirects now
TArray<UObjectRedirector*> Redirs;
// find all redirectors, put into an array so delete doesn't mess up the iterator
TArray<UObject *> ObjectsInOuter;
GetObjectsWithOuter(Package, ObjectsInOuter);
for( int32 Index = 0; Index < ObjectsInOuter.Num(); Index++ )
{
UObject* Obj = ObjectsInOuter[Index];
UObjectRedirector *Redir = Cast<UObjectRedirector>(Obj);
if (Redir)
{
int32 Dummy;
// if the redirector was marked as uncleanable, skip it
if (RedirectorsThatCantBeCleaned.Find(Redir->GetFullName(), Dummy))
{
SET_WARN_COLOR(COLOR_DARK_RED);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... skipping still-referenced %s"), *Redir->GetFullName());
CLEAR_WARN_COLOR();
bIsEmpty = false;
continue;
}
bIsDirty = true;
SET_WARN_COLOR(COLOR_DARK_GREEN);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... deleting %s"), *Redir->GetFullName());
CLEAR_WARN_COLOR();
Redirs.Add(Redir);
}
else if ( (!Obj->IsA(UMetaData::StaticClass()) && !Obj->IsA(UField::StaticClass()))
|| Obj->IsA(UUserDefinedEnum::StaticClass())) // UserDefinedEnum should not be deleted
{
// This package has a real object
bIsEmpty = false;
}
}
if (bIsEmpty && !bDeleteOnlyReferencedRedirects)
{
EmptyPackagesToDelete[PackageIndex] = 1;
}
// mark them for deletion.
for (int32 RedirIndex = 0; RedirIndex < Redirs.Num(); RedirIndex++)
{
UObjectRedirector* Redirector = Redirs[RedirIndex];
// Remove standalone flag and mark as pending kill.
Redirector->ClearFlags( RF_Standalone | RF_Public );
// We don't need redirectors in the root set, the references should already be fixed up
if ( Redirector->IsRooted() )
{
Redirector->RemoveFromRoot();
}
Redirector->MarkPendingKill();
}
Redirs.Empty();
if (bIsDirty)
{
// Don't save packages in the engine directory if we did not opt to update engine packages
if( bUpdateEnginePackages || !Filename.StartsWith( FPaths::EngineDir() ) )
{
// save the package
UWorld* World = UWorld::FindWorldInPackage(Package);
GEditor->SavePackage( Package, World, World ? RF_NoFlags : RF_Standalone, *Filename, GWarn );
}
}
// collect garbage every N packages, or if there was any redirectors in the package
if (((++GCIndex) % 50) == 0 || bIsDirty || Package->ContainsMap())
{
// collect garbage to close the package
CollectGarbage(RF_Native);
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("GC..."));
// reset our counter
GCIndex = 0;
}
}
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
}
else
{
SET_WARN_COLOR(COLOR_WHITE);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("----------------------------"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Not deleting any redirector objects:"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("----------------------------"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(""));
CLEAR_WARN_COLOR();
}
/////////////////////////////////////////////////////////////////////
// Clean up any unused trash
/////////////////////////////////////////////////////////////////////
SET_WARN_COLOR(COLOR_WHITE);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Deleting empty and unused packages:"));
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("---------------------------------------"));
CLEAR_WARN_COLOR();
LogOutputFile->Logf(TEXT(""));
LogOutputFile->Logf(TEXT("[Deleted]"));
// Iterate over packages, attempting to delete unreferenced packages
for( int32 PackageIndex = 0; PackageIndex < PackageList.Num(); PackageIndex++ )
{
bool bDelete = false;
const FString& Filename = PackageList[PackageIndex];
FString PackageName(FPackageName::FilenameToLongPackageName(Filename));
const bool bAllowDelete = bUpdateEnginePackages || !Filename.StartsWith( FPaths::EngineDir() );
if (bDeleteRedirects && !bDeleteOnlyReferencedRedirects && EmptyPackagesToDelete[PackageIndex] && bAllowDelete)
{
// this package is completely empty, delete it
bDelete = true;
}
if (bDelete == true)
{
struct Local
{
static bool DeleteFromDisk(const FString& InFilename, const FString& InMessage)
{
SET_WARN_COLOR(COLOR_GREEN);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("%s"), *InMessage);
if (!IFileManager::Get().Delete(*InFilename, false, true))
{
SET_WARN_COLOR(COLOR_RED);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT(" ... failed to delete from disk."), *InFilename);
CLEAR_WARN_COLOR();
return false;
}
CLEAR_WARN_COLOR();
return true;
}
};
if (!bIsSCCDisabled)
{
// get file SCC status
FString FileName = SourceControlHelpers::PackageFilename(PackageName);
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(FileName, EStateCacheUsage::ForceUpdate);
if(SourceControlState.IsValid() && (SourceControlState->IsCheckedOut() || SourceControlState->IsAdded()) )
{
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Revert '%s' from source control..."), *Filename);
SourceControlProvider.Execute(ISourceControlOperation::Create<FRevert>(), FileName);
SET_WARN_COLOR(COLOR_GREEN);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Deleting '%s' from source control..."), *Filename);
CLEAR_WARN_COLOR();
SourceControlProvider.Execute(ISourceControlOperation::Create<FDelete>(), FileName);
}
else if(SourceControlState.IsValid() && SourceControlState->CanCheckout())
{
SET_WARN_COLOR(COLOR_GREEN);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Deleting '%s' from source control..."), *Filename);
CLEAR_WARN_COLOR();
SourceControlProvider.Execute(ISourceControlOperation::Create<FDelete>(), FileName);
}
else if(SourceControlState.IsValid() && SourceControlState->IsCheckedOutOther())
{
SET_WARN_COLOR(COLOR_RED);
UE_LOG(LogFixupRedirectsCommandlet, Warning, TEXT("Couldn't delete '%s' from source control, someone has it checked out, skipping..."), *Filename);
CLEAR_WARN_COLOR();
}
else if(SourceControlState.IsValid() && !SourceControlState->IsSourceControlled())
{
if(Local::DeleteFromDisk(Filename, FString::Printf(TEXT("'%s' is not in source control, attempting to delete from disk..."), *Filename)))
{
LogOutputFile->Logf(*Filename);
}
}
else
{
if(Local::DeleteFromDisk(Filename, FString::Printf(TEXT("'%s' is in an unknown source control state, attempting to delete from disk..."), *Filename)))
{
LogOutputFile->Logf(*Filename);
}
}
}
else
{
if(Local::DeleteFromDisk(Filename, FString::Printf(TEXT("source control disabled while deleting '%s', attempting to delete from disk..."), *Filename)))
{
LogOutputFile->Logf(*Filename);
}
}
}
}
LogOutputFile->Close();
delete LogOutputFile;
LogOutputFile = NULL;
if (!bIsSCCDisabled && bAutoSubmit)
{
/////////////////////////////////////////////////////////////////////
// Submit the results to source control
/////////////////////////////////////////////////////////////////////
UE_LOG(LogFixupRedirectsCommandlet, Display, TEXT("Submiting the results to source control"));
// Work out the list of file to check in
TArray <FString> FilesToSubmit;
for( int32 PackageIndex = 0; PackageIndex < PackageList.Num(); PackageIndex++ )
{
const FString& Filename = PackageList[PackageIndex];
FString PackageName(FPackageName::FilenameToLongPackageName(Filename));
FSourceControlStatePtr SourceControlState = SourceControlProvider.GetState(SourceControlHelpers::PackageFilename(Filename), EStateCacheUsage::ForceUpdate);
if( SourceControlState.IsValid() && (SourceControlState->IsCheckedOut() || SourceControlState->IsAdded() || SourceControlState->IsDeleted()) )
{
// Only submit engine packages if we're requested to
if( bUpdateEnginePackages || !Filename.StartsWith( FPaths::EngineDir() ) )
{
FilesToSubmit.Add(*PackageName);
}
}
}
// Check in all changed files
const FText Description = NSLOCTEXT("FixupRedirectsCmdlet", "ChangelistDescription", "Fixed up Redirects");
TSharedRef<FCheckIn, ESPMode::ThreadSafe> CheckInOperation = ISourceControlOperation::Create<FCheckIn>();
CheckInOperation->SetDescription( Description );
SourceControlProvider.Execute(CheckInOperation, SourceControlHelpers::PackageFilenames(FilesToSubmit));
// toss the SCC manager
ISourceControlModule::Get().GetProvider().Close();
}
return 0;
}
bool FBuildPatchFileConstructor::ConstructFileFromChunks( const FString& Filename, bool bResumeExisting )
{
const bool bIsFileData = BuildManifest->IsFileDataManifest();
bResumeExisting = bResumeExisting && !bIsFileData;
bool bSuccess = true;
FString ErrorString;
FString NewFilename = StagingDirectory / Filename;
// Calculate the hash as we write the data
FSHA1 HashState;
FSHAHashData HashValue;
// First make sure we can get the file manifest
const FFileManifestData* FileManifest = BuildManifest->GetFileManifest(Filename);
bSuccess = FileManifest != nullptr;
if( bSuccess )
{
if( !FileManifest->SymlinkTarget.IsEmpty() )
{
#if PLATFORM_MAC
bSuccess = symlink(TCHAR_TO_UTF8(*FileManifest->SymlinkTarget), TCHAR_TO_UTF8(*NewFilename)) == 0;
#else
const bool bSymlinkNotImplemented = false;
check(bSymlinkNotImplemented);
bSuccess = false;
#endif
return bSuccess;
}
// Check for resuming of existing file
int64 StartPosition = 0;
int32 StartChunkPart = 0;
if( bResumeExisting )
{
// We have to read in the existing file so that the hash check can still be done.
FArchive* NewFileReader = IFileManager::Get().CreateFileReader( *NewFilename );
if( NewFileReader != NULL )
{
// Read buffer
uint8* ReadBuffer = new uint8[ FBuildPatchData::ChunkDataSize ];
// Reuse a certain amount of the file
StartPosition = FMath::Max<int64>( 0, NewFileReader->TotalSize() - NUM_BYTES_RESUME_IGNORE );
// We'll also find the correct chunkpart to start writing from
int64 ByteCounter = 0;
for( int32 ChunkPartIdx = StartChunkPart; ChunkPartIdx < FileManifest->FileChunkParts.Num() && !FBuildPatchInstallError::HasFatalError(); ++ChunkPartIdx )
{
const FChunkPartData& ChunkPart = FileManifest->FileChunkParts[ ChunkPartIdx ];
const int64 NextBytePosition = ByteCounter + ChunkPart.Size;
if( NextBytePosition <= StartPosition )
{
// Read data for hash check
NewFileReader->Serialize( ReadBuffer, ChunkPart.Size );
HashState.Update( ReadBuffer, ChunkPart.Size );
// Count bytes read from file
ByteCounter = NextBytePosition;
// Set to resume from next chunk part
StartChunkPart = ChunkPartIdx + 1;
// Inform the chunk cache of the chunk part skip
FBuildPatchChunkCache::Get().SkipChunkPart( ChunkPart );
// Wait if paused
BuildProgress->WaitWhilePaused();
}
else
{
// No more parts on disk
break;
}
}
// Set start position to the byte we got up to
StartPosition = ByteCounter;
// Clean read buffer
delete[] ReadBuffer;
// Close file
NewFileReader->Close();
delete NewFileReader;
}
}
// Now we can make sure the chunk cache knows to start downloading chunks
if( !bIsFileData && !bIsDownloadStarted && !FBuildPatchInstallError::HasFatalError() )
{
bIsDownloadStarted = true;
FBuildPatchChunkCache::Get().BeginDownloads();
}
// Attempt to create the file
FArchive* NewFile = IFileManager::Get().CreateFileWriter( *NewFilename, bResumeExisting ? EFileWrite::FILEWRITE_Append : 0 );
bSuccess = NewFile != NULL;
if( bSuccess )
{
// Whenever we start writing again, there's no more resuming to be done
BuildProgress->SetStateProgress( EBuildPatchProgress::Resuming, 1.0f );
// Seek to file write position
NewFile->Seek( StartPosition );
// For each chunk, load it, and place it's data into the file
for( int32 ChunkPartIdx = StartChunkPart; ChunkPartIdx < FileManifest->FileChunkParts.Num() && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++ChunkPartIdx )
{
const FChunkPartData& ChunkPart = FileManifest->FileChunkParts[ChunkPartIdx];
if( bIsFileData )
{
bSuccess = InsertFileData( ChunkPart, *NewFile, HashState );
}
else
{
bSuccess = InsertChunkData( ChunkPart, *NewFile, HashState );
}
if( bSuccess )
{
CountBytesProcessed( ChunkPart.Size );
// Wait if paused
BuildProgress->WaitWhilePaused();
}
else
{
ErrorString = TEXT( "Failed to construct file " );
ErrorString += Filename;
ErrorString += TEXT( " because of chunk " );
ErrorString += ChunkPart.Guid.ToString();
GWarn->Logf( TEXT( "BuildPatchFileConstructor: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
// Close the file writer
NewFile->Close();
delete NewFile;
}
else
{
FBuildPatchAnalytics::RecordConstructionError( Filename, FPlatformMisc::GetLastError(), TEXT( "Could Not Create File" ) );
ErrorString = TEXT( "Could not create new file " );
ErrorString += Filename;
GWarn->Logf( TEXT( "BuildPatchFileConstructor: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
else
{
FBuildPatchAnalytics::RecordConstructionError( Filename, INDEX_NONE, TEXT( "Missing File Manifest" ) );
ErrorString = TEXT( "Build manifest does not contain a file manifest for " );
ErrorString += Filename;
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
// Verify the hash for the file that we created
if( bSuccess )
{
HashState.Final();
HashState.GetHash( HashValue.Hash );
bSuccess = HashValue == FileManifest->FileHash;
if( !bSuccess )
{
FBuildPatchAnalytics::RecordConstructionError( Filename, INDEX_NONE, TEXT( "Serialised Verify Fail" ) );
ErrorString = TEXT( "Verify failed after constructing file " );
ErrorString += Filename;
GWarn->Logf( TEXT( "BuildDataGenerator: ERROR: %s" ), *ErrorString );
FBuildPatchInstallError::SetFatalError( EBuildPatchInstallError::FileConstructionFail, ErrorString );
}
}
#if PLATFORM_MAC
if( bSuccess && FileManifest->bIsUnixExecutable )
{
// Enable executable permission bit
struct stat FileInfo;
if (stat(TCHAR_TO_UTF8(*NewFilename), &FileInfo) == 0)
{
bSuccess = chmod(TCHAR_TO_UTF8(*NewFilename), FileInfo.st_mode | S_IXUSR | S_IXGRP | S_IXOTH) == 0;
}
}
#endif
// Delete the staging file if unsuccessful by means of construction fail (i.e. keep if canceled or download issue)
if( !bSuccess && FBuildPatchInstallError::GetErrorState() == EBuildPatchInstallError::FileConstructionFail )
{
IFileManager::Get().Delete( *NewFilename, false, true );
}
return bSuccess;
}
const bool FChunkWriter::FQueuedChunkWriter::WriteChunkData(const FString& ChunkFilename, FChunkFile* ChunkFile, const FGuid& ChunkGuid)
{
// Chunks are saved with GUID, so if a file already exists it will never be different.
// Skip with return true if already exists
if( FPaths::FileExists( ChunkFilename ) )
{
const int64 ChunkFilesSize = IFileManager::Get().FileSize(*ChunkFilename);
ChunkFileSizesCS.Lock();
ChunkFileSizes.Add(ChunkGuid, ChunkFilesSize);
ChunkFileSizesCS.Unlock();
return true;
}
FArchive* FileOut = IFileManager::Get().CreateFileWriter( *ChunkFilename );
bool bSuccess = FileOut != NULL;
if( bSuccess )
{
// Setup to handle compression
bool bDataIsCompressed = true;
uint8* ChunkDataSource = ChunkFile->ChunkData;
int32 ChunkDataSourceSize = FBuildPatchData::ChunkDataSize;
TArray< uint8 > TempCompressedData;
TempCompressedData.Empty( ChunkDataSourceSize );
TempCompressedData.AddUninitialized( ChunkDataSourceSize );
int32 CompressedSize = ChunkDataSourceSize;
// Compressed can increase in size, but the function will return as failure in that case
// we can allow that to happen since we would not keep larger compressed data anyway.
bDataIsCompressed = FCompression::CompressMemory(
static_cast< ECompressionFlags >( COMPRESS_ZLIB | COMPRESS_BiasMemory ),
TempCompressedData.GetData(),
CompressedSize,
ChunkFile->ChunkData,
FBuildPatchData::ChunkDataSize );
// If compression succeeded, set data vars
if( bDataIsCompressed )
{
ChunkDataSource = TempCompressedData.GetData();
ChunkDataSourceSize = CompressedSize;
}
// Setup Header
FChunkHeader& Header = ChunkFile->ChunkHeader;
*FileOut << Header;
Header.HeaderSize = FileOut->Tell();
Header.StoredAs = bDataIsCompressed ? FChunkHeader::STORED_COMPRESSED : FChunkHeader::STORED_RAW;
Header.DataSize = ChunkDataSourceSize;
Header.HashType = FChunkHeader::HASH_ROLLING;
// Write out files
FileOut->Seek( 0 );
*FileOut << Header;
FileOut->Serialize( ChunkDataSource, ChunkDataSourceSize );
const int64 ChunkFilesSize = FileOut->TotalSize();
FileOut->Close();
ChunkFileSizesCS.Lock();
ChunkFileSizes.Add(ChunkGuid, ChunkFilesSize);
ChunkFileSizesCS.Unlock();
bSuccess = !FileOut->GetError();
delete FileOut;
}
// Log errors
if( !bSuccess )
{
GLog->Logf( TEXT( "BuildPatchServices: Error: Could not save out generated chunk file %s" ), *ChunkFilename );
}
return bSuccess;
}
bool FBuildPatchInstaller::RunBackupAndMove()
{
GLog->Logf(TEXT("BuildPatchServices: Running backup and stage relocation"));
// If there's no error, move all complete files
bool bMoveSuccess = FBuildPatchInstallError::HasFatalError() == false;
if (bMoveSuccess)
{
// First handle files that should be removed for patching
TArray< FString > FilesToRemove;
if (CurrentBuildManifest.IsValid())
{
FBuildPatchAppManifest::GetRemovableFiles(CurrentBuildManifest.ToSharedRef(), NewBuildManifest, FilesToRemove);
}
// Add to build stats
ThreadLock.Lock();
BuildStats.NumFilesToRemove = FilesToRemove.Num();
ThreadLock.Unlock();
for (const FString& OldFilename : FilesToRemove)
{
BackupFileIfNecessary(OldFilename);
const bool bDeleteSuccess = IFileManager::Get().Delete(*(InstallDirectory / OldFilename), false, true, true);
const uint32 LastError = FPlatformMisc::GetLastError();
GLog->Logf(TEXT("BuildPatchServices: Removed (%u,%u) %s"), bDeleteSuccess ? 1 : 0, LastError, *OldFilename);
}
// Now handle files that have been constructed
bool bSavedMoveMarkerFile = false;
TArray< FString > ConstructionFiles;
NewBuildManifest->GetFileList(ConstructionFiles);
BuildProgress.SetStateProgress(EBuildPatchProgress::MovingToInstall, 0.0f);
const float NumConstructionFilesFloat = ConstructionFiles.Num();
for (auto ConstructionFilesIt = ConstructionFiles.CreateConstIterator(); ConstructionFilesIt && bMoveSuccess && !FBuildPatchInstallError::HasFatalError(); ++ConstructionFilesIt)
{
const FString& ConstructionFile = *ConstructionFilesIt;
const FString SrcFilename = InstallStagingDir / ConstructionFile;
const FString DestFilename = InstallDirectory / ConstructionFile;
const float FileIndexFloat = ConstructionFilesIt.GetIndex();
// Skip files not constructed
if (!FPaths::FileExists(SrcFilename))
{
BuildProgress.SetStateProgress(EBuildPatchProgress::MovingToInstall, FileIndexFloat / NumConstructionFilesFloat);
continue;
}
// Create the move marker file
if (!bSavedMoveMarkerFile)
{
bSavedMoveMarkerFile = true;
GLog->Logf(TEXT("BuildPatchServices: Create MM"));
FArchive* MoveMarkerFile = IFileManager::Get().CreateFileWriter(*PreviousMoveMarker, FILEWRITE_EvenIfReadOnly);
if (MoveMarkerFile != NULL)
{
MoveMarkerFile->Close();
delete MoveMarkerFile;
}
// Make sure we have some progress if we do some work
if (BuildProgress.GetStateWeight(EBuildPatchProgress::MovingToInstall) <= 0.0f)
{
BuildProgress.SetStateWeight(EBuildPatchProgress::MovingToInstall, 0.1f);
}
}
// Backup file if need be
BackupFileIfNecessary(ConstructionFile);
// Move the file to the installation directory
int32 MoveRetries = NUM_FILE_MOVE_RETRIES;
bMoveSuccess = IFileManager::Get().Move(*DestFilename, *SrcFilename, true, true);
uint32 ErrorCode = FPlatformMisc::GetLastError();
while (!bMoveSuccess && MoveRetries > 0)
{
--MoveRetries;
FBuildPatchAnalytics::RecordConstructionError(ConstructionFile, ErrorCode, TEXT("Failed To Move"));
GWarn->Logf(TEXT("BuildPatchServices: ERROR: Failed to move file %s (%d), trying copy"), *ConstructionFile, ErrorCode);
bMoveSuccess = IFileManager::Get().Copy(*DestFilename, *SrcFilename, true, true) == COPY_OK;
ErrorCode = FPlatformMisc::GetLastError();
if (!bMoveSuccess)
{
GWarn->Logf(TEXT("BuildPatchServices: ERROR: Failed to copy file %s (%d), retying after 0.5 sec"), *ConstructionFile, ErrorCode);
FPlatformProcess::Sleep(0.5f);
--MoveRetries;
bMoveSuccess = IFileManager::Get().Move(*DestFilename, *SrcFilename, true, true);
ErrorCode = FPlatformMisc::GetLastError();
}
else
{
IFileManager::Get().Delete(*SrcFilename, false, true, false);
}
}
if (!bMoveSuccess)
{
GWarn->Logf(TEXT("BuildPatchServices: ERROR: Failed to move file %s"), *FPaths::GetCleanFilename(ConstructionFile));
FBuildPatchInstallError::SetFatalError(EBuildPatchInstallError::MoveFileToInstall);
}
else
{
FilesInstalled.Add(ConstructionFile);
BuildProgress.SetStateProgress(EBuildPatchProgress::MovingToInstall, FileIndexFloat / NumConstructionFilesFloat);
}
}
bMoveSuccess = bMoveSuccess && (FBuildPatchInstallError::HasFatalError() == false);
if (bMoveSuccess)
{
BuildProgress.SetStateProgress(EBuildPatchProgress::MovingToInstall, 1.0f);
}
}
GLog->Logf(TEXT("BuildPatchServices: Relocation complete %d"), bMoveSuccess?1:0);
return bMoveSuccess;
}
/**
* Mimics Windows WER format
*/
void GenerateWindowsErrorReport(const FString & WERPath, bool bReportingNonCrash)
{
FArchive* ReportFile = IFileManager::Get().CreateFileWriter(*WERPath);
if (ReportFile != NULL)
{
// write BOM
static uint16 ByteOrderMarker = 0xFEFF;
ReportFile->Serialize(&ByteOrderMarker, sizeof(ByteOrderMarker));
WriteLine(ReportFile, TEXT("<?xml version=\"1.0\" encoding=\"UTF-16\"?>"));
WriteLine(ReportFile, TEXT("<WERReportMetadata>"));
WriteLine(ReportFile, TEXT("\t<OSVersionInformation>"));
WriteLine(ReportFile, TEXT("\t\t<WindowsNTVersion>0.0</WindowsNTVersion>"));
WriteLine(ReportFile, TEXT("\t\t<Build>No Build</Build>"));
WriteLine(ReportFile, TEXT("\t\t<Product>Linux</Product>"));
WriteLine(ReportFile, TEXT("\t\t<Edition>No Edition</Edition>"));
WriteLine(ReportFile, TEXT("\t\t<BuildString>No BuildString</BuildString>"));
WriteLine(ReportFile, TEXT("\t\t<Revision>0</Revision>"));
WriteLine(ReportFile, TEXT("\t\t<Flavor>No Flavor</Flavor>"));
WriteLine(ReportFile, TEXT("\t\t<Architecture>Unknown Architecture</Architecture>"));
WriteLine(ReportFile, TEXT("\t\t<LCID>0</LCID>"));
WriteLine(ReportFile, TEXT("\t</OSVersionInformation>"));
WriteLine(ReportFile, TEXT("\t<ParentProcessInformation>"));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<ParentProcessId>%d</ParentProcessId>"), getppid()));
WriteLine(ReportFile, TEXT("\t\t<ParentProcessPath>C:\\Windows\\explorer.exe</ParentProcessPath>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<ParentProcessCmdLine>C:\\Windows\\Explorer.EXE</ParentProcessCmdLine>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t</ParentProcessInformation>"));
WriteLine(ReportFile, TEXT("\t<ProblemSignatures>"));
WriteLine(ReportFile, TEXT("\t\t<EventType>APPCRASH</EventType>"));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter0>UE4-%s</Parameter0>"), FApp::GetGameName()));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter1>%d.%d.%d</Parameter1>"), FEngineVersion::Current().GetMajor(), FEngineVersion::Current().GetMinor(), FEngineVersion::Current().GetPatch()));
WriteLine(ReportFile, TEXT("\t\t<Parameter2>0</Parameter2>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter3>Unknown Fault Module</Parameter3>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter4>0.0.0.0</Parameter4>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter5>00000000</Parameter5>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter6>00000000</Parameter6>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter7>0000000000000000</Parameter7>")); // FIXME: supply valid?
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter8>!%s!</Parameter8>"), FCommandLine::Get())); // FIXME: supply valid? Only partially valid
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter9>%s!%s!%s!%d</Parameter9>"), *FApp::GetBranchName(), FPlatformProcess::BaseDir(), FPlatformMisc::GetEngineMode(), FEngineVersion::Current().GetChangelist()));
WriteLine(ReportFile, TEXT("\t</ProblemSignatures>"));
WriteLine(ReportFile, TEXT("\t<DynamicSignatures>"));
WriteLine(ReportFile, TEXT("\t\t<Parameter1>6.1.7601.2.1.0.256.48</Parameter1>"));
WriteLine(ReportFile, TEXT("\t\t<Parameter2>1033</Parameter2>"));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<DeploymentName>%s</DeploymentName>"), FApp::GetDeploymentName()));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<IsEnsure>%s</IsEnsure>"), bReportingNonCrash ? TEXT("1") : TEXT("0")));
WriteLine(ReportFile, TEXT("\t</DynamicSignatures>"));
WriteLine(ReportFile, TEXT("\t<SystemInformation>"));
WriteLine(ReportFile, TEXT("\t\t<MID>11111111-2222-3333-4444-555555555555</MID>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<SystemManufacturer>Unknown.</SystemManufacturer>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<SystemProductName>Linux machine</SystemProductName>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<BIOSVersion>A02</BIOSVersion>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t</SystemInformation>"));
WriteLine(ReportFile, TEXT("</WERReportMetadata>"));
ReportFile->Close();
delete ReportFile;
}
}
void FBuildPatchChunkCache::ReserveChunkInventorySlotForce( const FGuid& ChunkGuid )
{
// If already reserved, return immediate
if( ChunkCache.HasReservation( ChunkGuid ) || ChunkCache.Contains( ChunkGuid ) )
{
return;
}
// Begin by checking if any slots can be freed
ChunkCache.PurgeUnreferenced();
// Try to add the reservation
bool bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid );
// If we couldn't reserve, we need to boot out a chunk for this required one
if( bReservationAccepted == false )
{
// We create a unique ref array from the use order so that chunks not needed
// for longer times end up nearer the bottom of the array
TArray< FGuid > ChunkPriorityList;
ChunkInfoLock.Lock();
for( int32 ChunkUseOrderStackIdx = ChunkUseOrderStack.Num() - 1; ChunkUseOrderStackIdx >= 0 ; --ChunkUseOrderStackIdx )
{
ChunkPriorityList.AddUnique( ChunkUseOrderStack[ ChunkUseOrderStackIdx ] );
}
ChunkInfoLock.Unlock();
// Starting at the bottom of the list, we look for a chunk that is contained in the cache
for( int32 ChunkPriorityListIdx = ChunkPriorityList.Num() - 1; ChunkPriorityListIdx >= 0 && !bReservationAccepted; --ChunkPriorityListIdx )
{
const FGuid& LowPriChunk = ChunkPriorityList[ ChunkPriorityListIdx ];
BuildProgress->WaitWhilePaused();
// Check if there were any errors while paused, like canceling
if( FBuildPatchInstallError::HasFatalError() )
{
return;
}
if( ChunkCache.Contains( LowPriChunk ) )
{
GWarn->Logf( TEXT( "FBuildPatchChunkCache: Booting chunk %s" ), *LowPriChunk.ToString() );
// Save chunk to disk so we don't have to download again
bool bSuccess = true;
const FString NewChunkFilename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, LowPriChunk );
FChunkFile* LowPriChunkFile = ChunkCache.Get( LowPriChunk );
FChunkHeader* LowPriChunkHeader;
uint8* LowPriChunkData;
LowPriChunkFile->GetDataLock( &LowPriChunkData, &LowPriChunkHeader );
FArchive* FileOut = IFileManager::Get().CreateFileWriter( *NewChunkFilename );
bSuccess = FileOut != NULL;
const int32 LastError = FPlatformMisc::GetLastError();
if( bSuccess )
{
// Setup Header
*FileOut << *LowPriChunkHeader;
LowPriChunkHeader->HeaderSize = FileOut->Tell();
LowPriChunkHeader->StoredAs = FChunkHeader::STORED_RAW;
LowPriChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting
// Write out file
FileOut->Seek( 0 );
*FileOut << *LowPriChunkHeader;
FileOut->Serialize( LowPriChunkData, FBuildPatchData::ChunkDataSize );
FileOut->Close();
delete FileOut;
}
LowPriChunkFile->ReleaseDataLock();
// Setup new chunk origin
if( bSuccess )
{
ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Harddisk;
}
else
{
// Queue download if save failed
ChunkOrigins[ LowPriChunk ] = EChunkOrigin::Download;
FBuildPatchDownloader::Get().AddChunkToDownload( LowPriChunk );
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, NewChunkFilename, LastError, TEXT( "ChunkBooting" ), TEXT( "Chunk Save Failed" ) );
}
// Boot this chunk
ChunkCache.Remove( LowPriChunk );
// Try get the reservation again!
bReservationAccepted = ChunkCache.TryAddReservation( ChunkGuid );
// Count the boot
NumChunksCacheBooted.Increment();
}
}
// We must have been able to make room
check( bReservationAccepted );
}
}
// cooked package asset registry saves information about all the cooked packages and assets contained within for stats purposes
// in json format
bool FChunkManifestGenerator::SaveCookedPackageAssetRegistry( const FString& SandboxCookedRegistryFilename, const bool Append )
{
bool bSuccess = false;
for ( const auto& Platform : Platforms )
{
TSet<FName> CookedPackages;
// save the file
const FString CookedAssetRegistryFilename = SandboxCookedRegistryFilename.Replace(TEXT("[Platform]"), *Platform->PlatformName());
FString JsonOutString;
JsonWriter Json = TJsonWriterFactory<TCHAR, TPrettyJsonPrintPolicy<TCHAR> >::Create(&JsonOutString);
Json->WriteObjectStart();
Json->WriteArrayStart(TEXT("Packages"));
for ( const auto& Package : AllCookedPackages )
{
Json->WriteObjectStart(); // unnamed package start
const FName& PackageName = Package.Key;
const FString& SandboxPath = Package.Value;
CookedPackages.Add( PackageName );
FString PlatformSandboxPath = SandboxPath.Replace(TEXT("[Platform]"), *Platform->PlatformName());
FDateTime TimeStamp = IFileManager::Get().GetTimeStamp( *PlatformSandboxPath );
Json->WriteValue( "SourcePackageName", PackageName.ToString() );
Json->WriteValue( "CookedPackageName", PlatformSandboxPath );
Json->WriteValue( "CookedPackageTimeStamp", TimeStamp.ToString() );
Json->WriteArrayStart("AssetData");
for (const auto& AssetData : AssetRegistryData)
{ // Add only assets that have actually been cooked and belong to any chunk
if (AssetData.ChunkIDs.Num() > 0 && (AssetData.PackageName == PackageName))
{
Json->WriteObjectStart();
// save all their infos
Json->WriteValue(TEXT("ObjectPath"), AssetData.ObjectPath.ToString() );
Json->WriteValue(TEXT("PackageName"), AssetData.PackageName.ToString() );
Json->WriteValue(TEXT("PackagePath"), AssetData.PackagePath.ToString() );
Json->WriteValue(TEXT("GroupNames"), AssetData.GroupNames.ToString() );
Json->WriteValue(TEXT("AssetName"), AssetData.AssetName.ToString() );
Json->WriteValue(TEXT("AssetClass"), AssetData.AssetClass.ToString() );
Json->WriteObjectStart("TagsAndValues");
for ( const auto& Tag : AssetData.TagsAndValues )
{
Json->WriteValue( Tag.Key.ToString(), Tag.Value );
}
Json->WriteObjectEnd(); // end tags and values object
Json->WriteObjectEnd(); // end unnamed array object
}
}
Json->WriteArrayEnd();
Json->WriteObjectEnd(); // unnamed package
}
if ( Append )
{
FString JsonInString;
if ( FFileHelper::LoadFileToString(JsonInString, *CookedAssetRegistryFilename) )
{
// load up previous package asset registry and fill in any packages which weren't recooked on this run
JsonReader Reader = TJsonReaderFactory<TCHAR>::Create(JsonInString);
TSharedPtr<FJsonObject> JsonObject;
bool shouldRead = FJsonSerializer::Deserialize(Reader, JsonObject) && JsonObject.IsValid() && JsonObject->HasTypedField<EJson::Array>(TEXT("Packages"));
if ( shouldRead )
{
TArray<TSharedPtr<FJsonValue>> PackageList = JsonObject->GetArrayField(TEXT("Packages"));
for (auto PackageListIt = PackageList.CreateConstIterator(); PackageListIt && shouldRead; ++PackageListIt)
{
const TSharedPtr<FJsonValue>& JsonValue = *PackageListIt;
shouldRead = JsonValue->Type == EJson::Object;
if ( shouldRead )
{
const TSharedPtr<FJsonObject>& JsonPackage = JsonValue->AsObject();
// get the package name and see if we have already written it out this run
FString CookedPackageName;
verify( JsonPackage->TryGetStringField(TEXT("SourcePackageName"), CookedPackageName) );
const FName CookedPackageFName(*CookedPackageName);
if ( CookedPackages.Contains(CookedPackageFName))
{
// don't need to process this package
continue;
}
// check that the on disk version is still valid
FString SourcePackageName;
check( JsonPackage->TryGetStringField( TEXT("SourcePackageName"), SourcePackageName) );
// if our timestamp is different then don't copy the information over
FDateTime CurrentTimeStamp = IFileManager::Get().GetTimeStamp( *CookedPackageName );
FString SavedTimeString;
check( JsonPackage->TryGetStringField(TEXT("CookedPackageTimeStamp"), SavedTimeString) );
FDateTime SavedTimeStamp;
FDateTime::Parse(SavedTimeString, SavedTimeStamp);
if ( SavedTimeStamp != CurrentTimeStamp )
{
continue;
}
CopyJsonValueToWriter(Json, FString(), JsonValue);
// read in all the other stuff and copy it over to the new registry
/*Json->WriteObjectStart(); // open package
// copy all the values over
for ( const auto& JsonPackageValue : JsonPackage->Values)
{
CopyJsonValueToWriter(Json, JsonPackageValue.Key, JsonPackageValue.Value);
}
Json->WriteObjectEnd();*/
}
}
}
else
{
UE_LOG(LogChunkManifestGenerator, Warning, TEXT("Unable to read or json is invalid format %s"), *CookedAssetRegistryFilename);
}
}
}
Json->WriteArrayEnd();
Json->WriteObjectEnd();
if (Json->Close())
{
FArchive* ItemTemplatesFile = IFileManager::Get().CreateFileWriter(*CookedAssetRegistryFilename);
if (ItemTemplatesFile)
{
// serialize the file contents
TStringConversion<FTCHARToUTF8_Convert> Convert(*JsonOutString);
ItemTemplatesFile->Serialize(const_cast<ANSICHAR*>(Convert.Get()), Convert.Length());
ItemTemplatesFile->Close();
if ( !ItemTemplatesFile->IsError() )
{
bSuccess = true;
}
else
{
UE_LOG(LogChunkManifestGenerator, Error, TEXT("Unable to write to %s"), *CookedAssetRegistryFilename);
}
delete ItemTemplatesFile;
}
else
{
UE_LOG(LogChunkManifestGenerator, Error, TEXT("Unable to open %s for writing."), *CookedAssetRegistryFilename);
}
}
else
{
UE_LOG(LogChunkManifestGenerator, Error, TEXT("Error closing Json Writer"));
}
}
return bSuccess;
}
bool FBuildPatchChunkCache::RecycleChunkFromBuild( const FGuid& ChunkGuid )
{
// Must never double acquire
check( ChunkCache.Contains( ChunkGuid ) == false );
// Debug leaving any files open
bool bSuccess = true;
// Get the app manifest that this chunk can be sourced from
FBuildPatchAppManifestPtr ChunkSourceAppManifest = InstallationInfo.GetManifestContainingChunk(ChunkGuid);
if (!ChunkSourceAppManifest.IsValid())
{
return false;
}
// Get the install directory for this manifest
const FString ChunkSourceInstallDir = InstallationInfo.GetManifestInstallDir(ChunkSourceAppManifest);
if(ChunkSourceInstallDir.Len() <= 0)
{
return false;
}
// We need to generate an inventory of all chunk parts in this build that refer to the chunk that we require
TMap< FGuid, TArray< FFileChunkPart > > ChunkPartInventory;
TArray< FGuid > Array;
Array.Add( ChunkGuid );
ChunkSourceAppManifest->EnumerateChunkPartInventory(Array, ChunkPartInventory);
// Attempt construction of the chunk from the parts
FArchive* BuildFileIn = NULL;
FString BuildFileOpened;
int64 BuildFileInSize = 0;
// We must have a hash for this chunk or else we cant verify it
uint8 HashType = 0;
uint64 ChunkHash = 0;
FSHAHashData ChunkShaHash;
if (InstallManifet->GetChunkShaHash(ChunkGuid, ChunkShaHash))
{
HashType = FChunkHeader::HASH_SHA1;
}
else if (ChunkSourceAppManifest->GetChunkHash(ChunkGuid, ChunkHash))
{
HashType = FChunkHeader::HASH_ROLLING;
}
TArray< FFileChunkPart >* FileChunkPartsPtr = ChunkPartInventory.Find( ChunkGuid );
bSuccess = (FileChunkPartsPtr != NULL && HashType != 0);
if( bSuccess )
{
const TArray< FFileChunkPart >& FileChunkParts = *FileChunkPartsPtr;
TArray< uint8 > TempArray;
TempArray.AddUninitialized( FBuildPatchData::ChunkDataSize );
uint8* TempChunkConstruction = TempArray.GetData();
FMemory::Memzero( TempChunkConstruction, FBuildPatchData::ChunkDataSize );
bSuccess = FileChunkParts.Num() > 0;
for( auto FileChunkPartIt = FileChunkParts.CreateConstIterator(); FileChunkPartIt && bSuccess && !FBuildPatchInstallError::HasFatalError(); ++FileChunkPartIt )
{
const FFileChunkPart& FileChunkPart = *FileChunkPartIt;
FString FullFilename = ChunkSourceInstallDir / FileChunkPart.Filename;
// Close current build file ?
if( BuildFileIn != NULL && BuildFileOpened != FullFilename )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
BuildFileOpened = TEXT( "" );
BuildFileInSize = 0;
}
// Open build file ?
if( BuildFileIn == NULL )
{
BuildFileIn = IFileManager::Get().CreateFileReader( *FullFilename );
bSuccess = BuildFileIn != NULL;
if( !bSuccess )
{
BuildFileOpened = TEXT( "" );
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, FPlatformMisc::GetLastError(), TEXT( "ChunkRecycle" ), TEXT( "Source File Missing" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Failed to load source file for chunk. %s" ), *FullFilename );
}
else
{
BuildFileOpened = FullFilename;
BuildFileInSize = BuildFileIn->TotalSize();
}
}
// Grab the section of the chunk
if( BuildFileIn != NULL )
{
// Make sure we don't attempt to read off the end of the file
const int64 LastRequiredByte = FileChunkPart.FileOffset + FileChunkPart.ChunkPart.Size;
if( BuildFileInSize >= LastRequiredByte )
{
BuildFileIn->Seek( FileChunkPart.FileOffset );
BuildFileIn->Serialize( TempChunkConstruction + FileChunkPart.ChunkPart.Offset, FileChunkPart.ChunkPart.Size );
}
else
{
bSuccess = false;
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, FileChunkPart.Filename, INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Source File Too Small" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Source file too small for chunk position. %s" ), *FullFilename );
}
}
}
// Check no other fatal errors were registered in the meantime
bSuccess = bSuccess && !FBuildPatchInstallError::HasFatalError();
// Check chunk hash
if( bSuccess )
{
FSHAHashData ShaHashCheck;
switch (HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(TempChunkConstruction) == ChunkHash;
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(TempChunkConstruction, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkShaHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, TEXT( "" ), INDEX_NONE, TEXT( "ChunkRecycle" ), TEXT( "Chunk Hash Fail" ) );
GWarn->Logf( TEXT( "BuildPatchChunkConstruction: Warning: Hash check failed for recycled chunk %s" ), *ChunkGuid.ToString() );
}
}
// Save the chunk to cache if all went well
if( bSuccess )
{
// It was added asynchronously!!
check( ChunkCache.Contains( ChunkGuid ) == false );
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Copy the data
FMemoryReader MemReader( TempArray );
MemReader.Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Setup the header
ChunkHeader->Guid = ChunkGuid;
ChunkHeader->StoredAs = FChunkHeader::STORED_RAW;
ChunkHeader->DataSize = FBuildPatchData::ChunkDataSize; // This would change if compressing/encrypting
ChunkHeader->HashType = HashType;
ChunkHeader->RollingHash = ChunkHash;
ChunkHeader->SHAHash = ChunkShaHash;
// Release data
NewChunkFile->ReleaseDataLock();
// Count chunk
NumChunksRecycled.Increment();
// Add it to our cache.
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// Close any open file
if( BuildFileIn != NULL )
{
BuildFileIn->Close();
delete BuildFileIn;
BuildFileIn = NULL;
}
}
return bSuccess;
}
bool FBuildPatchChunkCache::ReadChunkFromDriveCache( const FGuid& ChunkGuid )
{
bool bSuccess = true;
// Get the chunk filename
const FString Filename = FBuildPatchUtils::GetChunkOldFilename( ChunkCacheStage, ChunkGuid );
// Read the chunk
FArchive* FileReader = IFileManager::Get().CreateFileReader( *Filename );
bSuccess = FileReader != NULL;
if( bSuccess )
{
// Get file size
const int64 FileSize = FileReader->TotalSize();
// Create the ChunkFile data structure
FChunkFile* NewChunkFile = new FChunkFile( GetRemainingReferenceCount( ChunkGuid ), true );
// Lock data
FChunkHeader* ChunkHeader;
uint8* ChunkData;
NewChunkFile->GetDataLock( &ChunkData, &ChunkHeader );
// Read the header
*FileReader << *ChunkHeader;
// Check header magic
bSuccess = ChunkHeader->IsValidMagic();
if ( bSuccess )
{
// Check the right data size
bSuccess = ChunkHeader->DataSize == FBuildPatchData::ChunkDataSize;
if( bSuccess )
{
// Check Header and data size
bSuccess = ( ChunkHeader->HeaderSize + ChunkHeader->DataSize ) == FileSize;
if( bSuccess )
{
// Read the data
FileReader->Serialize( ChunkData, FBuildPatchData::ChunkDataSize );
// Verify the data hash
FSHAHashData ShaHashCheck;
switch (ChunkHeader->HashType)
{
case FChunkHeader::HASH_ROLLING:
bSuccess = ChunkHeader->RollingHash == FRollingHash< FBuildPatchData::ChunkDataSize >::GetHashForDataSet(ChunkData);
break;
case FChunkHeader::HASH_SHA1:
FSHA1::HashBuffer(ChunkData, FBuildPatchData::ChunkDataSize, ShaHashCheck.Hash);
bSuccess = ShaHashCheck == ChunkHeader->SHAHash;
break;
default:
bSuccess = false;
}
if( !bSuccess )
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Hash Check Failed" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache chunk failed hash check %s" ), *ChunkGuid.ToString() );
}
else
{
// Count loads
NumDriveCacheChunkLoads.Increment();
GLog->Logf( TEXT( "FBuildPatchChunkCache: ReadChunkFromDriveCache loaded chunk %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Incorrect File Size" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache header info does not match file size %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Datasize/Hashtype Mismatch" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache mismatch datasize/hashtype combination %s" ), *ChunkGuid.ToString() );
}
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, INDEX_NONE, TEXT( "DriveCache" ), TEXT( "Corrupt Header" ) );
GLog->Logf( TEXT( "FBuildPatchChunkCache: ERROR: ReadChunkFromDriveCache corrupt header %s" ), *ChunkGuid.ToString() );
}
// Release data
NewChunkFile->ReleaseDataLock();
// Add the newly filled data to the cache if successful
if( bSuccess )
{
ChunkCache.Add( ChunkGuid, NewChunkFile );
}
// If there was a problem, remove from cache and reservation
else
{
ChunkCache.Remove( ChunkGuid );
}
// Close the file
FileReader->Close();
delete FileReader;
}
else
{
FBuildPatchAnalytics::RecordChunkCacheError( ChunkGuid, Filename, FPlatformMisc::GetLastError(), TEXT( "DriveCache" ), TEXT( "Open File Fail" ) );
GLog->Logf( TEXT( "BuildPatchServices: ERROR: GetChunkData could not open chunk file %s" ), *ChunkGuid.ToString() );
}
return bSuccess;
}
/**
* Mimics Windows WER format
*/
void GenerateWindowsErrorReport(const FString & WERPath)
{
FArchive* ReportFile = IFileManager::Get().CreateFileWriter(*WERPath);
if (ReportFile != NULL)
{
// write BOM
static uint16 ByteOrderMarker = 0xFEFF;
ReportFile->Serialize(&ByteOrderMarker, sizeof(ByteOrderMarker));
WriteLine(ReportFile, TEXT("<?xml version=\"1.0\" encoding=\"UTF-16\"?>"));
WriteLine(ReportFile, TEXT("<WERReportMetadata>"));
WriteLine(ReportFile, TEXT("\t<OSVersionInformation>"));
WriteLine(ReportFile, TEXT("\t\t<WindowsNTVersion>6.1</WindowsNTVersion>"));
WriteLine(ReportFile, TEXT("\t\t<Build>7601 Service Pack 1</Build>"));
WriteLine(ReportFile, TEXT("\t\t<Product>(0x30): Windows 7 Professional</Product>"));
WriteLine(ReportFile, TEXT("\t\t<Edition>Professional</Edition>"));
WriteLine(ReportFile, TEXT("\t\t<BuildString>7601.18044.amd64fre.win7sp1_gdr.130104-1431</BuildString>"));
WriteLine(ReportFile, TEXT("\t\t<Revision>1130</Revision>"));
WriteLine(ReportFile, TEXT("\t\t<Flavor>Multiprocessor Free</Flavor>"));
WriteLine(ReportFile, TEXT("\t\t<Architecture>X64</Architecture>"));
WriteLine(ReportFile, TEXT("\t\t<LCID>1033</LCID>"));
WriteLine(ReportFile, TEXT("\t</OSVersionInformation>"));
WriteLine(ReportFile, TEXT("\t<ParentProcessInformation>"));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<ParentProcessId>%d</ParentProcessId>"), getppid()));
WriteLine(ReportFile, TEXT("\t\t<ParentProcessPath>C:\\Windows\\explorer.exe</ParentProcessPath>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<ParentProcessCmdLine>C:\\Windows\\Explorer.EXE</ParentProcessCmdLine>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t</ParentProcessInformation>"));
WriteLine(ReportFile, TEXT("\t<ProblemSignatures>"));
WriteLine(ReportFile, TEXT("\t\t<EventType>APPCRASH</EventType>"));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter0>UE4-%s</Parameter0>"), FApp::GetGameName()));
WriteLine(ReportFile, *FString::Printf(TEXT("\t\t<Parameter1>1.0.%d.%d</Parameter1>"), ENGINE_VERSION_HIWORD, ENGINE_VERSION_LOWORD));
WriteLine(ReportFile, TEXT("\t\t<Parameter2>528f2d37</Parameter2>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter3>KERNELBASE.dll</Parameter3>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter4>6.1.7601.18015</Parameter4>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter5>50b8479b</Parameter5>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter6>00000001</Parameter6>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter7>0000000000009E5D</Parameter7>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter8>!!</Parameter8>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<Parameter9>UE4!D:/ShadowArtSync/UE4/Engine/Binaries/Win64/!Editor!0</Parameter9>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t</ProblemSignatures>"));
WriteLine(ReportFile, TEXT("\t<DynamicSignatures>"));
WriteLine(ReportFile, TEXT("\t\t<Parameter1>6.1.7601.2.1.0.256.48</Parameter1>"));
WriteLine(ReportFile, TEXT("\t\t<Parameter2>1033</Parameter2>"));
WriteLine(ReportFile, TEXT("\t</DynamicSignatures>"));
WriteLine(ReportFile, TEXT("\t<SystemInformation>"));
WriteLine(ReportFile, TEXT("\t\t<MID>11111111-2222-3333-4444-555555555555</MID>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<SystemManufacturer>Unknown.</SystemManufacturer>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<SystemProductName>Linux machine</SystemProductName>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t\t<BIOSVersion>A02</BIOSVersion>")); // FIXME: supply valid?
WriteLine(ReportFile, TEXT("\t</SystemInformation>"));
WriteLine(ReportFile, TEXT("</WERReportMetadata>"));
ReportFile->Close();
delete ReportFile;
}
}
void FCrashInfo::GenerateReport( const FString& DiagnosticsPath )
{
FArchive* ReportFile = IFileManager::Get().CreateFileWriter( *DiagnosticsPath );
if( ReportFile != NULL )
{
FString Line;
WriteLine( ReportFile, TCHAR_TO_UTF8( TEXT( "Generating report for minidump" ) ) );
WriteLine( ReportFile );
if ( EngineVersion.Len() > 0 )
{
Line = FString::Printf( TEXT( "Application version %s" ), *EngineVersion );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
}
else if( Modules.Num() > 0 )
{
Line = FString::Printf( TEXT( "Application version %d.%d.%d" ), Modules[0].Major, Modules[0].Minor, Modules[0].Patch );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
}
Line = FString::Printf( TEXT( " ... built from changelist %d" ), BuiltFromCL );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
if( LabelName.Len() > 0 )
{
Line = FString::Printf( TEXT( " ... based on label %s" ), *LabelName );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
}
WriteLine( ReportFile );
Line = FString::Printf( TEXT( "OS version %d.%d.%d.%d" ), SystemInfo.OSMajor, SystemInfo.OSMinor, SystemInfo.OSBuild, SystemInfo.OSRevision );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
Line = FString::Printf( TEXT( "Running %d %s processors" ), SystemInfo.ProcessorCount, GetProcessorArchitecture( SystemInfo.ProcessorArchitecture ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
Line = FString::Printf( TEXT( "Exception was \"%s\"" ), *Exception.ExceptionString );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
WriteLine( ReportFile );
Line = FString::Printf( TEXT( "Source context from \"%s\"" ), *SourceFile );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
WriteLine( ReportFile );
Line = FString::Printf( TEXT( "<SOURCE START>" ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
for( int32 LineIndex = 0; LineIndex < SourceContext.Num(); LineIndex++ )
{
Line = FString::Printf( TEXT( "%s" ), *SourceContext[LineIndex] );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
}
Line = FString::Printf( TEXT( "<SOURCE END>" ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
WriteLine( ReportFile );
Line = FString::Printf( TEXT( "<CALLSTACK START>" ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
for( int32 StackIndex = 0; StackIndex < Exception.CallStackString.Num(); StackIndex++ )
{
Line = FString::Printf( TEXT( "%s" ), *Exception.CallStackString[StackIndex] );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
}
Line = FString::Printf( TEXT( "<CALLSTACK END>" ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
WriteLine( ReportFile );
Line = FString::Printf( TEXT( "%d loaded modules" ), Modules.Num() );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
for( int32 ModuleIndex = 0; ModuleIndex < Modules.Num(); ModuleIndex++ )
{
FCrashModuleInfo& Module = Modules[ModuleIndex];
FString ModuleDirectory = FPaths::GetPath(Module.Name);
FString ModuleName = FPaths::GetBaseFilename( Module.Name, true ) + FPaths::GetExtension( Module.Name, true );
FString ModuleDetail = FString::Printf( TEXT( "%40s" ), *ModuleName );
FString Version = FString::Printf( TEXT( " (%d.%d.%d.%d)" ), Module.Major, Module.Minor, Module.Patch, Module.Revision );
ModuleDetail += FString::Printf( TEXT( " %22s" ), *Version );
ModuleDetail += FString::Printf( TEXT( " 0x%016x 0x%08x" ), Module.BaseOfImage, Module.SizeOfImage );
ModuleDetail += FString::Printf( TEXT( " %s" ), *ModuleDirectory );
WriteLine( ReportFile, TCHAR_TO_UTF8( *ModuleDetail ) );
}
WriteLine( ReportFile );
// Write out the processor debugging log
WriteLine( ReportFile, TCHAR_TO_UTF8( *Report ) );
Line = FString::Printf( TEXT( "Report end!" ) );
WriteLine( ReportFile, TCHAR_TO_UTF8( *Line ) );
ReportFile->Close();
delete ReportFile;
}
}
void USetupDefinition::ProcessCopy( FString Key, FString Value, UBOOL Selected, FInstallPoll* Poll )
{
guard(USetupDefinition::ProcessCopy);
BYTE Buffer[4096];
if( Selected && Key==TEXT("File") )
{
// Get source and dest filenames.
FFileInfo Info(*Value);
if( Info.Lang==TEXT("") || Info.Lang==UObject::GetLanguage() )
{
if( Info.Dest==TEXT("") )
Info.Dest = Info.Src;
if( !LocateSourceFile(Info.Src) )
LocalizedFileError( TEXT("MissingInstallerFile"), Patch ? TEXT("AdviseBadDownload") : TEXT("AdviseBadMedia"), *Info.Src );
FString FullDest = DestPath * Info.Dest;
FString FullSrc = Info.Ref==TEXT("") ? Info.Src : GetFullRef(*Info.Ref);
FString FullPatch = FullDest + TEXT("_tmp");
// Update uninstallation log.
UninstallLogAdd( TEXT("File"), *Info.Dest, 0, 1 );
// Make destination directory.
if( !GFileManager->MakeDirectory( *BasePath(FullDest), 1 ) )
LocalizedFileError( TEXT("FailedMakeDir"), TEXT("AdviseBadDest"), *FullDest );
// Status display.
if( !Poll->Poll(*FullDest,0,0,RunningBytes,TotalBytes) )
DidCancel();
// Copy SrcAr -> DestAr.
INT CalcOldCRC = 0;
guard(CopyFile);
FString ThisDest = Info.Ref==TEXT("") ? FullDest : FullPatch;
debugf( TEXT("Copying %s to %s"), *FullSrc, *ThisDest);
FArchive* SrcAr = GFileManager->CreateFileReader( *FullSrc );
if( !SrcAr )
LocalizedFileError( TEXT("FailedOpenSource"), Patch ? TEXT("AdviseBadDownload") : TEXT("AdviseBadMedia"), *FullSrc );
INT Size = SrcAr->TotalSize();
FArchive* DestAr = GFileManager->CreateFileWriter( *ThisDest, FILEWRITE_EvenIfReadOnly );
if( !DestAr )
LocalizedFileError( TEXT("FailedOpenDest"), TEXT("AdviseBadDest"), *ThisDest );
if( FullSrc.Right(3).Caps() == TEXT(".UZ") && ThisDest.Right(3).Caps() != TEXT(".UZ"))
{
INT Signature;
FString OrigFilename;
*SrcAr << Signature;
if( Signature != 5678 )
LocalizedFileError( TEXT("FailedOpenSource"), TEXT("AdviseBadMedia"), *FullSrc );
else
{
*SrcAr << OrigFilename;
FCodecFull Codec;
Codec.AddCodec(new FCodecRLE);
Codec.AddCodec(new FCodecBWT);
Codec.AddCodec(new FCodecMTF);
Codec.AddCodec(new FCodecRLE);
Codec.AddCodec(new FCodecHuffman);
Codec.Decode( *SrcAr, *DestAr );
if( !Poll->Poll(*FullDest,Size,Size,RunningBytes+=Size,TotalBytes) ) {
delete SrcAr;
delete DestAr;
DidCancel();
}
}
}
else
{
for( SQWORD Pos=0; Pos<Size; Pos+=sizeof(Buffer) )
{
INT Count = Min( Size-Pos, (SQWORD)sizeof(Buffer) );
SrcAr->Serialize( Buffer, Count );
if( SrcAr->IsError() )
{
delete SrcAr;
delete DestAr;
LocalizedFileError( TEXT("FailedReadingSource"), Patch ? TEXT("AdviseBadDownload") : TEXT("AdviseBadMedia"), *FullSrc );
}
if( Info.Ref!=TEXT("") )
{
CalcOldCRC = appMemCrc( Buffer, Count, CalcOldCRC );
}
DestAr->Serialize( Buffer, Count );
if( DestAr->IsError() )
{
delete SrcAr;
delete DestAr;
LocalizedFileError( TEXT("FailedWritingDest"), TEXT("AdviseBadDest"), *ThisDest );
}
if( !Poll->Poll(*FullDest,Pos,Size,RunningBytes+=Count,TotalBytes) )
{
delete SrcAr;
delete DestAr;
DidCancel();
}
}
}
delete SrcAr;
if( !DestAr->Close() )
LocalizedFileError( TEXT("FailedClosingDest"), TEXT("AdviseBadDest"), *ThisDest );
delete DestAr;
unguard;
// Patch SrcAr + DeltaFile -> DestAr.
if( Info.Ref!=TEXT("") )
{
guard(PatchFile);
BYTE Buffer[4096];
// Open files.
FString ThisSrc = FullPatch;
FArchive* SrcAr = GFileManager->CreateFileReader( *ThisSrc );
if( !SrcAr )
LocalizedFileError( TEXT("FailedOpenSource"), Patch ? TEXT("AdviseBadDownload") : TEXT("AdviseBadMedia"), *ThisSrc );
INT Size = SrcAr->TotalSize();
FArchive* DestAr = GFileManager->CreateFileWriter(*FullDest,FILEWRITE_EvenIfReadOnly);
if( !DestAr )
LocalizedFileError( TEXT("FailedOpenDest"), TEXT("AdviseBadDest"), *FullDest );
// Load delta file.
TArray<BYTE> Delta;
FString DeltaName = Info.Src;
if( !appLoadFileToArray( Delta, *DeltaName ) )
LocalizedFileError( TEXT("FailedLoadingUpdate"), TEXT("AdviseBadDownload"), *Info.Src );
debugf( TEXT("Patching %s to %s with %s"), *ThisSrc, *FullDest, *DeltaName );
// Decompress variables.
INT PrevSpot=0, CountSize=0, CRC=0;
INT Magic=0, OldSize=0, OldCRC=0, NewSize=0, NewCRC;
FBufferReader Reader( Delta );
Reader << Magic << OldSize << OldCRC << NewSize << NewCRC;
// Validate.
if( Magic!=0x92f92912 )
appErrorf( LineFormat(LocalizeError("PatchCorrupt")), *DeltaName, LocalizeError("AdviseBadDownload") );
if( OldSize!=Size || OldCRC!=CalcOldCRC )
appErrorf( LocalizeError("CdFileMismatch"), *Info.Ref, *LocalProduct );
// Delta decode it.
INT OldCountSize=0;
while( !Reader.AtEnd() )
{
INT Index;
Reader << AR_INDEX(Index);
if( Index<0 )
{
CRC = appMemCrc( &Delta(Reader.Tell()), -Index, CRC );
DestAr->Serialize( &Delta(Reader.Tell()), -Index );
if( DestAr->IsError() )
LocalizedFileError( TEXT("FailedWritingDest"), TEXT("AdviseBadDest"), *FullDest );
Reader.Seek( Reader.Tell() - Index );
CountSize -= Index;
}
else
{
INT CopyPos;
Reader << AR_INDEX(CopyPos);
CopyPos += PrevSpot;
check(CopyPos>=0);
check(CopyPos+Index<=Size);
SrcAr->Seek( CopyPos );
for( INT Base=Index; Base>0; Base-=sizeof(Buffer) )
{
INT Move = Min(Base,(INT)sizeof(Buffer));
SrcAr->Serialize( Buffer, Move );
if( SrcAr->IsError() )
LocalizedFileError( TEXT("FailedReadingSource"), Patch ? TEXT("AdviseBadDownload") : TEXT("AdviseBadDownload"), *ThisSrc );
CRC = appMemCrc( Buffer, Move, CRC );
DestAr->Serialize( Buffer, Move );
if( DestAr->IsError() )
LocalizedFileError( TEXT("FailedWritingDest"), TEXT("AdviseBadDest"), *FullDest );
}
CountSize += Index;
PrevSpot = CopyPos + Index;
}
if( ((CountSize^OldCountSize)&~(sizeof(Buffer)-1)) || Reader.AtEnd() )
{
if( !Poll->Poll(*FullDest,CountSize,Info.Size,RunningBytes+=(CountSize-OldCountSize),TotalBytes) )
{
delete SrcAr;
delete DestAr;
DidCancel();
}
OldCountSize = CountSize;
}
}
if( NewSize!=CountSize || NewCRC!=CRC )
appErrorf( LineFormat(LocalizeError("PatchCorrupt")), *DeltaName, LocalizeError("AdviseBadDownload") );
delete SrcAr;
if( !DestAr->Close() )
LocalizedFileError( TEXT("FailedClosingDest"), TEXT("AdviseBadDest"), *FullDest );
delete DestAr;
GFileManager->Delete( *ThisSrc );
unguard;
}
}
}
unguard;
}