/**
* The real thread entry point. It waits for work events to be queued. Once
* an event is queued, it executes it and goes back to waiting.
*/
virtual uint32 Run() override
{
while (!TimeToDie)
{
// This will force sending the stats packet from the previous frame.
SET_DWORD_STAT( STAT_ThreadPoolDummyCounter, 0 );
// We need to wait for shorter amount of time
bool bContinueWaiting = true;
while( bContinueWaiting )
{
DECLARE_SCOPE_CYCLE_COUNTER( TEXT( "FQueuedThread::Run.WaitForWork" ), STAT_FQueuedThread_Run_WaitForWork, STATGROUP_ThreadPoolAsyncTasks );
// Wait for some work to do
bContinueWaiting = !DoWorkEvent->Wait( 10 );
}
IQueuedWork* LocalQueuedWork = QueuedWork;
QueuedWork = nullptr;
FPlatformMisc::MemoryBarrier();
check(LocalQueuedWork || TimeToDie); // well you woke me up, where is the job or termination request?
while (LocalQueuedWork)
{
// Tell the object to do the work
LocalQueuedWork->DoThreadedWork();
// Let the object cleanup before we remove our ref to it
LocalQueuedWork = OwningThreadPool->ReturnToPoolOrGetNextJob(this);
}
}
return 0;
}
void FManifestBuilderImpl::AddDataScanner(FDataScannerRef Scanner)
{
DataScannersCS.Lock();
DataScanners.Add(MoveTemp(Scanner));
DataScannersCS.Unlock();
CheckForWork->Trigger();
}
void FEditPanel::ApplyModif(FEvent& oEvent)
{
oEvent.SetParent(m_pNameRef->GetValue());
m_pTagControl->ApplyModif(oEvent);
m_pRoleControl->ApplyModif(oEvent);
}
/**
* Tells the thread there is work to be done. Upon completion, the thread
* is responsible for adding itself back into the available pool.
*
* @param InQueuedWork The queued work to perform
*/
void DoWork(IQueuedWork* InQueuedWork)
{
check(QueuedWork == nullptr && "Can't do more than one task at a time");
// Tell the thread the work to be done
QueuedWork = InQueuedWork;
FPlatformMisc::MemoryBarrier();
// Tell the thread to wake up and do its job
DoWorkEvent->Trigger();
}
/**
* Tells the thread there is work to be done. Upon completion, the thread
* is responsible for adding itself back into the available pool.
*
* @param InQueuedWork The queued work to perform
*/
void DoWork(IQueuedWork* InQueuedWork)
{
DECLARE_SCOPE_CYCLE_COUNTER( TEXT( "FQueuedThread::DoWork" ), STAT_FQueuedThread_DoWork, STATGROUP_ThreadPoolAsyncTasks );
check(QueuedWork == nullptr && "Can't do more than one task at a time");
// Tell the thread the work to be done
QueuedWork = InQueuedWork;
FPlatformMisc::MemoryBarrier();
// Tell the thread to wake up and do its job
DoWorkEvent->Trigger();
}
FEvent* FWindowsPlatformProcess::CreateSynchEvent(bool bIsManualReset)
{
// Allocate the new object
FEvent* Event = NULL;
if (FPlatformProcess::SupportsMultithreading())
{
Event = new FEventWin();
}
else
{
// Fake event object.
Event = new FSingleThreadEvent();
}
// If the internal create fails, delete the instance and return NULL
if (!Event->Create(bIsManualReset))
{
delete Event;
Event = NULL;
}
return Event;
}
/**
* Tells the thread to exit. If the caller needs to know when the thread
* has exited, it should use the bShouldWait value and tell it how long
* to wait before deciding that it is deadlocked and needs to be destroyed.
* NOTE: having a thread forcibly destroyed can cause leaks in TLS, etc.
*
* @return True if the thread exited graceful, false otherwise
*/
virtual bool KillThread()
{
bool bDidExitOK = true;
// Tell the thread it needs to die
FPlatformAtomics::InterlockedExchange(&TimeToDie,1);
// Trigger the thread so that it will come out of the wait state if
// it isn't actively doing work
DoWorkEvent->Trigger();
// If waiting was specified, wait the amount of time. If that fails,
// brute force kill that thread. Very bad as that might leak.
Thread->WaitForCompletion();
// Clean up the event
FPlatformProcess::ReturnSynchEventToPool(DoWorkEvent);
DoWorkEvent = nullptr;
delete Thread;
return bDidExitOK;
}
/**
* The real thread entry point. It waits for work events to be queued. Once
* an event is queued, it executes it and goes back to waiting.
*/
virtual uint32 Run() override
{
while (!TimeToDie)
{
// Wait for some work to do
DoWorkEvent->Wait();
IQueuedWork* LocalQueuedWork = QueuedWork;
QueuedWork = nullptr;
FPlatformMisc::MemoryBarrier();
check(LocalQueuedWork || TimeToDie); // well you woke me up, where is the job or termination request?
while (LocalQueuedWork)
{
// Tell the object to do the work
LocalQueuedWork->DoThreadedWork();
// Let the object cleanup before we remove our ref to it
LocalQueuedWork = OwningThreadPool->ReturnToPoolOrGetNextJob(this);
}
}
return 0;
}
int32 UFileServerCommandlet::Main( const FString& Params )
{
GIsRequestingExit = false;
GIsRunning = true;
//@todo abstract properly or delete
#if PLATFORM_WINDOWS// Windows only
// Used by the .com wrapper to notify that the Ctrl-C handler was triggered.
// This shared event is checked each tick so that the log file can be cleanly flushed.
FEvent* ComWrapperShutdownEvent = FPlatformProcess::CreateSynchEvent(true);
#endif
// parse instance identifier
FString InstanceIdString;
if (FParse::Value(*Params, TEXT("InstanceId="), InstanceIdString))
{
if (!FGuid::Parse(InstanceIdString, InstanceId))
{
UE_LOG(LogFileServerCommandlet, Warning, TEXT("Invalid InstanceId on command line: %s"), *InstanceIdString);
}
}
// start the listening thread
INetworkFileServer* NetworkFileServer = FModuleManager::LoadModuleChecked<INetworkFileSystemModule>("NetworkFileSystem")
.CreateNetworkFileServer(true, InstanceId.IsValid() ? 0 : -1);
TArray<TSharedPtr<FInternetAddr> > AddressList;
if ((NetworkFileServer == NULL) || !NetworkFileServer->GetAddressList(AddressList))
{
UE_LOG(LogFileServerCommandlet, Error, TEXT("Failed to create network file server"));
return -1;
}
// broadcast our presence
if (InstanceId.IsValid())
{
TArray<FString> AddressStringList;
for (int32 AddressIndex = 0; AddressIndex < AddressList.Num(); ++AddressIndex)
{
AddressStringList.Add(AddressList[AddressIndex]->ToString(true));
}
FMessageEndpointPtr MessageEndpoint = FMessageEndpoint::Builder("UFileServerCommandlet").Build();
if (MessageEndpoint.IsValid())
{
MessageEndpoint->Publish(new FFileServerReady(AddressStringList, InstanceId), EMessageScope::Network);
}
}
// main loop
FDateTime LastConnectionTime = FDateTime::UtcNow();
while (GIsRunning && !GIsRequestingExit)
{
GEngine->UpdateTimeAndHandleMaxTickRate();
GEngine->Tick(FApp::GetDeltaTime(), false);
// tick the directory watcher
FDirectoryWatcherModule& DirectoryWatcherModule = FModuleManager::Get().LoadModuleChecked<FDirectoryWatcherModule>(TEXT("DirectoryWatcher"));
DirectoryWatcherModule.Get()->Tick(FApp::GetDeltaTime());
// update task graph
FTaskGraphInterface::Get().ProcessThreadUntilIdle(ENamedThreads::GameThread);
// execute deferred commands
for (int32 DeferredCommandsIndex=0; DeferredCommandsIndex<GEngine->DeferredCommands.Num(); DeferredCommandsIndex++)
{
GEngine->Exec( GWorld, *GEngine->DeferredCommands[DeferredCommandsIndex], *GLog);
}
GEngine->DeferredCommands.Empty();
// handle server timeout
if (InstanceId.IsValid())
{
if (NetworkFileServer->NumConnections() > 0)
{
LastConnectionTime = FDateTime::UtcNow();
}
if ((FDateTime::UtcNow() - LastConnectionTime) > FTimespan::FromMinutes(3.0))
{
uint32 Result = FMessageDialog::Open(EAppMsgType::YesNo, NSLOCTEXT("UnrealEd", "FileServerIdle", "The file server did not receive any connections in the past 3 minutes. Would you like to shut it down?"));
if (Result == EAppReturnType::No)
{
LastConnectionTime = FDateTime::UtcNow();
}
else
{
break;
}
}
}
// flush log
GLog->FlushThreadedLogs();
#if PLATFORM_WINDOWS
if (ComWrapperShutdownEvent->Wait(0))
{
GIsRequestingExit = true;
}
#endif
}
// shutdown the server
NetworkFileServer->Shutdown();
delete NetworkFileServer;
//@todo abstract properly or delete
#if PLATFORM_WINDOWS
delete ComWrapperShutdownEvent;
#endif
GIsRunning = false;
return 0;
}
void FManifestBuilderImpl::BuildManifest()
{
TMap<FGuid, FChunkInfo> ChunkInfoLookup;
bool Running = true;
while (Running)
{
FDataScannerPtr NextScanner = GetNextScanner();
if (NextScanner.IsValid())
{
FDataScanResult ScanResult = NextScanner->GetResultWhenComplete();
ChunkInfoLookup.Append(ScanResult.ChunkInfo);
// Always reverse for now
if (ScanResult.DataStructure.Num() > 0)
{
FChunkPart& ChunkPart = ScanResult.DataStructure[0];
if (ChunkPart.DataOffset != FileBuilder.CurrentDataPos)
{
check(ChunkPart.DataOffset < FileBuilder.CurrentDataPos); // Missing data!
bool FoundPosition = false;
uint64 DataCount = 0;
for (int32 FileIdx = 0; FileIdx < Manifest->Data->FileManifestList.Num() && !FoundPosition; ++FileIdx)
{
FFileManifestData& FileManifest = Manifest->Data->FileManifestList[FileIdx];
FileManifest.Init();
uint64 FileStartIdx = DataCount;
uint64 FileEndIdx = FileStartIdx + FileManifest.GetFileSize();
if (FileEndIdx > ChunkPart.DataOffset)
{
for (int32 ChunkIdx = 0; ChunkIdx < FileManifest.FileChunkParts.Num() && !FoundPosition; ++ChunkIdx)
{
FChunkPartData& ChunkPartData = FileManifest.FileChunkParts[ChunkIdx];
uint64 ChunkPartEndIdx = DataCount + ChunkPartData.Size;
if (ChunkPartEndIdx < ChunkPart.DataOffset)
{
DataCount += ChunkPartData.Size;
}
else if (ChunkPartEndIdx > ChunkPart.DataOffset)
{
ChunkPartData.Size = ChunkPart.DataOffset - DataCount;
FileBuilder.CurrentDataPos = DataCount + ChunkPartData.Size;
FileManifest.FileChunkParts.SetNum(ChunkIdx + 1, false);
FileManifest.FileChunkParts.Emplace();
Manifest->Data->FileManifestList.SetNum(FileIdx + 1, false);
FileBuilder.FileManifest = &Manifest->Data->FileManifestList.Last();
bool FoundFile = BuildStreamer->GetFileSpan(FileStartIdx, FileBuilder.FileSpan);
check(FoundFile); // Incorrect positional tracking
FoundPosition = true;
}
else
{
FileBuilder.CurrentDataPos = DataCount + ChunkPartData.Size;
FileManifest.FileChunkParts.SetNum(ChunkIdx + 1, false);
FileManifest.FileChunkParts.Emplace();
Manifest->Data->FileManifestList.SetNum(FileIdx + 1, false);
FileBuilder.FileManifest = &Manifest->Data->FileManifestList.Last();
bool FoundFile = BuildStreamer->GetFileSpan(FileStartIdx, FileBuilder.FileSpan);
check(FoundFile); // Incorrect positional tracking
FoundPosition = true;
}
}
}
else if (FileEndIdx < ChunkPart.DataOffset)
{
DataCount += FileManifest.GetFileSize();
}
else
{
FileBuilder.FileManifest = nullptr;
FileBuilder.CurrentDataPos = DataCount + FileManifest.GetFileSize();
Manifest->Data->FileManifestList.SetNum(FileIdx + 1, false);
FoundPosition = true;
}
}
check(ChunkPart.DataOffset == FileBuilder.CurrentDataPos);
check(FileBuilder.FileManifest == nullptr || FileBuilder.FileSpan.Filename == Manifest->Data->FileManifestList.Last().Filename);
}
}
for (int32 idx = 0; idx < ScanResult.DataStructure.Num(); ++idx)
{
FChunkPart& ChunkPart = ScanResult.DataStructure[idx];
// Starting new file?
if (FileBuilder.FileManifest == nullptr)
{
Manifest->Data->FileManifestList.Emplace();
FileBuilder.FileManifest = &Manifest->Data->FileManifestList.Last();
bool FoundFile = BuildStreamer->GetFileSpan(FileBuilder.CurrentDataPos, FileBuilder.FileSpan);
check(FoundFile); // Incorrect positional tracking
FileBuilder.FileManifest->Filename = FileBuilder.FileSpan.Filename;
FileBuilder.FileManifest->FileChunkParts.Emplace();
}
FChunkPartData& FileChunkPartData = FileBuilder.FileManifest->FileChunkParts.Last();
FileChunkPartData.Guid = ChunkPart.ChunkGuid;
FileChunkPartData.Offset = (FileBuilder.CurrentDataPos - ChunkPart.DataOffset) + ChunkPart.ChunkOffset;
// Process data into file manifests
int64 FileDataLeft = (FileBuilder.FileSpan.StartIdx + FileBuilder.FileSpan.Size) - FileBuilder.CurrentDataPos;
int64 ChunkDataLeft = (ChunkPart.DataOffset + ChunkPart.PartSize) - FileBuilder.CurrentDataPos;
check(FileDataLeft > 0);
check(ChunkDataLeft > 0);
if (ChunkDataLeft >= FileDataLeft)
{
FileBuilder.CurrentDataPos += FileDataLeft;
FileChunkPartData.Size = FileDataLeft;
}
else
{
FileBuilder.CurrentDataPos += ChunkDataLeft;
FileChunkPartData.Size = ChunkDataLeft;
}
FileDataLeft = (FileBuilder.FileSpan.StartIdx + FileBuilder.FileSpan.Size) - FileBuilder.CurrentDataPos;
ChunkDataLeft = (ChunkPart.DataOffset + ChunkPart.PartSize) - FileBuilder.CurrentDataPos;
check(FileDataLeft == 0 || ChunkDataLeft == 0);
// End of file?
if (FileDataLeft == 0)
{
// Fill out rest of data??
FFileSpan FileSpan;
bool FoundFile = BuildStreamer->GetFileSpan(FileBuilder.FileSpan.StartIdx, FileSpan);
check(FoundFile); // Incorrect positional tracking
check(FileSpan.Filename == FileBuilder.FileManifest->Filename);
FMemory::Memcpy(FileBuilder.FileManifest->FileHash.Hash, FileSpan.SHAHash.Hash, FSHA1::DigestSize);
FFileAttributes Attributes = FileAttributesMap.FindRef(FileSpan.Filename);
FileBuilder.FileManifest->bIsUnixExecutable = Attributes.bUnixExecutable || FileSpan.IsUnixExecutable;
FileBuilder.FileManifest->SymlinkTarget = FileSpan.SymlinkTarget;
FileBuilder.FileManifest->bIsReadOnly = Attributes.bReadOnly;
FileBuilder.FileManifest->bIsCompressed = Attributes.bCompressed;
FileBuilder.FileManifest->InstallTags = Attributes.InstallTags.Array();
FileBuilder.FileManifest->Init();
check(FileBuilder.FileManifest->GetFileSize() == FileBuilder.FileSpan.Size);
FileBuilder.FileManifest = nullptr;
}
else if (ChunkDataLeft == 0)
{
FileBuilder.FileManifest->FileChunkParts.Emplace();
}
// Continue with this chunk?
if (ChunkDataLeft > 0)
{
--idx;
}
}
}
else
{
if (EndOfData)
{
Running = false;
}
else
{
CheckForWork->Wait();
CheckForWork->Reset();
}
}
}
// Fill out chunk list from only chunks that remain referenced
TSet<FGuid> ReferencedChunks;
for (const auto& FileManifest : Manifest->Data->FileManifestList)
{
for (const auto& ChunkPart : FileManifest.FileChunkParts)
{
if (ReferencedChunks.Contains(ChunkPart.Guid) == false)
{
auto& ChunkInfo = ChunkInfoLookup[ChunkPart.Guid];
ReferencedChunks.Add(ChunkPart.Guid);
Manifest->Data->ChunkList.Emplace();
auto& ChunkInfoData = Manifest->Data->ChunkList.Last();
ChunkInfoData.Guid = ChunkPart.Guid;
ChunkInfoData.Hash = ChunkInfo.Hash;
FMemory::Memcpy(ChunkInfoData.ShaHash.Hash, ChunkInfo.ShaHash.Hash, FSHA1::DigestSize);
ChunkInfoData.FileSize = ChunkInfo.ChunkFileSize;
ChunkInfoData.GroupNumber = FCrc::MemCrc32(&ChunkPart.Guid, sizeof(FGuid)) % 100;
}
}
}
// Get empty files
FSHA1 EmptyHasher;
EmptyHasher.Final();
const TArray< FString >& EmptyFileList = BuildStreamer->GetEmptyFiles();
for (const auto& EmptyFile : EmptyFileList)
{
Manifest->Data->FileManifestList.Emplace();
FFileManifestData& EmptyFileManifest = Manifest->Data->FileManifestList.Last();
EmptyFileManifest.Filename = EmptyFile;
EmptyHasher.GetHash(EmptyFileManifest.FileHash.Hash);
}
// Fill out lookups
Manifest->InitLookups();
}
void FManifestBuilderImpl::SaveToFile(const FString& Filename)
{
// Wait for builder to complete
EndOfData = true;
CheckForWork->Trigger();
Future.Wait();
Manifest->Data->ManifestFileVersion = EBuildPatchAppManifestVersion::GetLatestJsonVersion();
Manifest->SaveToFile(Filename, false);
}
