void FAssetDataGatherer::DiscoverFilesToSearch()
{
if( PathsToSearch.Num() > 0 )
{
TArray<FString> DiscoveredFilesToSearch;
TSet<FString> LocalDiscoveredPathsSet;
TArray<FString> CopyOfPathsToSearch;
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
CopyOfPathsToSearch = PathsToSearch;
// Remove all of the existing paths from the list, since we'll process them all below. New paths may be
// added to the original list on a different thread as we go along, but those new paths won't be processed
// during this call of DisoverFilesToSearch(). But we'll get them on the next call!
PathsToSearch.Empty();
bIsDiscoveringFiles = true;
}
// Iterate over any paths that we have remaining to scan
for ( int32 PathIdx=0; PathIdx < CopyOfPathsToSearch.Num(); ++PathIdx )
{
const FString& Path = CopyOfPathsToSearch[PathIdx];
// Convert the package path to a filename with no extension (directory)
const FString FilePath = FPackageName::LongPackageNameToFilename(Path);
// Gather the package files in that directory and subdirectories
TArray<FString> Filenames;
FPackageName::FindPackagesInDirectory(Filenames, FilePath);
for (int32 FilenameIdx = 0; FilenameIdx < Filenames.Num(); FilenameIdx++)
{
FString Filename(Filenames[FilenameIdx]);
if ( IsValidPackageFileToRead(Filename) )
{
// Add the path to this asset into the list of discovered paths
const FString LongPackageName = FPackageName::FilenameToLongPackageName(Filename);
LocalDiscoveredPathsSet.Add( FPackageName::GetLongPackagePath(LongPackageName) );
DiscoveredFilesToSearch.Add(Filename);
}
}
}
// Turn the set into an array here before the critical section below
TArray<FString> LocalDiscoveredPathsArray = LocalDiscoveredPathsSet.Array();
{
// Place all the discovered files into the files to search list
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
FilesToSearch.Append(DiscoveredFilesToSearch);
DiscoveredPaths.Append(LocalDiscoveredPathsArray);
bIsDiscoveringFiles = false;
}
}
}
void FAssetDataGatherer::AddFilesToSearch(const TArray<FString>& Files)
{
TArray<FString> FilesToAdd;
for (int32 FilenameIdx = 0; FilenameIdx < Files.Num(); FilenameIdx++)
{
FString Filename(Files[FilenameIdx]);
if ( IsValidPackageFileToRead(Filename) )
{
// Add the path to this asset into the list of discovered paths
FilesToAdd.Add(Filename);
}
}
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
FilesToSearch.Append(FilesToAdd);
}
}
void FAssetDataGatherer::AddFilesToSearch(const TArray<FString>& Files)
{
TArray<FString> FilesToAdd;
for (const FString& Filename : Files)
{
if ( IsValidPackageFileToRead(Filename) )
{
// Add the path to this asset into the list of discovered paths
FilesToAdd.Add(Filename);
}
}
if (FilesToAdd.Num() > 0)
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
FilesToSearch.Append(FilesToAdd);
}
}
uint32 FAssetDataDiscovery::Run()
{
double DiscoverStartTime = FPlatformTime::Seconds();
int32 NumDiscoveredFiles = 0;
FString LocalFilenamePathToPrioritize;
TSet<FString> LocalDiscoveredPathsSet;
TArray<FString> LocalDiscoveredDirectories;
TArray<FDiscoveredPackageFile> LocalPriorityFilesToSearch;
TArray<FDiscoveredPackageFile> LocalNonPriorityFilesToSearch;
// This set contains the folders that we should hide by default unless they contain assets
TSet<FString> PathsToHideIfEmpty;
PathsToHideIfEmpty.Add(TEXT("/Game/Collections"));
auto FlushLocalResultsIfRequired = [&]()
{
if (LocalPriorityFilesToSearch.Num() > 0 || LocalNonPriorityFilesToSearch.Num() > 0 || LocalDiscoveredPathsSet.Num() > 0)
{
TArray<FString> LocalDiscoveredPathsArray = LocalDiscoveredPathsSet.Array();
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
// Place all the discovered files into the files to search list
DiscoveredPaths.Append(MoveTemp(LocalDiscoveredPathsArray));
PriorityDiscoveredFiles.Append(MoveTemp(LocalPriorityFilesToSearch));
NonPriorityDiscoveredFiles.Append(MoveTemp(LocalNonPriorityFilesToSearch));
}
}
LocalDiscoveredPathsSet.Reset();
LocalPriorityFilesToSearch.Reset();
LocalNonPriorityFilesToSearch.Reset();
};
auto IsPriorityFile = [&](const FString& InPackageFilename) -> bool
{
return !bIsSynchronous && !LocalFilenamePathToPrioritize.IsEmpty() && InPackageFilename.StartsWith(LocalFilenamePathToPrioritize);
};
auto OnIterateDirectoryItem = [&](const TCHAR* InPackageFilename, const FFileStatData& InPackageStatData) -> bool
{
if (StopTaskCounter.GetValue() != 0)
{
// Requested to stop - break out of the directory iteration
return false;
}
const FString PackageFilenameStr = InPackageFilename;
if (InPackageStatData.bIsDirectory)
{
LocalDiscoveredDirectories.Add(PackageFilenameStr / TEXT(""));
FString PackagePath;
if (FPackageName::TryConvertFilenameToLongPackageName(PackageFilenameStr, PackagePath) && !PathsToHideIfEmpty.Contains(PackagePath))
{
LocalDiscoveredPathsSet.Add(PackagePath);
}
}
else if (FPackageName::IsPackageFilename(PackageFilenameStr))
{
if (IsValidPackageFileToRead(PackageFilenameStr))
{
const FString LongPackageNameStr = FPackageName::FilenameToLongPackageName(PackageFilenameStr);
if (IsPriorityFile(PackageFilenameStr))
{
LocalPriorityFilesToSearch.Add(FDiscoveredPackageFile(PackageFilenameStr, InPackageStatData.ModificationTime));
}
else
{
LocalNonPriorityFilesToSearch.Add(FDiscoveredPackageFile(PackageFilenameStr, InPackageStatData.ModificationTime));
}
LocalDiscoveredPathsSet.Add(FPackageName::GetLongPackagePath(LongPackageNameStr));
++NumDiscoveredFiles;
// Flush the data if we've processed enough
if (!bIsSynchronous && (LocalPriorityFilesToSearch.Num() + LocalNonPriorityFilesToSearch.Num()) >= AssetDataGathererConstants::MaxFilesToDiscoverBeforeFlush)
{
FlushLocalResultsIfRequired();
}
}
}
return true;
};
bool bIsIdle = true;
while (StopTaskCounter.GetValue() == 0)
{
FString LocalDirectoryToSearch;
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
if (DirectoriesToSearch.Num() > 0)
{
bIsDiscoveringFiles = true;
LocalFilenamePathToPrioritize = FilenamePathToPrioritize;
// Pop off the first path to search
LocalDirectoryToSearch = DirectoriesToSearch[0];
DirectoriesToSearch.RemoveAt(0, 1, false);
}
}
if (LocalDirectoryToSearch.Len() > 0)
{
if (bIsIdle)
{
bIsIdle = false;
// About to start work - reset these
DiscoverStartTime = FPlatformTime::Seconds();
NumDiscoveredFiles = 0;
}
// Iterate the current search directory
FLambdaDirectoryStatVisitor Visitor(OnIterateDirectoryItem);
IFileManager::Get().IterateDirectoryStat(*LocalDirectoryToSearch, Visitor);
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
// Push back any newly discovered sub-directories
if (LocalDiscoveredDirectories.Num() > 0)
{
// Use LocalDiscoveredDirectories as scratch space, then move it back out - this puts the directories we just
// discovered at the start of the list for the next iteration, which can help with disk locality
LocalDiscoveredDirectories.Append(MoveTemp(DirectoriesToSearch));
DirectoriesToSearch = MoveTemp(LocalDiscoveredDirectories);
}
LocalDiscoveredDirectories.Reset();
if (!bIsSynchronous)
{
FlushLocalResultsIfRequired();
SortPathsByPriority(1);
}
}
}
else
{
if (!bIsIdle)
{
bIsIdle = true;
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
bIsDiscoveringFiles = false;
}
UE_LOG(LogAssetRegistry, Verbose, TEXT("Discovery took %0.6f seconds and found %d files to process"), FPlatformTime::Seconds() - DiscoverStartTime, NumDiscoveredFiles);
}
// Ran out of things to do... if we have any pending results, flush those now
FlushLocalResultsIfRequired();
if (bIsSynchronous)
{
// This is synchronous. Since our work is done, we should safely exit
Stop();
}
else
{
// No work to do. Sleep for a little and try again later.
FPlatformProcess::Sleep(0.1);
}
}
}
return 0;
}
