void FAssetDataGatherer::DiscoverFilesToSearch() { if( PathsToSearch.Num() > 0 ) { TArray<FString> DiscoveredFilesToSearch; TSet<FString> LocalDiscoveredPathsSet; TArray<FString> CopyOfPathsToSearch; { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); CopyOfPathsToSearch = PathsToSearch; // Remove all of the existing paths from the list, since we'll process them all below. New paths may be // added to the original list on a different thread as we go along, but those new paths won't be processed // during this call of DisoverFilesToSearch(). But we'll get them on the next call! PathsToSearch.Empty(); bIsDiscoveringFiles = true; } // Iterate over any paths that we have remaining to scan for ( int32 PathIdx=0; PathIdx < CopyOfPathsToSearch.Num(); ++PathIdx ) { const FString& Path = CopyOfPathsToSearch[PathIdx]; // Convert the package path to a filename with no extension (directory) const FString FilePath = FPackageName::LongPackageNameToFilename(Path); // Gather the package files in that directory and subdirectories TArray<FString> Filenames; FPackageName::FindPackagesInDirectory(Filenames, FilePath); for (int32 FilenameIdx = 0; FilenameIdx < Filenames.Num(); FilenameIdx++) { FString Filename(Filenames[FilenameIdx]); if ( IsValidPackageFileToRead(Filename) ) { // Add the path to this asset into the list of discovered paths const FString LongPackageName = FPackageName::FilenameToLongPackageName(Filename); LocalDiscoveredPathsSet.Add( FPackageName::GetLongPackagePath(LongPackageName) ); DiscoveredFilesToSearch.Add(Filename); } } } // Turn the set into an array here before the critical section below TArray<FString> LocalDiscoveredPathsArray = LocalDiscoveredPathsSet.Array(); { // Place all the discovered files into the files to search list FScopeLock CritSectionLock(&WorkerThreadCriticalSection); FilesToSearch.Append(DiscoveredFilesToSearch); DiscoveredPaths.Append(LocalDiscoveredPathsArray); bIsDiscoveringFiles = false; } } }
void FAssetDataGatherer::AddFilesToSearch(const TArray<FString>& Files) { TArray<FString> FilesToAdd; for (int32 FilenameIdx = 0; FilenameIdx < Files.Num(); FilenameIdx++) { FString Filename(Files[FilenameIdx]); if ( IsValidPackageFileToRead(Filename) ) { // Add the path to this asset into the list of discovered paths FilesToAdd.Add(Filename); } } { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); FilesToSearch.Append(FilesToAdd); } }
void FAssetDataGatherer::AddFilesToSearch(const TArray<FString>& Files) { TArray<FString> FilesToAdd; for (const FString& Filename : Files) { if ( IsValidPackageFileToRead(Filename) ) { // Add the path to this asset into the list of discovered paths FilesToAdd.Add(Filename); } } if (FilesToAdd.Num() > 0) { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); FilesToSearch.Append(FilesToAdd); } }
uint32 FAssetDataDiscovery::Run() { double DiscoverStartTime = FPlatformTime::Seconds(); int32 NumDiscoveredFiles = 0; FString LocalFilenamePathToPrioritize; TSet<FString> LocalDiscoveredPathsSet; TArray<FString> LocalDiscoveredDirectories; TArray<FDiscoveredPackageFile> LocalPriorityFilesToSearch; TArray<FDiscoveredPackageFile> LocalNonPriorityFilesToSearch; // This set contains the folders that we should hide by default unless they contain assets TSet<FString> PathsToHideIfEmpty; PathsToHideIfEmpty.Add(TEXT("/Game/Collections")); auto FlushLocalResultsIfRequired = [&]() { if (LocalPriorityFilesToSearch.Num() > 0 || LocalNonPriorityFilesToSearch.Num() > 0 || LocalDiscoveredPathsSet.Num() > 0) { TArray<FString> LocalDiscoveredPathsArray = LocalDiscoveredPathsSet.Array(); { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); // Place all the discovered files into the files to search list DiscoveredPaths.Append(MoveTemp(LocalDiscoveredPathsArray)); PriorityDiscoveredFiles.Append(MoveTemp(LocalPriorityFilesToSearch)); NonPriorityDiscoveredFiles.Append(MoveTemp(LocalNonPriorityFilesToSearch)); } } LocalDiscoveredPathsSet.Reset(); LocalPriorityFilesToSearch.Reset(); LocalNonPriorityFilesToSearch.Reset(); }; auto IsPriorityFile = [&](const FString& InPackageFilename) -> bool { return !bIsSynchronous && !LocalFilenamePathToPrioritize.IsEmpty() && InPackageFilename.StartsWith(LocalFilenamePathToPrioritize); }; auto OnIterateDirectoryItem = [&](const TCHAR* InPackageFilename, const FFileStatData& InPackageStatData) -> bool { if (StopTaskCounter.GetValue() != 0) { // Requested to stop - break out of the directory iteration return false; } const FString PackageFilenameStr = InPackageFilename; if (InPackageStatData.bIsDirectory) { LocalDiscoveredDirectories.Add(PackageFilenameStr / TEXT("")); FString PackagePath; if (FPackageName::TryConvertFilenameToLongPackageName(PackageFilenameStr, PackagePath) && !PathsToHideIfEmpty.Contains(PackagePath)) { LocalDiscoveredPathsSet.Add(PackagePath); } } else if (FPackageName::IsPackageFilename(PackageFilenameStr)) { if (IsValidPackageFileToRead(PackageFilenameStr)) { const FString LongPackageNameStr = FPackageName::FilenameToLongPackageName(PackageFilenameStr); if (IsPriorityFile(PackageFilenameStr)) { LocalPriorityFilesToSearch.Add(FDiscoveredPackageFile(PackageFilenameStr, InPackageStatData.ModificationTime)); } else { LocalNonPriorityFilesToSearch.Add(FDiscoveredPackageFile(PackageFilenameStr, InPackageStatData.ModificationTime)); } LocalDiscoveredPathsSet.Add(FPackageName::GetLongPackagePath(LongPackageNameStr)); ++NumDiscoveredFiles; // Flush the data if we've processed enough if (!bIsSynchronous && (LocalPriorityFilesToSearch.Num() + LocalNonPriorityFilesToSearch.Num()) >= AssetDataGathererConstants::MaxFilesToDiscoverBeforeFlush) { FlushLocalResultsIfRequired(); } } } return true; }; bool bIsIdle = true; while (StopTaskCounter.GetValue() == 0) { FString LocalDirectoryToSearch; { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); if (DirectoriesToSearch.Num() > 0) { bIsDiscoveringFiles = true; LocalFilenamePathToPrioritize = FilenamePathToPrioritize; // Pop off the first path to search LocalDirectoryToSearch = DirectoriesToSearch[0]; DirectoriesToSearch.RemoveAt(0, 1, false); } } if (LocalDirectoryToSearch.Len() > 0) { if (bIsIdle) { bIsIdle = false; // About to start work - reset these DiscoverStartTime = FPlatformTime::Seconds(); NumDiscoveredFiles = 0; } // Iterate the current search directory FLambdaDirectoryStatVisitor Visitor(OnIterateDirectoryItem); IFileManager::Get().IterateDirectoryStat(*LocalDirectoryToSearch, Visitor); { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); // Push back any newly discovered sub-directories if (LocalDiscoveredDirectories.Num() > 0) { // Use LocalDiscoveredDirectories as scratch space, then move it back out - this puts the directories we just // discovered at the start of the list for the next iteration, which can help with disk locality LocalDiscoveredDirectories.Append(MoveTemp(DirectoriesToSearch)); DirectoriesToSearch = MoveTemp(LocalDiscoveredDirectories); } LocalDiscoveredDirectories.Reset(); if (!bIsSynchronous) { FlushLocalResultsIfRequired(); SortPathsByPriority(1); } } } else { if (!bIsIdle) { bIsIdle = true; { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); bIsDiscoveringFiles = false; } UE_LOG(LogAssetRegistry, Verbose, TEXT("Discovery took %0.6f seconds and found %d files to process"), FPlatformTime::Seconds() - DiscoverStartTime, NumDiscoveredFiles); } // Ran out of things to do... if we have any pending results, flush those now FlushLocalResultsIfRequired(); if (bIsSynchronous) { // This is synchronous. Since our work is done, we should safely exit Stop(); } else { // No work to do. Sleep for a little and try again later. FPlatformProcess::Sleep(0.1); } } } return 0; }