uint32 FAssetDataGatherer::Run() { int32 CacheSerializationVersion = CACHE_SERIALIZATION_VERSION; static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets(); if ( bUsingWorldAssets ) { // Bump the serialization version to refresh the cache when switching between -WorldAssets and without. // This is a temporary hack just while WorldAssets are under development CacheSerializationVersion++; } if ( bLoadAndSaveCache ) { // load the cached data FNameTableArchiveReader CachedAssetDataReader; if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion)) { SerializeCache(CachedAssetDataReader); } } TArray<FString> LocalFilesToSearch; TArray<IGatheredAssetData*> LocalAssetResults; TArray<FPackageDependencyData> LocalDependencyResults; while ( StopTaskCounter.GetValue() == 0 ) { // Check to see if there are any paths that need scanning for files. On the first iteration, there will always // be work to do here. Later, if new paths are added on the fly, we'll also process those. DiscoverFilesToSearch(); { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); if ( LocalAssetResults.Num() ) { AssetResults.Append(LocalAssetResults); } if ( LocalDependencyResults.Num() ) { DependencyResults.Append(LocalDependencyResults); } if ( FilesToSearch.Num() ) { if (SearchStartTime == 0) { SearchStartTime = FPlatformTime::Seconds(); } const int32 NumFilesToProcess = FMath::Min<int32>(MAX_FILES_TO_PROCESS_BEFORE_FLUSH, FilesToSearch.Num()); for (int32 FileIdx = 0; FileIdx < NumFilesToProcess; ++FileIdx) { LocalFilesToSearch.Add(FilesToSearch[FileIdx]); } FilesToSearch.RemoveAt(0, NumFilesToProcess); } else if (SearchStartTime != 0) { SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime); SearchStartTime = 0; } } if ( LocalAssetResults.Num() ) { LocalAssetResults.Empty(); } if ( LocalDependencyResults.Num() ) { LocalDependencyResults.Empty(); } if ( LocalFilesToSearch.Num() ) { for (int32 FileIdx = 0; FileIdx < LocalFilesToSearch.Num(); ++FileIdx) { const FString& AssetFile = LocalFilesToSearch[FileIdx]; if ( StopTaskCounter.GetValue() != 0 ) { // We have been asked to stop, so don't read any more files break; } bool bLoadedFromCache = false; if ( bLoadAndSaveCache ) { const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile)); FDiskCachedAssetData** DiskCachedAssetDataPtr = DiskCachedAssetDataMap.Find(PackageName); FDiskCachedAssetData* DiskCachedAssetData = NULL; if ( DiskCachedAssetDataPtr && *DiskCachedAssetDataPtr ) { const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile); const FDateTime& CachedTimestamp = (*DiskCachedAssetDataPtr)->Timestamp; if ( FileTimestamp == CachedTimestamp ) { DiskCachedAssetData = *DiskCachedAssetDataPtr; } } if ( DiskCachedAssetData ) { for ( auto CacheIt = DiskCachedAssetData->AssetDataList.CreateConstIterator(); CacheIt; ++CacheIt ) { LocalAssetResults.Add(new FAssetDataWrapper(*CacheIt)); } LocalDependencyResults.Add(DiskCachedAssetData->DependencyData); NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData); bLoadedFromCache = true; } } if ( !bLoadedFromCache ) { TArray<FBackgroundAssetData*> AssetDataFromFile; FPackageDependencyData DependencyData; if ( ReadAssetFile(AssetFile, AssetDataFromFile, DependencyData) ) { LocalAssetResults.Append(AssetDataFromFile); LocalDependencyResults.Add(DependencyData); if ( bLoadAndSaveCache ) { // Update the cache const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile)); const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile); FDiskCachedAssetData* NewData = new FDiskCachedAssetData(PackageName, FileTimestamp); for ( auto AssetIt = AssetDataFromFile.CreateConstIterator(); AssetIt; ++AssetIt ) { NewData->AssetDataList.Add((*AssetIt)->ToAssetData()); } NewData->DependencyData = DependencyData; NewCachedAssetData.Add(NewData); NewCachedAssetDataMap.Add(PackageName, NewData); } } } } LocalFilesToSearch.Empty(); } else { if (bIsSynchronous) { // This is synchronous. Since our work is done, we should safely exit Stop(); } else { // If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly if (bLoadAndSaveCache && !bSavedCacheAfterInitialDiscovery) { FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion); SerializeCache(CachedAssetDataWriter); CachedAssetDataWriter.SaveToFile(*CacheFilename); bSavedCacheAfterInitialDiscovery = true; } // No work to do. Sleep for a little and try again later. FPlatformProcess::Sleep(0.1); } } } if ( bLoadAndSaveCache ) { FNameTableArchiveWriter CachedAssetData(CacheSerializationVersion); SerializeCache(CachedAssetData); CachedAssetData.SaveToFile(*CacheFilename); } return 0; }
uint32 FAssetDataGatherer::Run() { int32 CacheSerializationVersion = AssetDataGathererConstants::CacheSerializationVersion; static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets(); if ( bUsingWorldAssets ) { // Bump the serialization version to refresh the cache when switching between -WorldAssets and without. // This is a temporary hack just while WorldAssets are under development CacheSerializationVersion++; } if ( bLoadAndSaveCache ) { // load the cached data FNameTableArchiveReader CachedAssetDataReader; if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion)) { SerializeCache(CachedAssetDataReader); } } TArray<FDiscoveredPackageFile> LocalFilesToSearch; TArray<FAssetData*> LocalAssetResults; TArray<FPackageDependencyData> LocalDependencyResults; TArray<FString> LocalCookedPackageNamesWithoutAssetDataResults; const double InitialScanStartTime = FPlatformTime::Seconds(); int32 NumCachedFiles = 0; int32 NumUncachedFiles = 0; int32 NumFilesProcessedSinceLastCacheSave = 0; auto WriteAssetCacheFile = [&]() { FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion, CacheFilename); SerializeCache(CachedAssetDataWriter); NumFilesProcessedSinceLastCacheSave = 0; }; while ( StopTaskCounter.GetValue() == 0 ) { bool LocalIsDiscoveringFiles = false; { FScopeLock CritSectionLock(&WorkerThreadCriticalSection); // Grab any new package files from the background directory scan if (BackgroundPackageFileDiscovery.IsValid()) { bIsDiscoveringFiles = BackgroundPackageFileDiscovery->GetAndTrimSearchResults(DiscoveredPaths, FilesToSearch, NumPathsToSearchAtLastSyncPoint); LocalIsDiscoveringFiles = bIsDiscoveringFiles; } AssetResults.Append(MoveTemp(LocalAssetResults)); DependencyResults.Append(MoveTemp(LocalDependencyResults)); CookedPackageNamesWithoutAssetDataResults.Append(MoveTemp(LocalCookedPackageNamesWithoutAssetDataResults)); if (FilesToSearch.Num() > 0) { if (SearchStartTime == 0) { SearchStartTime = FPlatformTime::Seconds(); } const int32 NumFilesToProcess = FMath::Min<int32>(AssetDataGathererConstants::MaxFilesToGatherBeforeFlush, FilesToSearch.Num()); LocalFilesToSearch.Append(FilesToSearch.GetData(), NumFilesToProcess); FilesToSearch.RemoveAt(0, NumFilesToProcess, false); } else if (SearchStartTime != 0 && !LocalIsDiscoveringFiles) { SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime); SearchStartTime = 0; } } LocalAssetResults.Reset(); LocalDependencyResults.Reset(); LocalCookedPackageNamesWithoutAssetDataResults.Reset(); if (LocalFilesToSearch.Num() > 0) { for (const FDiscoveredPackageFile& AssetFileData : LocalFilesToSearch) { if (StopTaskCounter.GetValue() != 0) { // We have been asked to stop, so don't read any more files break; } const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFileData.PackageFilename)); bool bLoadedFromCache = false; if (bLoadAndSaveCache) { FDiskCachedAssetData* DiskCachedAssetData = DiskCachedAssetDataMap.Find(PackageName); if (DiskCachedAssetData) { const FDateTime& CachedTimestamp = DiskCachedAssetData->Timestamp; if (AssetFileData.PackageTimestamp != CachedTimestamp) { DiskCachedAssetData = nullptr; } } if (DiskCachedAssetData) { bLoadedFromCache = true; ++NumCachedFiles; LocalAssetResults.Reserve(LocalAssetResults.Num() + DiskCachedAssetData->AssetDataList.Num()); for (const FAssetData& AssetData : DiskCachedAssetData->AssetDataList) { LocalAssetResults.Add(new FAssetData(AssetData)); } LocalDependencyResults.Add(DiskCachedAssetData->DependencyData); NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData); } } if (!bLoadedFromCache) { TArray<FAssetData*> AssetDataFromFile; FPackageDependencyData DependencyData; TArray<FString> CookedPackageNamesWithoutAssetData; if (ReadAssetFile(AssetFileData.PackageFilename, AssetDataFromFile, DependencyData, CookedPackageNamesWithoutAssetData)) { ++NumUncachedFiles; LocalAssetResults.Append(AssetDataFromFile); LocalDependencyResults.Add(DependencyData); LocalCookedPackageNamesWithoutAssetDataResults.Append(CookedPackageNamesWithoutAssetData); // Don't store info on cooked packages bool bCachePackage = bLoadAndSaveCache && LocalCookedPackageNamesWithoutAssetDataResults.Num() == 0; if (bCachePackage) { // Don't store info on cooked packages for (const auto& AssetData : AssetDataFromFile) { if (!!(AssetData->PackageFlags & PKG_FilterEditorOnly)) { bCachePackage = false; break; } } } if (bCachePackage) { ++NumFilesProcessedSinceLastCacheSave; // Update the cache FDiskCachedAssetData* NewData = new FDiskCachedAssetData(AssetFileData.PackageTimestamp); NewData->AssetDataList.Reserve(AssetDataFromFile.Num()); for (const FAssetData* BackgroundAssetData : AssetDataFromFile) { NewData->AssetDataList.Add(*BackgroundAssetData); } NewData->DependencyData = DependencyData; NewCachedAssetData.Add(NewData); NewCachedAssetDataMap.Add(PackageName, NewData); } } } } LocalFilesToSearch.Reset(); if (bLoadAndSaveCache) { // Save off the cache files if we're processed enough data since the last save if (NumFilesProcessedSinceLastCacheSave >= AssetDataGathererConstants::MaxFilesToProcessBeforeCacheWrite) { WriteAssetCacheFile(); } } } else { if (bIsSynchronous) { // This is synchronous. Since our work is done, we should safely exit Stop(); } else { if (!LocalIsDiscoveringFiles && !bFinishedInitialDiscovery) { bFinishedInitialDiscovery = true; UE_LOG(LogAssetRegistry, Verbose, TEXT("Initial scan took %0.6f seconds (found %d cached assets, and loaded %d)"), FPlatformTime::Seconds() - InitialScanStartTime, NumCachedFiles, NumUncachedFiles); // If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly if (bLoadAndSaveCache) { WriteAssetCacheFile(); } } // No work to do. Sleep for a little and try again later. FPlatformProcess::Sleep(0.1); } } } if ( bLoadAndSaveCache ) { WriteAssetCacheFile(); } return 0; }