uint32 FAssetDataGatherer::Run()
{
int32 CacheSerializationVersion = CACHE_SERIALIZATION_VERSION;
static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets();
if ( bUsingWorldAssets )
{
// Bump the serialization version to refresh the cache when switching between -WorldAssets and without.
// This is a temporary hack just while WorldAssets are under development
CacheSerializationVersion++;
}
if ( bLoadAndSaveCache )
{
// load the cached data
FNameTableArchiveReader CachedAssetDataReader;
if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion))
{
SerializeCache(CachedAssetDataReader);
}
}
TArray<FString> LocalFilesToSearch;
TArray<IGatheredAssetData*> LocalAssetResults;
TArray<FPackageDependencyData> LocalDependencyResults;
while ( StopTaskCounter.GetValue() == 0 )
{
// Check to see if there are any paths that need scanning for files. On the first iteration, there will always
// be work to do here. Later, if new paths are added on the fly, we'll also process those.
DiscoverFilesToSearch();
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
if ( LocalAssetResults.Num() )
{
AssetResults.Append(LocalAssetResults);
}
if ( LocalDependencyResults.Num() )
{
DependencyResults.Append(LocalDependencyResults);
}
if ( FilesToSearch.Num() )
{
if (SearchStartTime == 0)
{
SearchStartTime = FPlatformTime::Seconds();
}
const int32 NumFilesToProcess = FMath::Min<int32>(MAX_FILES_TO_PROCESS_BEFORE_FLUSH, FilesToSearch.Num());
for (int32 FileIdx = 0; FileIdx < NumFilesToProcess; ++FileIdx)
{
LocalFilesToSearch.Add(FilesToSearch[FileIdx]);
}
FilesToSearch.RemoveAt(0, NumFilesToProcess);
}
else if (SearchStartTime != 0)
{
SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime);
SearchStartTime = 0;
}
}
if ( LocalAssetResults.Num() )
{
LocalAssetResults.Empty();
}
if ( LocalDependencyResults.Num() )
{
LocalDependencyResults.Empty();
}
if ( LocalFilesToSearch.Num() )
{
for (int32 FileIdx = 0; FileIdx < LocalFilesToSearch.Num(); ++FileIdx)
{
const FString& AssetFile = LocalFilesToSearch[FileIdx];
if ( StopTaskCounter.GetValue() != 0 )
{
// We have been asked to stop, so don't read any more files
break;
}
bool bLoadedFromCache = false;
if ( bLoadAndSaveCache )
{
const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile));
FDiskCachedAssetData** DiskCachedAssetDataPtr = DiskCachedAssetDataMap.Find(PackageName);
FDiskCachedAssetData* DiskCachedAssetData = NULL;
if ( DiskCachedAssetDataPtr && *DiskCachedAssetDataPtr )
{
const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile);
const FDateTime& CachedTimestamp = (*DiskCachedAssetDataPtr)->Timestamp;
if ( FileTimestamp == CachedTimestamp )
{
DiskCachedAssetData = *DiskCachedAssetDataPtr;
}
}
if ( DiskCachedAssetData )
{
for ( auto CacheIt = DiskCachedAssetData->AssetDataList.CreateConstIterator(); CacheIt; ++CacheIt )
{
LocalAssetResults.Add(new FAssetDataWrapper(*CacheIt));
}
LocalDependencyResults.Add(DiskCachedAssetData->DependencyData);
NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData);
bLoadedFromCache = true;
}
}
if ( !bLoadedFromCache )
{
TArray<FBackgroundAssetData*> AssetDataFromFile;
FPackageDependencyData DependencyData;
if ( ReadAssetFile(AssetFile, AssetDataFromFile, DependencyData) )
{
LocalAssetResults.Append(AssetDataFromFile);
LocalDependencyResults.Add(DependencyData);
if ( bLoadAndSaveCache )
{
// Update the cache
const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile));
const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile);
FDiskCachedAssetData* NewData = new FDiskCachedAssetData(PackageName, FileTimestamp);
for ( auto AssetIt = AssetDataFromFile.CreateConstIterator(); AssetIt; ++AssetIt )
{
NewData->AssetDataList.Add((*AssetIt)->ToAssetData());
}
NewData->DependencyData = DependencyData;
NewCachedAssetData.Add(NewData);
NewCachedAssetDataMap.Add(PackageName, NewData);
}
}
}
}
LocalFilesToSearch.Empty();
}
else
{
if (bIsSynchronous)
{
// This is synchronous. Since our work is done, we should safely exit
Stop();
}
else
{
// If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly
if (bLoadAndSaveCache && !bSavedCacheAfterInitialDiscovery)
{
FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion);
SerializeCache(CachedAssetDataWriter);
CachedAssetDataWriter.SaveToFile(*CacheFilename);
bSavedCacheAfterInitialDiscovery = true;
}
// No work to do. Sleep for a little and try again later.
FPlatformProcess::Sleep(0.1);
}
}
}
if ( bLoadAndSaveCache )
{
FNameTableArchiveWriter CachedAssetData(CacheSerializationVersion);
SerializeCache(CachedAssetData);
CachedAssetData.SaveToFile(*CacheFilename);
}
return 0;
}
uint32 FAssetDataGatherer::Run()
{
int32 CacheSerializationVersion = AssetDataGathererConstants::CacheSerializationVersion;
static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets();
if ( bUsingWorldAssets )
{
// Bump the serialization version to refresh the cache when switching between -WorldAssets and without.
// This is a temporary hack just while WorldAssets are under development
CacheSerializationVersion++;
}
if ( bLoadAndSaveCache )
{
// load the cached data
FNameTableArchiveReader CachedAssetDataReader;
if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion))
{
SerializeCache(CachedAssetDataReader);
}
}
TArray<FDiscoveredPackageFile> LocalFilesToSearch;
TArray<FAssetData*> LocalAssetResults;
TArray<FPackageDependencyData> LocalDependencyResults;
TArray<FString> LocalCookedPackageNamesWithoutAssetDataResults;
const double InitialScanStartTime = FPlatformTime::Seconds();
int32 NumCachedFiles = 0;
int32 NumUncachedFiles = 0;
int32 NumFilesProcessedSinceLastCacheSave = 0;
auto WriteAssetCacheFile = [&]()
{
FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion, CacheFilename);
SerializeCache(CachedAssetDataWriter);
NumFilesProcessedSinceLastCacheSave = 0;
};
while ( StopTaskCounter.GetValue() == 0 )
{
bool LocalIsDiscoveringFiles = false;
{
FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
// Grab any new package files from the background directory scan
if (BackgroundPackageFileDiscovery.IsValid())
{
bIsDiscoveringFiles = BackgroundPackageFileDiscovery->GetAndTrimSearchResults(DiscoveredPaths, FilesToSearch, NumPathsToSearchAtLastSyncPoint);
LocalIsDiscoveringFiles = bIsDiscoveringFiles;
}
AssetResults.Append(MoveTemp(LocalAssetResults));
DependencyResults.Append(MoveTemp(LocalDependencyResults));
CookedPackageNamesWithoutAssetDataResults.Append(MoveTemp(LocalCookedPackageNamesWithoutAssetDataResults));
if (FilesToSearch.Num() > 0)
{
if (SearchStartTime == 0)
{
SearchStartTime = FPlatformTime::Seconds();
}
const int32 NumFilesToProcess = FMath::Min<int32>(AssetDataGathererConstants::MaxFilesToGatherBeforeFlush, FilesToSearch.Num());
LocalFilesToSearch.Append(FilesToSearch.GetData(), NumFilesToProcess);
FilesToSearch.RemoveAt(0, NumFilesToProcess, false);
}
else if (SearchStartTime != 0 && !LocalIsDiscoveringFiles)
{
SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime);
SearchStartTime = 0;
}
}
LocalAssetResults.Reset();
LocalDependencyResults.Reset();
LocalCookedPackageNamesWithoutAssetDataResults.Reset();
if (LocalFilesToSearch.Num() > 0)
{
for (const FDiscoveredPackageFile& AssetFileData : LocalFilesToSearch)
{
if (StopTaskCounter.GetValue() != 0)
{
// We have been asked to stop, so don't read any more files
break;
}
const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFileData.PackageFilename));
bool bLoadedFromCache = false;
if (bLoadAndSaveCache)
{
FDiskCachedAssetData* DiskCachedAssetData = DiskCachedAssetDataMap.Find(PackageName);
if (DiskCachedAssetData)
{
const FDateTime& CachedTimestamp = DiskCachedAssetData->Timestamp;
if (AssetFileData.PackageTimestamp != CachedTimestamp)
{
DiskCachedAssetData = nullptr;
}
}
if (DiskCachedAssetData)
{
bLoadedFromCache = true;
++NumCachedFiles;
LocalAssetResults.Reserve(LocalAssetResults.Num() + DiskCachedAssetData->AssetDataList.Num());
for (const FAssetData& AssetData : DiskCachedAssetData->AssetDataList)
{
LocalAssetResults.Add(new FAssetData(AssetData));
}
LocalDependencyResults.Add(DiskCachedAssetData->DependencyData);
NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData);
}
}
if (!bLoadedFromCache)
{
TArray<FAssetData*> AssetDataFromFile;
FPackageDependencyData DependencyData;
TArray<FString> CookedPackageNamesWithoutAssetData;
if (ReadAssetFile(AssetFileData.PackageFilename, AssetDataFromFile, DependencyData, CookedPackageNamesWithoutAssetData))
{
++NumUncachedFiles;
LocalAssetResults.Append(AssetDataFromFile);
LocalDependencyResults.Add(DependencyData);
LocalCookedPackageNamesWithoutAssetDataResults.Append(CookedPackageNamesWithoutAssetData);
// Don't store info on cooked packages
bool bCachePackage = bLoadAndSaveCache && LocalCookedPackageNamesWithoutAssetDataResults.Num() == 0;
if (bCachePackage)
{
// Don't store info on cooked packages
for (const auto& AssetData : AssetDataFromFile)
{
if (!!(AssetData->PackageFlags & PKG_FilterEditorOnly))
{
bCachePackage = false;
break;
}
}
}
if (bCachePackage)
{
++NumFilesProcessedSinceLastCacheSave;
// Update the cache
FDiskCachedAssetData* NewData = new FDiskCachedAssetData(AssetFileData.PackageTimestamp);
NewData->AssetDataList.Reserve(AssetDataFromFile.Num());
for (const FAssetData* BackgroundAssetData : AssetDataFromFile)
{
NewData->AssetDataList.Add(*BackgroundAssetData);
}
NewData->DependencyData = DependencyData;
NewCachedAssetData.Add(NewData);
NewCachedAssetDataMap.Add(PackageName, NewData);
}
}
}
}
LocalFilesToSearch.Reset();
if (bLoadAndSaveCache)
{
// Save off the cache files if we're processed enough data since the last save
if (NumFilesProcessedSinceLastCacheSave >= AssetDataGathererConstants::MaxFilesToProcessBeforeCacheWrite)
{
WriteAssetCacheFile();
}
}
}
else
{
if (bIsSynchronous)
{
// This is synchronous. Since our work is done, we should safely exit
Stop();
}
else
{
if (!LocalIsDiscoveringFiles && !bFinishedInitialDiscovery)
{
bFinishedInitialDiscovery = true;
UE_LOG(LogAssetRegistry, Verbose, TEXT("Initial scan took %0.6f seconds (found %d cached assets, and loaded %d)"), FPlatformTime::Seconds() - InitialScanStartTime, NumCachedFiles, NumUncachedFiles);
// If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly
if (bLoadAndSaveCache)
{
WriteAssetCacheFile();
}
}
// No work to do. Sleep for a little and try again later.
FPlatformProcess::Sleep(0.1);
}
}
}
if ( bLoadAndSaveCache )
{
WriteAssetCacheFile();
}
return 0;
}
