uint32 FAssetDataGatherer::Run()
{
	int32 CacheSerializationVersion = CACHE_SERIALIZATION_VERSION;
	
	static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets();
	if ( bUsingWorldAssets )
	{
		// Bump the serialization version to refresh the cache when switching between -WorldAssets and without.
		// This is a temporary hack just while WorldAssets are under development
		CacheSerializationVersion++;
	}

	if ( bLoadAndSaveCache )
	{
		// load the cached data
		FNameTableArchiveReader CachedAssetDataReader;
		if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion))
		{
			SerializeCache(CachedAssetDataReader);
		}
	}

	TArray<FString> LocalFilesToSearch;
	TArray<IGatheredAssetData*> LocalAssetResults;
	TArray<FPackageDependencyData> LocalDependencyResults;

	while ( StopTaskCounter.GetValue() == 0 )
	{
		// Check to see if there are any paths that need scanning for files.  On the first iteration, there will always
		// be work to do here.  Later, if new paths are added on the fly, we'll also process those.
		DiscoverFilesToSearch();

		{
			FScopeLock CritSectionLock(&WorkerThreadCriticalSection);
			if ( LocalAssetResults.Num() )
			{
				AssetResults.Append(LocalAssetResults);
			}

			if ( LocalDependencyResults.Num() )
			{
				DependencyResults.Append(LocalDependencyResults);
			}

			if ( FilesToSearch.Num() )
			{
				if (SearchStartTime == 0)
				{
					SearchStartTime = FPlatformTime::Seconds();
				}

				const int32 NumFilesToProcess = FMath::Min<int32>(MAX_FILES_TO_PROCESS_BEFORE_FLUSH, FilesToSearch.Num());

				for (int32 FileIdx = 0; FileIdx < NumFilesToProcess; ++FileIdx)
				{
					LocalFilesToSearch.Add(FilesToSearch[FileIdx]);
				}

				FilesToSearch.RemoveAt(0, NumFilesToProcess);
			}
			else if (SearchStartTime != 0)
			{
				SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime);
				SearchStartTime = 0;
			}
		}

		if ( LocalAssetResults.Num() )
		{
			LocalAssetResults.Empty();
		}

		if ( LocalDependencyResults.Num() )
		{
			LocalDependencyResults.Empty();
		}

		if ( LocalFilesToSearch.Num() )
		{
			for (int32 FileIdx = 0; FileIdx < LocalFilesToSearch.Num(); ++FileIdx)
			{
				const FString& AssetFile = LocalFilesToSearch[FileIdx];

				if ( StopTaskCounter.GetValue() != 0 )
				{
					// We have been asked to stop, so don't read any more files
					break;
				}

				bool bLoadedFromCache = false;
				if ( bLoadAndSaveCache )
				{
					const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile));
					FDiskCachedAssetData** DiskCachedAssetDataPtr = DiskCachedAssetDataMap.Find(PackageName);
					FDiskCachedAssetData* DiskCachedAssetData = NULL;
					if ( DiskCachedAssetDataPtr && *DiskCachedAssetDataPtr )
					{
						const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile);
						const FDateTime& CachedTimestamp = (*DiskCachedAssetDataPtr)->Timestamp;
						if ( FileTimestamp == CachedTimestamp )
						{
							DiskCachedAssetData = *DiskCachedAssetDataPtr;
						}
					}

					if ( DiskCachedAssetData )
					{
						for ( auto CacheIt = DiskCachedAssetData->AssetDataList.CreateConstIterator(); CacheIt; ++CacheIt )
						{
							LocalAssetResults.Add(new FAssetDataWrapper(*CacheIt));
						}

						LocalDependencyResults.Add(DiskCachedAssetData->DependencyData);

						NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData);
						bLoadedFromCache = true;
					}
				}

				if ( !bLoadedFromCache )
				{
					TArray<FBackgroundAssetData*> AssetDataFromFile;
					FPackageDependencyData DependencyData;
					if ( ReadAssetFile(AssetFile, AssetDataFromFile, DependencyData) )
					{
						LocalAssetResults.Append(AssetDataFromFile);
						LocalDependencyResults.Add(DependencyData);

						if ( bLoadAndSaveCache )
						{
							// Update the cache
							const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFile));
							const FDateTime& FileTimestamp = IFileManager::Get().GetTimeStamp(*AssetFile);
							FDiskCachedAssetData* NewData = new FDiskCachedAssetData(PackageName, FileTimestamp);
							for ( auto AssetIt = AssetDataFromFile.CreateConstIterator(); AssetIt; ++AssetIt )
							{
								NewData->AssetDataList.Add((*AssetIt)->ToAssetData());
							}
							NewData->DependencyData = DependencyData;
							NewCachedAssetData.Add(NewData);
							NewCachedAssetDataMap.Add(PackageName, NewData);
						}
					}
				}
			}

			LocalFilesToSearch.Empty();
		}
		else
		{
			if (bIsSynchronous)
			{
				// This is synchronous. Since our work is done, we should safely exit
				Stop();
			}
			else
			{
				// If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly
				if (bLoadAndSaveCache && !bSavedCacheAfterInitialDiscovery)
				{
					FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion);
					SerializeCache(CachedAssetDataWriter);
					CachedAssetDataWriter.SaveToFile(*CacheFilename);

					bSavedCacheAfterInitialDiscovery = true;
				}

				// No work to do. Sleep for a little and try again later.
				FPlatformProcess::Sleep(0.1);
			}
		}
	}

	if ( bLoadAndSaveCache )
	{
		FNameTableArchiveWriter CachedAssetData(CacheSerializationVersion);
		SerializeCache(CachedAssetData);
		CachedAssetData.SaveToFile(*CacheFilename);
	}

	return 0;
}
uint32 FAssetDataGatherer::Run()
{
	int32 CacheSerializationVersion = AssetDataGathererConstants::CacheSerializationVersion;
	
	static const bool bUsingWorldAssets = FAssetRegistry::IsUsingWorldAssets();
	if ( bUsingWorldAssets )
	{
		// Bump the serialization version to refresh the cache when switching between -WorldAssets and without.
		// This is a temporary hack just while WorldAssets are under development
		CacheSerializationVersion++;
	}

	if ( bLoadAndSaveCache )
	{
		// load the cached data
		FNameTableArchiveReader CachedAssetDataReader;
		if (CachedAssetDataReader.LoadFile(*CacheFilename, CacheSerializationVersion))
		{
			SerializeCache(CachedAssetDataReader);
		}
	}

	TArray<FDiscoveredPackageFile> LocalFilesToSearch;
	TArray<FAssetData*> LocalAssetResults;
	TArray<FPackageDependencyData> LocalDependencyResults;
	TArray<FString> LocalCookedPackageNamesWithoutAssetDataResults;

	const double InitialScanStartTime = FPlatformTime::Seconds();
	int32 NumCachedFiles = 0;
	int32 NumUncachedFiles = 0;

	int32 NumFilesProcessedSinceLastCacheSave = 0;
	auto WriteAssetCacheFile = [&]()
	{
		FNameTableArchiveWriter CachedAssetDataWriter(CacheSerializationVersion, CacheFilename);
		SerializeCache(CachedAssetDataWriter);

		NumFilesProcessedSinceLastCacheSave = 0;
	};

	while ( StopTaskCounter.GetValue() == 0 )
	{
		bool LocalIsDiscoveringFiles = false;

		{
			FScopeLock CritSectionLock(&WorkerThreadCriticalSection);

			// Grab any new package files from the background directory scan
			if (BackgroundPackageFileDiscovery.IsValid())
			{
				bIsDiscoveringFiles = BackgroundPackageFileDiscovery->GetAndTrimSearchResults(DiscoveredPaths, FilesToSearch, NumPathsToSearchAtLastSyncPoint);
				LocalIsDiscoveringFiles = bIsDiscoveringFiles;
			}

			AssetResults.Append(MoveTemp(LocalAssetResults));
			DependencyResults.Append(MoveTemp(LocalDependencyResults));
			CookedPackageNamesWithoutAssetDataResults.Append(MoveTemp(LocalCookedPackageNamesWithoutAssetDataResults));

			if (FilesToSearch.Num() > 0)
			{
				if (SearchStartTime == 0)
				{
					SearchStartTime = FPlatformTime::Seconds();
				}

				const int32 NumFilesToProcess = FMath::Min<int32>(AssetDataGathererConstants::MaxFilesToGatherBeforeFlush, FilesToSearch.Num());
				LocalFilesToSearch.Append(FilesToSearch.GetData(), NumFilesToProcess);
				FilesToSearch.RemoveAt(0, NumFilesToProcess, false);
			}
			else if (SearchStartTime != 0 && !LocalIsDiscoveringFiles)
			{
				SearchTimes.Add(FPlatformTime::Seconds() - SearchStartTime);
				SearchStartTime = 0;
			}
		}

		LocalAssetResults.Reset();
		LocalDependencyResults.Reset();
		LocalCookedPackageNamesWithoutAssetDataResults.Reset();

		if (LocalFilesToSearch.Num() > 0)
		{
			for (const FDiscoveredPackageFile& AssetFileData : LocalFilesToSearch)
			{
				if (StopTaskCounter.GetValue() != 0)
				{
					// We have been asked to stop, so don't read any more files
					break;
				}

				const FName PackageName = FName(*FPackageName::FilenameToLongPackageName(AssetFileData.PackageFilename));

				bool bLoadedFromCache = false;
				if (bLoadAndSaveCache)
				{
					FDiskCachedAssetData* DiskCachedAssetData = DiskCachedAssetDataMap.Find(PackageName);
					if (DiskCachedAssetData)
					{
						const FDateTime& CachedTimestamp = DiskCachedAssetData->Timestamp;
						if (AssetFileData.PackageTimestamp != CachedTimestamp)
						{
							DiskCachedAssetData = nullptr;
						}
					}

					if (DiskCachedAssetData)
					{
						bLoadedFromCache = true;

						++NumCachedFiles;

						LocalAssetResults.Reserve(LocalAssetResults.Num() + DiskCachedAssetData->AssetDataList.Num());
						for (const FAssetData& AssetData : DiskCachedAssetData->AssetDataList)
						{
							LocalAssetResults.Add(new FAssetData(AssetData));
						}

						LocalDependencyResults.Add(DiskCachedAssetData->DependencyData);

						NewCachedAssetDataMap.Add(PackageName, DiskCachedAssetData);
					}
				}

				if (!bLoadedFromCache)
				{
					TArray<FAssetData*> AssetDataFromFile;
					FPackageDependencyData DependencyData;
					TArray<FString> CookedPackageNamesWithoutAssetData;
					if (ReadAssetFile(AssetFileData.PackageFilename, AssetDataFromFile, DependencyData, CookedPackageNamesWithoutAssetData))
					{
						++NumUncachedFiles;

						LocalAssetResults.Append(AssetDataFromFile);
						LocalDependencyResults.Add(DependencyData);
						LocalCookedPackageNamesWithoutAssetDataResults.Append(CookedPackageNamesWithoutAssetData);

						// Don't store info on cooked packages
						bool bCachePackage = bLoadAndSaveCache && LocalCookedPackageNamesWithoutAssetDataResults.Num() == 0;
						if (bCachePackage)
						{
							// Don't store info on cooked packages
							for (const auto& AssetData : AssetDataFromFile)
							{
								if (!!(AssetData->PackageFlags & PKG_FilterEditorOnly))
								{
									bCachePackage = false;
									break;
								}
							}
						}

						if (bCachePackage)
						{
							++NumFilesProcessedSinceLastCacheSave;

							// Update the cache
							FDiskCachedAssetData* NewData = new FDiskCachedAssetData(AssetFileData.PackageTimestamp);
							NewData->AssetDataList.Reserve(AssetDataFromFile.Num());
							for (const FAssetData* BackgroundAssetData : AssetDataFromFile)
							{
								NewData->AssetDataList.Add(*BackgroundAssetData);
							}
							NewData->DependencyData = DependencyData;

							NewCachedAssetData.Add(NewData);
							NewCachedAssetDataMap.Add(PackageName, NewData);
						}
					}
				}
			}

			LocalFilesToSearch.Reset();

			if (bLoadAndSaveCache)
			{
				// Save off the cache files if we're processed enough data since the last save
				if (NumFilesProcessedSinceLastCacheSave >= AssetDataGathererConstants::MaxFilesToProcessBeforeCacheWrite)
				{
					WriteAssetCacheFile();
				}
			}
		}
		else
		{
			if (bIsSynchronous)
			{
				// This is synchronous. Since our work is done, we should safely exit
				Stop();
			}
			else
			{
				if (!LocalIsDiscoveringFiles && !bFinishedInitialDiscovery)
				{
					bFinishedInitialDiscovery = true;

					UE_LOG(LogAssetRegistry, Verbose, TEXT("Initial scan took %0.6f seconds (found %d cached assets, and loaded %d)"), FPlatformTime::Seconds() - InitialScanStartTime, NumCachedFiles, NumUncachedFiles);

					// If we are caching discovered assets and this is the first time we had no work to do, save off the cache now in case the user terminates unexpectedly
					if (bLoadAndSaveCache)
					{
						WriteAssetCacheFile();
					}
				}

				// No work to do. Sleep for a little and try again later.
				FPlatformProcess::Sleep(0.1);
			}
		}
	}

	if ( bLoadAndSaveCache )
	{
		WriteAssetCacheFile();
	}

	return 0;
}