bool FWorldTileInfo::Read(const FString& InPackageFileName, FWorldTileInfo& OutInfo)
{
// Fill with default information
OutInfo = FWorldTileInfo();
// Create a file reader to load the file
TScopedPointer<FArchive> FileReader(IFileManager::Get().CreateFileReader(*InPackageFileName));
if (FileReader == NULL)
{
// Couldn't open the file
return false;
}
// Read package file summary from the file
FPackageFileSummary FileSummary;
(*FileReader) << FileSummary;
// Make sure this is indeed a package
if (FileSummary.Tag != PACKAGE_FILE_TAG)
{
// Unrecognized or malformed package file
return false;
}
// Does the package contains a level info?
if (FileSummary.WorldTileInfoDataOffset != 0)
{
if (!!(FileSummary.PackageFlags & PKG_StoreCompressed))
{
check(FileSummary.CompressedChunks.Num() > 0);
if (!FileReader->SetCompressionMap(&FileSummary.CompressedChunks, (ECompressionFlags)FileSummary.CompressionFlags))
{
FileReader = new FArchiveAsync(*InPackageFileName); // re-assign scope pointer
check(!FileReader->IsError());
verify(FileReader->SetCompressionMap(&FileSummary.CompressedChunks, (ECompressionFlags)FileSummary.CompressionFlags));
}
}
// Seek the the part of the file where the structure lives
FileReader->Seek(FileSummary.WorldTileInfoDataOffset);
//make sure the filereader gets the correct version number (it defaults to latest version)
FileReader->SetUE4Ver(FileSummary.GetFileVersionUE4());
FileReader->SetEngineVer(FileSummary.EngineVersion);
// Load the structure
*FileReader << OutInfo;
}
return true;
}
int32 UGatherTextFromAssetsCommandlet::Main(const FString& Params)
{
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Modules to Preload
TArray<FString> ModulesToPreload;
GetStringArrayFromConfig(*SectionName, TEXT("ModulesToPreload"), ModulesToPreload, GatherTextConfigPath);
for (const FString& ModuleName : ModulesToPreload)
{
FModuleManager::Get().LoadModule(*ModuleName);
}
// IncludePathFilters
TArray<FString> IncludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePathFilters"), IncludePathFilters, GatherTextConfigPath);
// IncludePaths (DEPRECATED)
{
TArray<FString> IncludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num())
{
IncludePathFilters.Append(IncludePaths);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("IncludePaths detected in section %s. IncludePaths is deprecated, please use IncludePathFilters."), *SectionName);
}
}
if (IncludePathFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No include path filters in section %s."), *SectionName);
return -1;
}
// ExcludePathFilters
TArray<FString> ExcludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePathFilters"), ExcludePathFilters, GatherTextConfigPath);
// ExcludePaths (DEPRECATED)
{
TArray<FString> ExcludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
if (ExcludePaths.Num())
{
ExcludePathFilters.Append(ExcludePaths);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("ExcludePaths detected in section %s. ExcludePaths is deprecated, please use ExcludePathFilters."), *SectionName);
}
}
// PackageNameFilters
TArray<FString> PackageFileNameFilters;
GetStringArrayFromConfig(*SectionName, TEXT("PackageFileNameFilters"), PackageFileNameFilters, GatherTextConfigPath);
// PackageExtensions (DEPRECATED)
{
TArray<FString> PackageExtensions;
GetStringArrayFromConfig(*SectionName, TEXT("PackageExtensions"), PackageExtensions, GatherTextConfigPath);
if (PackageExtensions.Num())
{
PackageFileNameFilters.Append(PackageExtensions);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("PackageExtensions detected in section %s. PackageExtensions is deprecated, please use PackageFileNameFilters."), *SectionName);
}
}
if (PackageFileNameFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No package file name filters in section %s."), *SectionName);
return -1;
}
//asset class exclude
TArray<FString> ExcludeClasses;
GetStringArrayFromConfig(*SectionName, TEXT("ExcludeClasses"), ExcludeClasses, GatherTextConfigPath);
FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>(TEXT("AssetRegistry"));
AssetRegistryModule.Get().SearchAllAssets( true );
FARFilter Filter;
for(const auto& ExcludeClass : ExcludeClasses)
{
UClass* FilterClass = FindObject<UClass>(ANY_PACKAGE, *ExcludeClass);
if(FilterClass)
{
Filter.ClassNames.Add( FilterClass->GetFName() );
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Invalid exclude class %s"), *ExcludeClass);
}
}
TArray<FAssetData> AssetDataArray;
AssetRegistryModule.Get().GetAssets(Filter, AssetDataArray);
FString UAssetPackageExtension = FPackageName::GetAssetPackageExtension();
TSet< FString > LongPackageNamesToExclude;
for (int Index = 0; Index < AssetDataArray.Num(); Index++)
{
LongPackageNamesToExclude.Add( FPackageName::LongPackageNameToFilename( AssetDataArray[Index].PackageName.ToString(), UAssetPackageExtension ) );
}
//Get whether we should fix broken properties that we find.
if (!GetBoolFromConfig(*SectionName, TEXT("bFixBroken"), bFixBroken, GatherTextConfigPath))
{
bFixBroken = false;
}
// Get whether we should gather editor-only data. Typically only useful for the localization of UE4 itself.
if (!GetBoolFromConfig(*SectionName, TEXT("ShouldGatherFromEditorOnlyData"), ShouldGatherFromEditorOnlyData, GatherTextConfigPath))
{
ShouldGatherFromEditorOnlyData = false;
}
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetPathArrayFromConfig(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("The GatherTextFromAssets commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
//The main array of files to work from.
TArray< FString > PackageFileNamesToProcess;
TArray<FString> PackageFilesNotInIncludePath;
TArray<FString> PackageFilesInExcludePath;
TArray<FString> PackageFilesExcludedByClass;
//Fill the list of packages to work from.
uint8 PackageFilter = NORMALIZE_DefaultFlags;
TArray<FString> Unused;
for ( int32 PackageFilenameWildcardIdx = 0; PackageFilenameWildcardIdx < PackageFileNameFilters.Num(); PackageFilenameWildcardIdx++ )
{
const bool IsAssetPackage = PackageFileNameFilters[PackageFilenameWildcardIdx] == ( FString( TEXT("*") )+ FPackageName::GetAssetPackageExtension() );
TArray<FString> PackageFiles;
if ( !NormalizePackageNames( Unused, PackageFiles, PackageFileNameFilters[PackageFilenameWildcardIdx], PackageFilter) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("No packages found with extension %i: '%s'"), PackageFilenameWildcardIdx, *PackageFileNameFilters[PackageFilenameWildcardIdx]);
continue;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("Found %i packages with extension %i: '%s'"), PackageFiles.Num(), PackageFilenameWildcardIdx, *PackageFileNameFilters[PackageFilenameWildcardIdx]);
}
//Run through all the files found and add any that pass the include, exclude and filter constraints to OrderedPackageFilesToLoad
for (FString& PackageFile : PackageFiles)
{
PackageFile = FPaths::ConvertRelativePathToFull(PackageFile);
bool bExclude = false;
//Ensure it matches the include paths if there are some.
for (FString& IncludePath : IncludePathFilters)
{
bExclude = true;
if( PackageFile.MatchesWildcard(IncludePath) )
{
bExclude = false;
break;
}
}
if ( bExclude )
{
PackageFilesNotInIncludePath.Add(PackageFile);
}
//Ensure it does not match the exclude paths if there are some.
for (const FString& ExcludePath : ExcludePathFilters)
{
if (PackageFile.MatchesWildcard(ExcludePath))
{
bExclude = true;
PackageFilesInExcludePath.Add(PackageFile);
break;
}
}
//Check that this is not on the list of packages that we don't care about e.g. textures.
if ( !bExclude && IsAssetPackage && LongPackageNamesToExclude.Contains( PackageFile ) )
{
bExclude = true;
PackageFilesExcludedByClass.Add(PackageFile);
}
//If we haven't failed one of the above checks, add it to the array of packages to process.
if(!bExclude)
{
PackageFileNamesToProcess.Add(PackageFile);
}
}
}
if ( PackageFileNamesToProcess.Num() == 0 )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No files found or none passed the include/exclude criteria."));
}
bool bSkipGatherCache = FParse::Param(FCommandLine::Get(), TEXT("SkipGatherCache"));
if (!bSkipGatherCache)
{
GetBoolFromConfig(*SectionName, TEXT("SkipGatherCache"), bSkipGatherCache, GatherTextConfigPath);
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("SkipGatherCache: %s"), bSkipGatherCache ? TEXT("true") : TEXT("false"));
TArray< FString > PackageFileNamesToLoad;
for (FString& PackageFile : PackageFileNamesToProcess)
{
TScopedPointer< FArchive > FileReader( IFileManager::Get().CreateFileReader( *PackageFile ) );
if( FileReader )
{
// Read package file summary from the file
FPackageFileSummary PackageFileSummary;
(*FileReader) << PackageFileSummary;
bool MustLoadForGather = false;
// Have we been asked to skip the cache of text that exists in the header of newer packages?
if (bSkipGatherCache && PackageFileSummary.GetFileVersionUE4() >= VER_UE4_SERIALIZE_TEXT_IN_PACKAGES)
{
// Fallback on the old package flag check.
if (PackageFileSummary.PackageFlags & PKG_RequiresLocalizationGather)
{
MustLoadForGather = true;
}
}
const FCustomVersion* const EditorVersion = PackageFileSummary.GetCustomVersionContainer().GetVersion(FEditorObjectVersion::GUID);
// Packages not resaved since localization gathering flagging was added to packages must be loaded.
if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_PACKAGE_REQUIRES_LOCALIZATION_GATHER_FLAGGING)
{
MustLoadForGather = true;
}
// Package not resaved since gatherable text data was added to package headers must be loaded, since their package header won't contain pregathered text data.
else if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_SERIALIZE_TEXT_IN_PACKAGES)
{
// Fallback on the old package flag check.
if (PackageFileSummary.PackageFlags & PKG_RequiresLocalizationGather)
{
MustLoadForGather = true;
}
}
else if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_DIALOGUE_WAVE_NAMESPACE_AND_CONTEXT_CHANGES)
{
IAssetRegistry& AssetRegistry = AssetRegistryModule.Get();
TArray<FAssetData> AssetDataInPackage;
AssetRegistry.GetAssetsByPackageName(*FPackageName::FilenameToLongPackageName(PackageFile), AssetDataInPackage);
for (const FAssetData& AssetData : AssetDataInPackage)
{
if (AssetData.AssetClass == UDialogueWave::StaticClass()->GetFName())
{
MustLoadForGather = true;
}
}
}
// Add package to list of packages to load fully and process.
if (MustLoadForGather)
{
PackageFileNamesToLoad.Add(PackageFile);
}
// Process immediately packages that don't require loading to process.
else if (PackageFileSummary.GatherableTextDataOffset > 0)
{
FileReader->Seek(PackageFileSummary.GatherableTextDataOffset);
TArray<FGatherableTextData> GatherableTextDataArray;
GatherableTextDataArray.SetNum(PackageFileSummary.GatherableTextDataCount);
for (int32 GatherableTextDataIndex = 0; GatherableTextDataIndex < PackageFileSummary.GatherableTextDataCount; ++GatherableTextDataIndex)
{
(*FileReader) << GatherableTextDataArray[GatherableTextDataIndex];
}
ProcessGatherableTextDataArray(PackageFile, GatherableTextDataArray);
}
}
}
CollectGarbage(RF_NoFlags);
//Now go through the remaining packages in the main array and process them in batches.
int32 PackagesPerBatchCount = 100;
TArray< UPackage* > LoadedPackages;
TArray< FString > LoadedPackageFileNames;
TArray< FString > FailedPackageFileNames;
TArray< UPackage* > LoadedPackagesToProcess;
const int32 PackageCount = PackageFileNamesToLoad.Num();
const int32 BatchCount = PackageCount / PackagesPerBatchCount + (PackageCount % PackagesPerBatchCount > 0 ? 1 : 0); // Add an extra batch for any remainder if necessary
if(PackageCount > 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loading %i packages in %i batches of %i."), PackageCount, BatchCount, PackagesPerBatchCount);
}
FLoadPackageLogOutputRedirector LogOutputRedirector;
//Load the packages in batches
int32 PackageIndex = 0;
for( int32 BatchIndex = 0; BatchIndex < BatchCount; ++BatchIndex )
{
int32 PackagesInThisBatch = 0;
int32 FailuresInThisBatch = 0;
for( ; PackageIndex < PackageCount && PackagesInThisBatch < PackagesPerBatchCount; ++PackageIndex )
{
FString PackageFileName = PackageFileNamesToLoad[PackageIndex];
UE_LOG(LogGatherTextFromAssetsCommandlet, Verbose, TEXT("Loading package: '%s'."), *PackageFileName);
UPackage *Package = nullptr;
{
FString LongPackageName;
if (!FPackageName::TryConvertFilenameToLongPackageName(PackageFileName, LongPackageName))
{
LongPackageName = FPaths::GetCleanFilename(PackageFileName);
}
FLoadPackageLogOutputRedirector::FScopedCapture ScopedCapture(&LogOutputRedirector, LongPackageName);
Package = LoadPackage( NULL, *PackageFileName, LOAD_NoWarn | LOAD_Quiet );
}
if( Package )
{
LoadedPackages.Add(Package);
LoadedPackageFileNames.Add(PackageFileName);
// Because packages may not have been resaved after this flagging was implemented, we may have added packages to load that weren't flagged - potential false positives.
// The loading process should have reflagged said packages so that only true positives will have this flag.
if( Package->RequiresLocalizationGather() )
{
LoadedPackagesToProcess.Add( Package );
}
}
else
{
FailedPackageFileNames.Add( PackageFileName );
++FailuresInThisBatch;
continue;
}
++PackagesInThisBatch;
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loaded %i packages in batch %i of %i. %i failed."), PackagesInThisBatch, BatchIndex + 1, BatchCount, FailuresInThisBatch);
ProcessPackages(LoadedPackagesToProcess);
LoadedPackagesToProcess.Empty(PackagesPerBatchCount);
if( bFixBroken )
{
for( int32 LoadedPackageIndex=0; LoadedPackageIndex < LoadedPackages.Num() ; ++LoadedPackageIndex )
{
UPackage *Package = LoadedPackages[LoadedPackageIndex];
const FString PackageName = LoadedPackageFileNames[LoadedPackageIndex];
//Todo - link with source control.
if( Package )
{
if( Package->IsDirty() )
{
if( SavePackageHelper( Package, *PackageName ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Saved Package %s."),*PackageName);
}
else
{
//TODO - Work out how to integrate with source control. The code from the source gatherer doesn't work.
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Could not save package %s. Probably due to source control. "),*PackageName);
}
}
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Failed to find one of the loaded packages."));
}
}
}
CollectGarbage(RF_NoFlags);
LoadedPackages.Empty(PackagesPerBatchCount);
LoadedPackageFileNames.Empty(PackagesPerBatchCount);
}
return 0;
}
void UUnrealEdEngine::OnPackageDirtyStateUpdated( UPackage* Pkg)
{
// The passed in object should never be NULL
check(Pkg);
UPackage* Package = Pkg->GetOutermost();
const FString PackageName = Package->GetName();
// Alert the user if they have modified a package that won't be able to be saved because
// it's already been saved with an engine version that is newer than the current one.
if (!FUObjectThreadContext::Get().IsRoutingPostLoad && Package->IsDirty() && !PackagesCheckedForEngineVersion.Contains(PackageName))
{
EWriteDisallowedWarningState WarningStateToSet = WDWS_WarningUnnecessary;
FString PackageFileName;
if ( FPackageName::DoesPackageExist( Package->GetName(), NULL, &PackageFileName ) )
{
// If a package has never been loaded, a file reader is necessary to find the package file summary for its saved engine version.
FArchive* PackageReader = IFileManager::Get().CreateFileReader( *PackageFileName );
if ( PackageReader )
{
FPackageFileSummary Summary;
*PackageReader << Summary;
if ( Summary.GetFileVersionUE4() > GPackageFileUE4Version || !GEngineVersion.IsCompatibleWith(Summary.CompatibleWithEngineVersion) )
{
WarningStateToSet = WDWS_PendingWarn;
bNeedWarningForPkgEngineVer = true;
}
}
delete PackageReader;
}
PackagesCheckedForEngineVersion.Add( PackageName, WarningStateToSet );
}
// Alert the user if they have modified a package that they do not have sufficient permission to write to disk.
// This can be due to the content being in the "Program Files" folder and the user does not have admin privileges.
if (!FUObjectThreadContext::Get().IsRoutingPostLoad && Package->IsDirty() && !PackagesCheckedForWritePermission.Contains(PackageName))
{
EWriteDisallowedWarningState WarningStateToSet = GetWarningStateForWritePermission(PackageName);
if ( WarningStateToSet == WDWS_PendingWarn )
{
bNeedWarningForWritePermission = true;
}
PackagesCheckedForWritePermission.Add( PackageName, WarningStateToSet );
}
if( Package->IsDirty() )
{
// Find out if we have already asked the user to modify this package
const uint8* PromptState = PackageToNotifyState.Find( Package );
const bool bAlreadyAsked = PromptState != NULL;
// During an autosave, packages are saved in the autosave directory which switches off their dirty flags.
// To preserve the pre-autosave state, any saved package is then remarked as dirty because it wasn't saved in the normal location where it would be picked up by source control.
// Any callback that happens during an autosave is bogus since a package wasn't marked dirty due to a user modification.
const bool bIsAutoSaving = PackageAutoSaver.Get() && PackageAutoSaver->IsAutoSaving();
const UEditorLoadingSavingSettings* Settings = GetDefault<UEditorLoadingSavingSettings>();
if( !bIsAutoSaving &&
!GIsEditorLoadingPackage && // Don't ask if the package was modified as a result of a load
!bAlreadyAsked && // Don't ask if we already asked once!
(Settings->bPromptForCheckoutOnAssetModification || Settings->bAutomaticallyCheckoutOnAssetModification) )
{
// Force source control state to be updated
ISourceControlProvider& SourceControlProvider = ISourceControlModule::Get().GetProvider();
TArray<FString> Files;
Files.Add(SourceControlHelpers::PackageFilename(Package));
SourceControlProvider.Execute(ISourceControlOperation::Create<FUpdateStatus>(), SourceControlHelpers::AbsoluteFilenames(Files), EConcurrency::Asynchronous, FSourceControlOperationComplete::CreateUObject(this, &UUnrealEdEngine::OnSourceControlStateUpdated, TWeakObjectPtr<UPackage>(Package)));
}
}
else
{
// This package was saved, the user should be prompted again if they checked in the package
PackageToNotifyState.Remove( Package );
}
}
int32 UGatherTextFromAssetsCommandlet::Main(const FString& Params)
{
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Include paths
TArray<FString> IncludePaths;
GetConfigArray(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No include paths in section %s"), *SectionName);
return -1;
}
//Exclude paths
TArray<FString> ExcludePaths;
GetConfigArray(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
//package extensions
TArray<FString> PackageExts;
GetConfigArray(*SectionName, TEXT("PackageExtensions"), PackageExts, GatherTextConfigPath);
if (PackageExts.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No package extensions specified in section %s, using defaults"), *SectionName);
PackageExts.Add(FString("*") + FPackageName::GetAssetPackageExtension());
PackageExts.Add(FString("*") + FPackageName::GetMapPackageExtension());
}
//asset class exclude
TArray<FString> ExcludeClasses;
GetConfigArray(*SectionName, TEXT("ExcludeClasses"), ExcludeClasses, GatherTextConfigPath);
FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>(TEXT("AssetRegistry"));
AssetRegistryModule.Get().SearchAllAssets( true );
FARFilter Filter;
for(int32 i = 0; i < ExcludeClasses.Num(); i++)
{
UClass* FilterClass = FindObject<UClass>(ANY_PACKAGE, *ExcludeClasses[i]);
if(FilterClass)
{
Filter.ClassNames.Add( FilterClass->GetFName() );
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Invalid exclude class %s"), *ExcludeClasses[i]);
}
}
TArray<FAssetData> AssetData;
AssetRegistryModule.Get().GetAssets(Filter, AssetData);
FString UAssetPackageExtension = FPackageName::GetAssetPackageExtension();
TSet< FString > LongPackageNamesToExclude;
for (int Index = 0; Index < AssetData.Num(); Index++)
{
LongPackageNamesToExclude.Add( FPackageName::LongPackageNameToFilename( AssetData[Index].PackageName.ToString(), UAssetPackageExtension ) );
}
//Get whether we should fix broken properties that we find.
GetConfigBool(*SectionName, TEXT("bFixBroken"), bFixBroken, GatherTextConfigPath);
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetConfigArray(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("The GatherTextFromAssets commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
//The main array of files to work from.
TArray< FString > PackageFileNamesToLoad;
TSet< FString > LongPackageNamesToProcess;
TArray<FString> PackageFilesNotInIncludePath;
TArray<FString> PackageFilesInExcludePath;
TArray<FString> PackageFilesExcludedByClass;
//Fill the list of packages to work from.
uint8 PackageFilter = NORMALIZE_DefaultFlags;
TArray<FString> Unused;
for ( int32 PackageFilenameWildcardIdx = 0; PackageFilenameWildcardIdx < PackageExts.Num(); PackageFilenameWildcardIdx++ )
{
const bool IsAssetPackage = PackageExts[PackageFilenameWildcardIdx] == ( FString( TEXT("*") )+ FPackageName::GetAssetPackageExtension() );
TArray<FString> PackageFiles;
if ( !NormalizePackageNames( Unused, PackageFiles, PackageExts[PackageFilenameWildcardIdx], PackageFilter) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("No packages found with extension %i: '%s'"), PackageFilenameWildcardIdx, *PackageExts[PackageFilenameWildcardIdx]);
continue;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("Found %i packages with extension %i: '%s'"), PackageFiles.Num(), PackageFilenameWildcardIdx, *PackageExts[PackageFilenameWildcardIdx]);
}
//Run through all the files found and add any that pass the include, exclude and filter constraints to OrderedPackageFilesToLoad
for( int32 PackageFileIdx=0; PackageFileIdx<PackageFiles.Num(); ++PackageFileIdx )
{
bool bExclude = false;
//Ensure it matches the include paths if there are some.
for( int32 IncludePathIdx=0; IncludePathIdx<IncludePaths.Num() ; ++IncludePathIdx )
{
bExclude = true;
if( PackageFiles[PackageFileIdx].MatchesWildcard(IncludePaths[IncludePathIdx]) )
{
bExclude = false;
break;
}
}
if ( bExclude )
{
PackageFilesNotInIncludePath.Add(PackageFiles[PackageFileIdx]);
}
//Ensure it does not match the exclude paths if there are some.
for( int32 ExcludePathIdx=0; !bExclude && ExcludePathIdx<ExcludePaths.Num() ; ++ExcludePathIdx )
{
if( PackageFiles[PackageFileIdx].MatchesWildcard(ExcludePaths[ExcludePathIdx]) )
{
bExclude = true;
PackageFilesInExcludePath.Add(PackageFiles[PackageFileIdx]);
break;
}
}
//Check that this is not on the list of packages that we don't care about e.g. textures.
if ( !bExclude && IsAssetPackage && LongPackageNamesToExclude.Contains( PackageFiles[PackageFileIdx] ) )
{
bExclude = true;
PackageFilesExcludedByClass.Add(PackageFiles[PackageFileIdx]);
}
//If we haven't failed one of the above checks, add it to the array of packages to process.
if(!bExclude)
{
TScopedPointer< FArchive > FileReader( IFileManager::Get().CreateFileReader( *PackageFiles[PackageFileIdx] ) );
if( FileReader )
{
// Read package file summary from the file
FPackageFileSummary PackageSummary;
(*FileReader) << PackageSummary;
// Early out check if the package has been flagged as needing localization gathering
if( PackageSummary.PackageFlags & PKG_RequiresLocalizationGather || PackageSummary.GetFileVersionUE4() < VER_UE4_PACKAGE_REQUIRES_LOCALIZATION_GATHER_FLAGGING )
{
PackageFileNamesToLoad.Add( PackageFiles[PackageFileIdx] );
}
}
}
}
}
if ( PackageFileNamesToLoad.Num() == 0 )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No files found. Or none passed the include/exclude criteria."));
}
CollectGarbage( RF_Native );
//Now go through the remaining packages in the main array and process them in batches.
int32 PackagesPerBatchCount = 100;
TArray< UPackage* > LoadedPackages;
TArray< FString > LoadedPackageFileNames;
TArray< FString > FailedPackageFileNames;
TArray< UPackage* > PackagesToProcess;
const int32 PackageCount = PackageFileNamesToLoad.Num();
const int32 BatchCount = PackageCount / PackagesPerBatchCount + (PackageCount % PackagesPerBatchCount > 0 ? 1 : 0); // Add an extra batch for any remainder if necessary
if(PackageCount > 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loading %i packages in %i batches of %i."), PackageCount, BatchCount, PackagesPerBatchCount);
}
//Load the packages in batches
int32 PackageIndex = 0;
for( int32 BatchIndex = 0; BatchIndex < BatchCount; ++BatchIndex )
{
int32 PackagesInThisBatch = 0;
for( PackageIndex; PackageIndex < PackageCount && PackagesInThisBatch < PackagesPerBatchCount; ++PackageIndex )
{
FString PackageFileName = PackageFileNamesToLoad[PackageIndex];
UPackage *Package = LoadPackage( NULL, *PackageFileName, LOAD_None );
if( Package )
{
LoadedPackages.Add(Package);
LoadedPackageFileNames.Add(PackageFileName);
// Because packages may not have been resaved after this flagging was implemented, we may have added packages to load that weren't flagged - potential false positives.
// The loading process should have reflagged said packages so that only true positives will have this flag.
if( Package->RequiresLocalizationGather() )
{
PackagesToProcess.Add( Package );
}
}
else
{
FailedPackageFileNames.Add( PackageFileName );
continue;
}
++PackagesInThisBatch;
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loaded %i packages in batch %i of %i."), PackagesInThisBatch, BatchIndex + 1, BatchCount);
ProcessPackages(PackagesToProcess);
PackagesToProcess.Empty(PackagesPerBatchCount);
if( bFixBroken )
{
for( int32 LoadedPackageIndex=0; LoadedPackageIndex < LoadedPackages.Num() ; ++LoadedPackageIndex )
{
UPackage *Package = LoadedPackages[LoadedPackageIndex];
const FString PackageName = LoadedPackageFileNames[LoadedPackageIndex];
//Todo - link with source control.
if( Package )
{
if( Package->IsDirty() )
{
if( SavePackageHelper( Package, *PackageName ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Saved Package %s."),*PackageName);
}
else
{
//TODO - Work out how to integrate with source control. The code from the source gatherer doesn't work.
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Could not save package %s. Probably due to source control. "),*PackageName);
}
}
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Failed to find one of the loaded packages."));
}
}
}
CollectGarbage( RF_Native );
LoadedPackages.Empty(PackagesPerBatchCount);
LoadedPackageFileNames.Empty(PackagesPerBatchCount);
}
for(auto i = ConflictTracker.Namespaces.CreateConstIterator(); i; ++i)
{
const FString& NamespaceName = i.Key();
const FConflictTracker::FKeyTable& KeyTable = i.Value();
for(auto j = KeyTable.CreateConstIterator(); j; ++j)
{
const FString& KeyName = j.Key();
const FConflictTracker::FEntryArray& EntryArray = j.Value();
for(int k = 0; k < EntryArray.Num(); ++k)
{
const FConflictTracker::FEntry& Entry = EntryArray[k];
switch(Entry.Status)
{
case EAssetTextGatherStatus::MissingKey:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Detected missing key on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::MissingKey_Resolved:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Fixed missing key on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::IdentityConflict:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Detected duplicate identity with differing source on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::IdentityConflict_Resolved:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Fixed duplicate identity with differing source on asset \"%s\"."), *Entry.ObjectPath);
}
break;
}
}
}
}
return 0;
}