int32 UGatherTextCommandletBase::GetPathArrayFromConfig( const TCHAR* Section, const TCHAR* Key, TArray<FString>& OutArr, const FString& Filename )
{
int32 count = GetStringArrayFromConfig( Section, Key, OutArr, Filename );
for (int32 i = 0; i < count; ++i)
{
if (FPaths::IsRelative(OutArr[i]))
{
const FString ProjectBasePath = FPaths::GameDir().IsEmpty() ? FPaths::EngineDir() : FPaths::GameDir();
OutArr[i] = FPaths::Combine( *ProjectBasePath, *OutArr[i] );
OutArr[i] = FPaths::ConvertRelativePathToFull(OutArr[i]);
}
FPaths::CollapseRelativeDirectories(OutArr[i]);
}
return count;
}
int32 UGatherTextFromAssetsCommandlet::Main(const FString& Params)
{
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Modules to Preload
TArray<FString> ModulesToPreload;
GetStringArrayFromConfig(*SectionName, TEXT("ModulesToPreload"), ModulesToPreload, GatherTextConfigPath);
for (const FString& ModuleName : ModulesToPreload)
{
FModuleManager::Get().LoadModule(*ModuleName);
}
// IncludePathFilters
TArray<FString> IncludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePathFilters"), IncludePathFilters, GatherTextConfigPath);
// IncludePaths (DEPRECATED)
{
TArray<FString> IncludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num())
{
IncludePathFilters.Append(IncludePaths);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("IncludePaths detected in section %s. IncludePaths is deprecated, please use IncludePathFilters."), *SectionName);
}
}
if (IncludePathFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No include path filters in section %s."), *SectionName);
return -1;
}
// ExcludePathFilters
TArray<FString> ExcludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePathFilters"), ExcludePathFilters, GatherTextConfigPath);
// ExcludePaths (DEPRECATED)
{
TArray<FString> ExcludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
if (ExcludePaths.Num())
{
ExcludePathFilters.Append(ExcludePaths);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("ExcludePaths detected in section %s. ExcludePaths is deprecated, please use ExcludePathFilters."), *SectionName);
}
}
// PackageNameFilters
TArray<FString> PackageFileNameFilters;
GetStringArrayFromConfig(*SectionName, TEXT("PackageFileNameFilters"), PackageFileNameFilters, GatherTextConfigPath);
// PackageExtensions (DEPRECATED)
{
TArray<FString> PackageExtensions;
GetStringArrayFromConfig(*SectionName, TEXT("PackageExtensions"), PackageExtensions, GatherTextConfigPath);
if (PackageExtensions.Num())
{
PackageFileNameFilters.Append(PackageExtensions);
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("PackageExtensions detected in section %s. PackageExtensions is deprecated, please use PackageFileNameFilters."), *SectionName);
}
}
if (PackageFileNameFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No package file name filters in section %s."), *SectionName);
return -1;
}
//asset class exclude
TArray<FString> ExcludeClasses;
GetStringArrayFromConfig(*SectionName, TEXT("ExcludeClasses"), ExcludeClasses, GatherTextConfigPath);
FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>(TEXT("AssetRegistry"));
AssetRegistryModule.Get().SearchAllAssets( true );
FARFilter Filter;
for(const auto& ExcludeClass : ExcludeClasses)
{
UClass* FilterClass = FindObject<UClass>(ANY_PACKAGE, *ExcludeClass);
if(FilterClass)
{
Filter.ClassNames.Add( FilterClass->GetFName() );
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Invalid exclude class %s"), *ExcludeClass);
}
}
TArray<FAssetData> AssetDataArray;
AssetRegistryModule.Get().GetAssets(Filter, AssetDataArray);
FString UAssetPackageExtension = FPackageName::GetAssetPackageExtension();
TSet< FString > LongPackageNamesToExclude;
for (int Index = 0; Index < AssetDataArray.Num(); Index++)
{
LongPackageNamesToExclude.Add( FPackageName::LongPackageNameToFilename( AssetDataArray[Index].PackageName.ToString(), UAssetPackageExtension ) );
}
//Get whether we should fix broken properties that we find.
if (!GetBoolFromConfig(*SectionName, TEXT("bFixBroken"), bFixBroken, GatherTextConfigPath))
{
bFixBroken = false;
}
// Get whether we should gather editor-only data. Typically only useful for the localization of UE4 itself.
if (!GetBoolFromConfig(*SectionName, TEXT("ShouldGatherFromEditorOnlyData"), ShouldGatherFromEditorOnlyData, GatherTextConfigPath))
{
ShouldGatherFromEditorOnlyData = false;
}
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetPathArrayFromConfig(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("The GatherTextFromAssets commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
//The main array of files to work from.
TArray< FString > PackageFileNamesToProcess;
TArray<FString> PackageFilesNotInIncludePath;
TArray<FString> PackageFilesInExcludePath;
TArray<FString> PackageFilesExcludedByClass;
//Fill the list of packages to work from.
uint8 PackageFilter = NORMALIZE_DefaultFlags;
TArray<FString> Unused;
for ( int32 PackageFilenameWildcardIdx = 0; PackageFilenameWildcardIdx < PackageFileNameFilters.Num(); PackageFilenameWildcardIdx++ )
{
const bool IsAssetPackage = PackageFileNameFilters[PackageFilenameWildcardIdx] == ( FString( TEXT("*") )+ FPackageName::GetAssetPackageExtension() );
TArray<FString> PackageFiles;
if ( !NormalizePackageNames( Unused, PackageFiles, PackageFileNameFilters[PackageFilenameWildcardIdx], PackageFilter) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("No packages found with extension %i: '%s'"), PackageFilenameWildcardIdx, *PackageFileNameFilters[PackageFilenameWildcardIdx]);
continue;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("Found %i packages with extension %i: '%s'"), PackageFiles.Num(), PackageFilenameWildcardIdx, *PackageFileNameFilters[PackageFilenameWildcardIdx]);
}
//Run through all the files found and add any that pass the include, exclude and filter constraints to OrderedPackageFilesToLoad
for (FString& PackageFile : PackageFiles)
{
PackageFile = FPaths::ConvertRelativePathToFull(PackageFile);
bool bExclude = false;
//Ensure it matches the include paths if there are some.
for (FString& IncludePath : IncludePathFilters)
{
bExclude = true;
if( PackageFile.MatchesWildcard(IncludePath) )
{
bExclude = false;
break;
}
}
if ( bExclude )
{
PackageFilesNotInIncludePath.Add(PackageFile);
}
//Ensure it does not match the exclude paths if there are some.
for (const FString& ExcludePath : ExcludePathFilters)
{
if (PackageFile.MatchesWildcard(ExcludePath))
{
bExclude = true;
PackageFilesInExcludePath.Add(PackageFile);
break;
}
}
//Check that this is not on the list of packages that we don't care about e.g. textures.
if ( !bExclude && IsAssetPackage && LongPackageNamesToExclude.Contains( PackageFile ) )
{
bExclude = true;
PackageFilesExcludedByClass.Add(PackageFile);
}
//If we haven't failed one of the above checks, add it to the array of packages to process.
if(!bExclude)
{
PackageFileNamesToProcess.Add(PackageFile);
}
}
}
if ( PackageFileNamesToProcess.Num() == 0 )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No files found or none passed the include/exclude criteria."));
}
bool bSkipGatherCache = FParse::Param(FCommandLine::Get(), TEXT("SkipGatherCache"));
if (!bSkipGatherCache)
{
GetBoolFromConfig(*SectionName, TEXT("SkipGatherCache"), bSkipGatherCache, GatherTextConfigPath);
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("SkipGatherCache: %s"), bSkipGatherCache ? TEXT("true") : TEXT("false"));
TArray< FString > PackageFileNamesToLoad;
for (FString& PackageFile : PackageFileNamesToProcess)
{
TScopedPointer< FArchive > FileReader( IFileManager::Get().CreateFileReader( *PackageFile ) );
if( FileReader )
{
// Read package file summary from the file
FPackageFileSummary PackageFileSummary;
(*FileReader) << PackageFileSummary;
bool MustLoadForGather = false;
// Have we been asked to skip the cache of text that exists in the header of newer packages?
if (bSkipGatherCache && PackageFileSummary.GetFileVersionUE4() >= VER_UE4_SERIALIZE_TEXT_IN_PACKAGES)
{
// Fallback on the old package flag check.
if (PackageFileSummary.PackageFlags & PKG_RequiresLocalizationGather)
{
MustLoadForGather = true;
}
}
const FCustomVersion* const EditorVersion = PackageFileSummary.GetCustomVersionContainer().GetVersion(FEditorObjectVersion::GUID);
// Packages not resaved since localization gathering flagging was added to packages must be loaded.
if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_PACKAGE_REQUIRES_LOCALIZATION_GATHER_FLAGGING)
{
MustLoadForGather = true;
}
// Package not resaved since gatherable text data was added to package headers must be loaded, since their package header won't contain pregathered text data.
else if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_SERIALIZE_TEXT_IN_PACKAGES)
{
// Fallback on the old package flag check.
if (PackageFileSummary.PackageFlags & PKG_RequiresLocalizationGather)
{
MustLoadForGather = true;
}
}
else if (PackageFileSummary.GetFileVersionUE4() < VER_UE4_DIALOGUE_WAVE_NAMESPACE_AND_CONTEXT_CHANGES)
{
IAssetRegistry& AssetRegistry = AssetRegistryModule.Get();
TArray<FAssetData> AssetDataInPackage;
AssetRegistry.GetAssetsByPackageName(*FPackageName::FilenameToLongPackageName(PackageFile), AssetDataInPackage);
for (const FAssetData& AssetData : AssetDataInPackage)
{
if (AssetData.AssetClass == UDialogueWave::StaticClass()->GetFName())
{
MustLoadForGather = true;
}
}
}
// Add package to list of packages to load fully and process.
if (MustLoadForGather)
{
PackageFileNamesToLoad.Add(PackageFile);
}
// Process immediately packages that don't require loading to process.
else if (PackageFileSummary.GatherableTextDataOffset > 0)
{
FileReader->Seek(PackageFileSummary.GatherableTextDataOffset);
TArray<FGatherableTextData> GatherableTextDataArray;
GatherableTextDataArray.SetNum(PackageFileSummary.GatherableTextDataCount);
for (int32 GatherableTextDataIndex = 0; GatherableTextDataIndex < PackageFileSummary.GatherableTextDataCount; ++GatherableTextDataIndex)
{
(*FileReader) << GatherableTextDataArray[GatherableTextDataIndex];
}
ProcessGatherableTextDataArray(PackageFile, GatherableTextDataArray);
}
}
}
CollectGarbage(RF_NoFlags);
//Now go through the remaining packages in the main array and process them in batches.
int32 PackagesPerBatchCount = 100;
TArray< UPackage* > LoadedPackages;
TArray< FString > LoadedPackageFileNames;
TArray< FString > FailedPackageFileNames;
TArray< UPackage* > LoadedPackagesToProcess;
const int32 PackageCount = PackageFileNamesToLoad.Num();
const int32 BatchCount = PackageCount / PackagesPerBatchCount + (PackageCount % PackagesPerBatchCount > 0 ? 1 : 0); // Add an extra batch for any remainder if necessary
if(PackageCount > 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loading %i packages in %i batches of %i."), PackageCount, BatchCount, PackagesPerBatchCount);
}
FLoadPackageLogOutputRedirector LogOutputRedirector;
//Load the packages in batches
int32 PackageIndex = 0;
for( int32 BatchIndex = 0; BatchIndex < BatchCount; ++BatchIndex )
{
int32 PackagesInThisBatch = 0;
int32 FailuresInThisBatch = 0;
for( ; PackageIndex < PackageCount && PackagesInThisBatch < PackagesPerBatchCount; ++PackageIndex )
{
FString PackageFileName = PackageFileNamesToLoad[PackageIndex];
UE_LOG(LogGatherTextFromAssetsCommandlet, Verbose, TEXT("Loading package: '%s'."), *PackageFileName);
UPackage *Package = nullptr;
{
FString LongPackageName;
if (!FPackageName::TryConvertFilenameToLongPackageName(PackageFileName, LongPackageName))
{
LongPackageName = FPaths::GetCleanFilename(PackageFileName);
}
FLoadPackageLogOutputRedirector::FScopedCapture ScopedCapture(&LogOutputRedirector, LongPackageName);
Package = LoadPackage( NULL, *PackageFileName, LOAD_NoWarn | LOAD_Quiet );
}
if( Package )
{
LoadedPackages.Add(Package);
LoadedPackageFileNames.Add(PackageFileName);
// Because packages may not have been resaved after this flagging was implemented, we may have added packages to load that weren't flagged - potential false positives.
// The loading process should have reflagged said packages so that only true positives will have this flag.
if( Package->RequiresLocalizationGather() )
{
LoadedPackagesToProcess.Add( Package );
}
}
else
{
FailedPackageFileNames.Add( PackageFileName );
++FailuresInThisBatch;
continue;
}
++PackagesInThisBatch;
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loaded %i packages in batch %i of %i. %i failed."), PackagesInThisBatch, BatchIndex + 1, BatchCount, FailuresInThisBatch);
ProcessPackages(LoadedPackagesToProcess);
LoadedPackagesToProcess.Empty(PackagesPerBatchCount);
if( bFixBroken )
{
for( int32 LoadedPackageIndex=0; LoadedPackageIndex < LoadedPackages.Num() ; ++LoadedPackageIndex )
{
UPackage *Package = LoadedPackages[LoadedPackageIndex];
const FString PackageName = LoadedPackageFileNames[LoadedPackageIndex];
//Todo - link with source control.
if( Package )
{
if( Package->IsDirty() )
{
if( SavePackageHelper( Package, *PackageName ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Saved Package %s."),*PackageName);
}
else
{
//TODO - Work out how to integrate with source control. The code from the source gatherer doesn't work.
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Could not save package %s. Probably due to source control. "),*PackageName);
}
}
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Failed to find one of the loaded packages."));
}
}
}
CollectGarbage(RF_NoFlags);
LoadedPackages.Empty(PackagesPerBatchCount);
LoadedPackageFileNames.Empty(PackagesPerBatchCount);
}
return 0;
}
int32 UGatherTextFromSourceCommandlet::Main( const FString& Params )
{
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// SearchDirectoryPaths
TArray<FString> SearchDirectoryPaths;
GetPathArrayFromConfig(*SectionName, TEXT("SearchDirectoryPaths"), SearchDirectoryPaths, GatherTextConfigPath);
// IncludePaths (DEPRECATED)
{
TArray<FString> IncludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num())
{
SearchDirectoryPaths.Append(IncludePaths);
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("IncludePaths detected in section %s. IncludePaths is deprecated, please use SearchDirectoryPaths."), *SectionName);
}
}
if (SearchDirectoryPaths.Num() == 0)
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No search directory paths in section %s."), *SectionName);
return -1;
}
// ExcludePathFilters
TArray<FString> ExcludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePathFilters"), ExcludePathFilters, GatherTextConfigPath);
// ExcludePaths (DEPRECATED)
{
TArray<FString> ExcludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
if (ExcludePaths.Num())
{
ExcludePathFilters.Append(ExcludePaths);
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("ExcludePaths detected in section %s. ExcludePaths is deprecated, please use ExcludePathFilters."), *SectionName);
}
}
// FileNameFilters
TArray<FString> FileNameFilters;
GetStringArrayFromConfig(*SectionName, TEXT("FileNameFilters"), FileNameFilters, GatherTextConfigPath);
// SourceFileSearchFilters (DEPRECATED)
{
TArray<FString> SourceFileSearchFilters;
GetStringArrayFromConfig(*SectionName, TEXT("SourceFileSearchFilters"), SourceFileSearchFilters, GatherTextConfigPath);
if (SourceFileSearchFilters.Num())
{
FileNameFilters.Append(SourceFileSearchFilters);
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("SourceFileSearchFilters detected in section %s. SourceFileSearchFilters is deprecated, please use FileNameFilters."), *SectionName);
}
}
if (FileNameFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No source filters in section %s"), *SectionName);
return -1;
}
//Ensure all filters are unique.
TArray<FString> UniqueSourceFileSearchFilters;
for (const FString& SourceFileSearchFilter : FileNameFilters)
{
UniqueSourceFileSearchFilters.AddUnique(SourceFileSearchFilter);
}
// Search in the root folder for each of the wildcard filters specified and build a list of files
TArray<FString> AllFoundFiles;
for (FString& SearchDirectoryPath : SearchDirectoryPaths)
{
for (const FString& UniqueSourceFileSearchFilter : UniqueSourceFileSearchFilters)
{
TArray<FString> RootSourceFiles;
IFileManager::Get().FindFilesRecursive(RootSourceFiles, *SearchDirectoryPath, *UniqueSourceFileSearchFilter, true, false,false);
for (FString& RootSourceFile : RootSourceFiles)
{
if (FPaths::IsRelative(RootSourceFile))
{
RootSourceFile = FPaths::ConvertRelativePathToFull(RootSourceFile);
}
}
AllFoundFiles.Append(RootSourceFiles);
}
}
TArray<FString> FilesToProcess;
TArray<FString> RemovedList;
//Run through all the files found and add any that pass the exclude and filter constraints to PackageFilesToProcess
for (const FString& FoundFile : AllFoundFiles)
{
bool bExclude = false;
//Ensure it does not match the exclude paths if there are some.
for (FString& ExcludePath : ExcludePathFilters)
{
if (FoundFile.MatchesWildcard(ExcludePath) )
{
bExclude = true;
RemovedList.Add(FoundFile);
break;
}
}
//If we haven't failed any checks, add it to the array of files to process.
if( !bExclude )
{
FilesToProcess.Add(FoundFile);
}
}
// Return if no source files were found
if( FilesToProcess.Num() == 0 )
{
FString SpecifiedDirectoriesString;
for (FString& SearchDirectoryPath : SearchDirectoryPaths)
{
SpecifiedDirectoriesString.Append(FString(SpecifiedDirectoriesString.IsEmpty() ? TEXT("") : TEXT("\n")) + FString::Printf(TEXT("+ %s"), *SearchDirectoryPath));
}
for (FString& ExcludePath : ExcludePathFilters)
{
SpecifiedDirectoriesString.Append(FString(SpecifiedDirectoriesString.IsEmpty() ? TEXT("") : TEXT("\n")) + FString::Printf(TEXT("- %s"), *ExcludePath));
}
FString SourceFileSearchFiltersString;
for (const auto& Filter : UniqueSourceFileSearchFilters)
{
SourceFileSearchFiltersString += FString(SourceFileSearchFiltersString.IsEmpty() ? TEXT("") : TEXT(", ")) + Filter;
}
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("The GatherTextFromSource commandlet couldn't find any source files matching (%s) in the specified directories:\n%s"), *SourceFileSearchFiltersString, *SpecifiedDirectoriesString);
return -1;
}
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetPathArrayFromConfig(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("The GatherTextFromSource commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
// Get the loc macros and their syntax
TArray<FParsableDescriptor*> Parsables;
Parsables.Add(new FDefineDescriptor());
Parsables.Add(new FUndefDescriptor());
Parsables.Add(new FCommandMacroDescriptor());
// New Localization System with Namespace as literal argument.
Parsables.Add(new FStringMacroDescriptor( FString(TEXT("NSLOCTEXT")),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Namespace, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Identifier, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_SourceText, true)));
// New Localization System with Namespace as preprocessor define.
Parsables.Add(new FStringMacroDescriptor( FString(TEXT("LOCTEXT")),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Identifier, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_SourceText, true)));
Parsables.Add(new FIniNamespaceDescriptor());
// Init a parse context to track the state of the file parsing
FSourceFileParseContext ParseCtxt;
ParseCtxt.ManifestInfo = ManifestInfo;
// Parse all source files for macros and add entries to SourceParsedEntries
for ( FString& SourceFile : FilesToProcess)
{
FString ProjectBasePath;
if (!FPaths::GameDir().IsEmpty())
{
ProjectBasePath = FPaths::GameDir();
}
else
{
ProjectBasePath = FPaths::EngineDir();
}
ParseCtxt.Filename = SourceFile;
FPaths::MakePathRelativeTo(ParseCtxt.Filename, *ProjectBasePath);
ParseCtxt.LineNumber = 0;
ParseCtxt.LineText.Empty();
ParseCtxt.Namespace.Empty();
ParseCtxt.ExcludedRegion = false;
ParseCtxt.WithinBlockComment = false;
ParseCtxt.WithinLineComment = false;
ParseCtxt.WithinStringLiteral = false;
ParseCtxt.WithinNamespaceDefine = false;
ParseCtxt.WithinStartingLine.Empty();
FString SourceFileText;
if (!FFileHelper::LoadFileToString(SourceFileText, *SourceFile))
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("GatherTextSource failed to open file %s"), *ParseCtxt.Filename);
}
else
{
if (!ParseSourceText(SourceFileText, Parsables, ParseCtxt))
{
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("GatherTextSource error(s) parsing source file %s"), *ParseCtxt.Filename);
}
else
{
if (ParseCtxt.WithinNamespaceDefine == true)
{
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Non-matching LOCTEXT_NAMESPACE defines found in %s"), *ParseCtxt.Filename);
}
}
}
}
// Clear parsables list safely
for (int32 i=0; i<Parsables.Num(); i++)
{
delete Parsables[i];
}
return 0;
}
int32 UInternationalizationExportCommandlet::Main( const FString& Params )
{
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
if ( ParamVal )
{
ConfigPath = *ParamVal;
}
else
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No config section specified."));
return -1;
}
FString SourcePath; // Source path to the root folder that manifest/archive files live in.
if( !GetPathFromConfig( *SectionName, TEXT("SourcePath"), SourcePath, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No source path specified.") );
return -1;
}
FString DestinationPath; // Destination path that we will write files to.
if( !GetPathFromConfig( *SectionName, TEXT("DestinationPath"), DestinationPath, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
FString Filename; // Name of the file to read or write from
if (!GetStringFromConfig(*SectionName, TEXT("PortableObjectName"), Filename, ConfigPath))
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No portable object name specified."));
return -1;
}
// Get cultures to generate.
if( GetStringArrayFromConfig(*SectionName, TEXT("CulturesToGenerate"), CulturesToGenerate, ConfigPath) == 0 )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No cultures specified for generation."));
return -1;
}
bool bDoExport = false;
bool bDoImport = false;
ShouldPersistComments = false;
GetBoolFromConfig( *SectionName, TEXT("bImportLoc"), bDoImport, ConfigPath );
GetBoolFromConfig( *SectionName, TEXT("bExportLoc"), bDoExport, ConfigPath );
GetBoolFromConfig(*SectionName, TEXT("ShouldPersistComments"), ShouldPersistComments, ConfigPath);
// Reject the ShouldPersistComments flag and warn if not exporting - we're not writing to anything, so we can't persist.
if (ShouldPersistComments && !bDoExport)
{
UE_LOG(LogInternationalizationExportCommandlet, Warning, TEXT("ShouldPersistComments is true, but bExportLoc is false - can't persist comments if not writing PO files."));
ShouldPersistComments = false;
}
if( !bDoImport && !bDoExport )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Import/Export operation not detected. Use bExportLoc or bImportLoc in config section."));
return -1;
}
if( bDoImport )
{
if (!DoImport(SourcePath, DestinationPath, Filename))
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Failed to import localization files."));
return -1;
}
}
if( bDoExport )
{
if (!DoExport(SourcePath, DestinationPath, Filename))
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Failed to export localization files."));
return -1;
}
}
return 0;
}
int32 UGatherTextFromMetaDataCommandlet::Main( const FString& Params )
{
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Modules to Preload
TArray<FString> ModulesToPreload;
GetStringArrayFromConfig(*SectionName, TEXT("ModulesToPreload"), ModulesToPreload, GatherTextConfigPath);
for (const FString& ModuleName : ModulesToPreload)
{
FModuleManager::Get().LoadModule(*ModuleName);
}
// IncludePathFilters
TArray<FString> IncludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePathFilters"), IncludePathFilters, GatherTextConfigPath);
// IncludePaths (DEPRECATED)
{
TArray<FString> IncludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num())
{
IncludePathFilters.Append(IncludePaths);
UE_LOG(LogGatherTextFromMetaDataCommandlet, Warning, TEXT("IncludePaths detected in section %s. IncludePaths is deprecated, please use IncludePathFilters."), *SectionName);
}
}
if (IncludePathFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No include path filters in section %s."), *SectionName);
return -1;
}
// ExcludePathFilters
TArray<FString> ExcludePathFilters;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePathFilters"), ExcludePathFilters, GatherTextConfigPath);
// ExcludePaths (DEPRECATED)
{
TArray<FString> ExcludePaths;
GetPathArrayFromConfig(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
if (ExcludePaths.Num())
{
ExcludePathFilters.Append(ExcludePaths);
UE_LOG(LogGatherTextFromMetaDataCommandlet, Warning, TEXT("ExcludePaths detected in section %s. ExcludePaths is deprecated, please use ExcludePathFilters."), *SectionName);
}
}
// Get whether we should gather editor-only data. Typically only useful for the localization of UE4 itself.
if (!GetBoolFromConfig(*SectionName, TEXT("ShouldGatherFromEditorOnlyData"), ShouldGatherFromEditorOnlyData, GatherTextConfigPath))
{
ShouldGatherFromEditorOnlyData = false;
}
FGatherParameters Arguments;
GetStringArrayFromConfig(*SectionName, TEXT("InputKeys"), Arguments.InputKeys, GatherTextConfigPath);
GetStringArrayFromConfig(*SectionName, TEXT("OutputNamespaces"), Arguments.OutputNamespaces, GatherTextConfigPath);
TArray<FString> OutputKeys;
GetStringArrayFromConfig(*SectionName, TEXT("OutputKeys"), OutputKeys, GatherTextConfigPath);
for(const auto& OutputKey : OutputKeys)
{
Arguments.OutputKeys.Add(FText::FromString(OutputKey));
}
// Execute gather.
GatherTextFromUObjects(IncludePathFilters, ExcludePathFilters, Arguments);
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetPathArrayFromConfig(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
for (const FString& ManifestDependency : ManifestDependenciesList)
{
FText OutError;
if (!GatherManifestHelper->AddDependency(ManifestDependency, &OutError))
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("The GatherTextFromMetaData commandlet couldn't load the specified manifest dependency: '%'. %s"), *ManifestDependency, *OutError.ToString());
return -1;
}
}
return 0;
}
int32 URepairLocalizationDataCommandlet::Main(const FString& Params)
{
FInternationalization& I18N = FInternationalization::Get();
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// Get destination path.
FString DestinationPath;
if( !GetPathFromConfig( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath ) )
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
// Get manifest name.
FString ManifestName;
if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, GatherTextConfigPath ) )
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error, TEXT("No manifest name specified.") );
return -1;
}
// Get archive name.
FString ArchiveName;
if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath ) ) )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No archive name specified."));
return -1;
}
// Get cultures to generate.
TArray<FString> CulturesToGenerate;
GetStringArrayFromConfig(*SectionName, TEXT("CulturesToGenerate"), CulturesToGenerate, GatherTextConfigPath);
if( CulturesToGenerate.Num() == 0 )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No cultures specified for generation."));
return -1;
}
for(int32 i = 0; i < CulturesToGenerate.Num(); ++i)
{
if( I18N.GetCulture( CulturesToGenerate[i] ).IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(CulturesToGenerate[i]) );
}
}
//////////////////////////////////////////////////////////////////////////
// Read the damaged manifest.
const FString ManifestFilePath = DestinationPath / ManifestName;
const TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return -1;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
const TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
// Read the damaged archives.
TArray< TSharedRef<FInternationalizationArchive> > InternationalizationArchives;
TArray<FString> ArchiveCultures;
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
// Read in any existing archive for this culture.
const FString CulturePath = DestinationPath / CulturesToGenerate[Culture];
const FString ArchiveFilePath = CulturePath / ArchiveName;
if( FPaths::FileExists(ArchiveFilePath) )
{
TSharedPtr< FJsonObject > ArchiveJsonObject = ReadJSONTextFile( ArchiveFilePath );
if( !ArchiveJsonObject.IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("Could not read archive file %s."), *ArchiveFilePath);
return -1;
}
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
const TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
InternationalizationArchives.Add(InternationalizationArchive);
ArchiveCultures.Add(CulturesToGenerate[Culture]);
}
}
// Repair.
RepairManifestAndArchives(InternationalizationManifest, InternationalizationArchives);
// Write the repaired manifests.
TSharedRef< FJsonObject > OutputManifestJsonObj = MakeShareable( new FJsonObject );
ManifestSerializer.SerializeManifest( InternationalizationManifest, OutputManifestJsonObj );
if (!WriteJSONToTextFile( OutputManifestJsonObj, DestinationPath / ManifestName, SourceControlInfo ))
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error,TEXT("Failed to write manifest to %s."), *DestinationPath );
return -1;
}
// Write the repaired archives.
{
int Index = 0;
for (const TSharedRef<FInternationalizationArchive>& InternationalizationArchive : InternationalizationArchives)
{
TSharedRef< FJsonObject > OutputArchiveJsonObj = MakeShareable( new FJsonObject );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
ArchiveSerializer.SerializeArchive( InternationalizationArchive, OutputArchiveJsonObj );
int32 Culture = 0;
if (!WriteJSONToTextFile( OutputArchiveJsonObj, (DestinationPath / *ArchiveCultures[Index] / ArchiveName), SourceControlInfo ))
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error,TEXT("Failed to write archive to %s."), *DestinationPath );
return -1;
}
++Index;
}
}
return 0;
}
