int32 UGenerateGatherArchiveCommandlet::Main( const FString& Params )
{
FInternationalization& I18N = FInternationalization::Get();
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// Get manifest name.
FString ManifestName;
if( !GetConfigString( *SectionName, TEXT("ManifestName"), ManifestName, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error, TEXT("No manifest name specified.") );
return -1;
}
// Get source culture.
FString SourceCulture;
if( GetConfigString( *SectionName, TEXT("SourceCulture"), SourceCulture, GatherTextConfigPath ) )
{
if( I18N.GetCulture( SourceCulture ).IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(SourceCulture) );
}
}
// Get cultures to generate.
TArray<FString> CulturesToGenerate;
GetConfigArray(*SectionName, TEXT("CulturesToGenerate"), CulturesToGenerate, GatherTextConfigPath);
if( CulturesToGenerate.Num() == 0 )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No cultures specified for generation."));
return -1;
}
for(int32 i = 0; i < CulturesToGenerate.Num(); ++i)
{
if( I18N.GetCulture( CulturesToGenerate[i] ).IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(CulturesToGenerate[i]) );
}
}
// Get destination path.
FString DestinationPath;
if( !GetConfigString( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
if (FPaths::IsRelative(DestinationPath))
{
if (!FPaths::GameDir().IsEmpty())
{
DestinationPath = FPaths::Combine( *( FPaths::GameDir() ), *DestinationPath );
}
else
{
DestinationPath = FPaths::Combine( *( FPaths::EngineDir() ), *DestinationPath );
}
}
// Get archive name.
FString ArchiveName;
if( !( GetConfigString(* SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath ) ) )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No archive name specified."));
return -1;
}
// Get bPurgeOldEmptyEntries option.
bool ShouldPurgeOldEmptyEntries;
if ( !GetConfigBool( *SectionName, TEXT("bPurgeOldEmptyEntries"), ShouldPurgeOldEmptyEntries, GatherTextConfigPath) )
{
ShouldPurgeOldEmptyEntries = false;
}
FString ManifestFilePath = DestinationPath / ManifestName;
TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return -1;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
BuildArchiveFromManifest( InternationalizationManifest, InternationalizationArchive, SourceCulture, CulturesToGenerate[Culture] );
const FString CulturePath = DestinationPath / CulturesToGenerate[Culture];
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
TSharedRef< FInternationalizationArchive > OutputInternationalizationArchive = MakeShareable( new FInternationalizationArchive );
// Read in any existing archive for this culture.
FString ExistingArchiveFileName = CulturePath / ArchiveName;
TSharedPtr< FJsonObject > ExistingArchiveJsonObject = NULL;
if( FPaths::FileExists(ExistingArchiveFileName) )
{
ExistingArchiveJsonObject = ReadJSONTextFile( ExistingArchiveFileName );
// Some of the existing archives were saved out with an "Unnamed" namespace for the root instead of the empty string. We try to fix that here.
if( ExistingArchiveJsonObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE ) )
{
FString RootNamespace = ExistingArchiveJsonObject->GetStringField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE );
if( RootNamespace == TEXT("Unnamed") )
{
ExistingArchiveJsonObject->RemoveField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE );
ExistingArchiveJsonObject->SetStringField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE, TEXT("") );
}
}
struct Local
{
// Purges this JSONObject of an entries with no translated text and purges empty namespaces.
// Returns true if the object was modified, false if not.
static bool PurgeNamespaceOfEmptyEntries(const TSharedPtr<FJsonObject>& JSONObject)
{
bool ModifiedChildrenArray = false;
if( JSONObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_CHILDREN ) )
{
TArray<TSharedPtr<FJsonValue>> ChildrenArray = JSONObject->GetArrayField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN);
for( int32 ChildIndex = ChildrenArray.Num() - 1; ChildIndex >= 0; --ChildIndex )
{
TSharedPtr<FJsonObject> Child = ChildrenArray[ ChildIndex ]->AsObject();
TSharedPtr<FJsonObject> TranslationObject = Child->GetObjectField(FJsonInternationalizationArchiveSerializer::TAG_TRANSLATION);
const FString& TranslatedText = TranslationObject->GetStringField(FJsonInternationalizationArchiveSerializer::TAG_TRANSLATION_TEXT);
if(TranslatedText.IsEmpty())
{
ChildrenArray.RemoveAt( ChildIndex );
Child = NULL;
ModifiedChildrenArray = true;
}
}
if(ModifiedChildrenArray)
{
JSONObject->RemoveField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN);
if(ChildrenArray.Num())
{
JSONObject->SetArrayField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN, ChildrenArray);
}
}
}
bool ModifiedSubnamespaceArray = false;
if( JSONObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES ) )
{
TArray<TSharedPtr<FJsonValue>> SubnamespaceArray = JSONObject->GetArrayField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES);
for( int32 Index = SubnamespaceArray.Num() - 1; Index >= 0; --Index )
{
TSharedPtr<FJsonObject> Subnamespace = SubnamespaceArray[ Index ]->AsObject();
ModifiedSubnamespaceArray = PurgeNamespaceOfEmptyEntries(Subnamespace);
bool HasChildren = Subnamespace->HasField( FJsonInternationalizationArchiveSerializer::TAG_CHILDREN );
bool HasSubnamespaces = Subnamespace->HasField( FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES );
if(!HasChildren && !HasSubnamespaces)
{
SubnamespaceArray.RemoveAt( Index );
Subnamespace = NULL;
ModifiedSubnamespaceArray = true;
}
}
if(ModifiedSubnamespaceArray)
{
JSONObject->RemoveField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES);
if(SubnamespaceArray.Num())
{
JSONObject->SetArrayField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES, SubnamespaceArray);
}
}
}
return ModifiedChildrenArray || ModifiedSubnamespaceArray;
}
};
if(ShouldPurgeOldEmptyEntries)
{
// Remove entries lacking translations from pre-existing archive.
// If they are absent in the source manifest, we save on not translating non-existent text.
// If they are present in the source manifest, then the newly generated entries will contain the empty text again.
Local::PurgeNamespaceOfEmptyEntries(ExistingArchiveJsonObject);
}
ArchiveSerializer.DeserializeArchive( ExistingArchiveJsonObject.ToSharedRef(), OutputInternationalizationArchive );
}
if (InternationalizationArchive->GetFormatVersion() < FInternationalizationArchive::EFormatVersion::Latest)
{
UE_LOG( LogGenerateArchiveCommandlet, Error,TEXT("Archive version is out of date. Repair the archives or manually set the version to %d."), static_cast<int32>(FInternationalizationArchive::EFormatVersion::Latest));
return -1;
}
// Combine the generated gather archive with the contents of the archive structure we will write out.
AppendArchiveData( InternationalizationArchive, OutputInternationalizationArchive );
InternationalizationArchive->SetFormatVersion(FInternationalizationArchive::EFormatVersion::Latest);
TSharedRef< FJsonObject > OutputArchiveJsonObj = MakeShareable( new FJsonObject );
ArchiveSerializer.SerializeArchive( OutputInternationalizationArchive, OutputArchiveJsonObj );
if( !WriteArchiveToFile( OutputArchiveJsonObj, DestinationPath, *CulturesToGenerate[Culture], *ArchiveName ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error,TEXT("Failed to write archive to %s."), *DestinationPath );
return -1;
}
}
return 0;
}
int32 UGenerateGatherManifestCommandlet::Main( const FString& Params )
{
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine( *Params, Tokens, Switches, ParamVals );
// Set config file.
const FString* ParamVal = ParamVals.Find( FString( TEXT("Config") ) );
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG( LogGenerateManifestCommandlet, Error, TEXT("No config specified.") );
return -1;
}
// Set config section.
ParamVal = ParamVals.Find( FString( TEXT("Section") ) );
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG( LogGenerateManifestCommandlet, Error, TEXT("No config section specified.") );
return -1;
}
// Get destination path.
FString DestinationPath;
if( !GetConfigString( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateManifestCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
// Get manifest name.
FString ManifestName;
if( !GetConfigString( *SectionName, TEXT("ManifestName"), ManifestName, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateManifestCommandlet, Error, TEXT("No manifest name specified.") );
return -1;
}
//Grab any manifest dependencies
TArray<FString> ManifestDependenciesList;
GetConfigArray(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( ManifestDependenciesList.Num() > 0 )
{
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGenerateManifestCommandlet, Error, TEXT("The GenerateGatherManifest commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
ManifestInfo->ApplyManifestDependencies();
}
if( !WriteManifest( ManifestInfo->GetManifest(), DestinationPath / ManifestName ) )
{
UE_LOG( LogGenerateManifestCommandlet, Error,TEXT("Failed to write manifest to %s."), *DestinationPath );
return -1;
}
return 0;
}
int32 UGatherTextFromAssetsCommandlet::Main(const FString& Params)
{
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Include paths
TArray<FString> IncludePaths;
GetConfigArray(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("No include paths in section %s"), *SectionName);
return -1;
}
//Exclude paths
TArray<FString> ExcludePaths;
GetConfigArray(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
//package extensions
TArray<FString> PackageExts;
GetConfigArray(*SectionName, TEXT("PackageExtensions"), PackageExts, GatherTextConfigPath);
if (PackageExts.Num() == 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No package extensions specified in section %s, using defaults"), *SectionName);
PackageExts.Add(FString("*") + FPackageName::GetAssetPackageExtension());
PackageExts.Add(FString("*") + FPackageName::GetMapPackageExtension());
}
//asset class exclude
TArray<FString> ExcludeClasses;
GetConfigArray(*SectionName, TEXT("ExcludeClasses"), ExcludeClasses, GatherTextConfigPath);
FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>(TEXT("AssetRegistry"));
AssetRegistryModule.Get().SearchAllAssets( true );
FARFilter Filter;
for(int32 i = 0; i < ExcludeClasses.Num(); i++)
{
UClass* FilterClass = FindObject<UClass>(ANY_PACKAGE, *ExcludeClasses[i]);
if(FilterClass)
{
Filter.ClassNames.Add( FilterClass->GetFName() );
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Invalid exclude class %s"), *ExcludeClasses[i]);
}
}
TArray<FAssetData> AssetData;
AssetRegistryModule.Get().GetAssets(Filter, AssetData);
FString UAssetPackageExtension = FPackageName::GetAssetPackageExtension();
TSet< FString > LongPackageNamesToExclude;
for (int Index = 0; Index < AssetData.Num(); Index++)
{
LongPackageNamesToExclude.Add( FPackageName::LongPackageNameToFilename( AssetData[Index].PackageName.ToString(), UAssetPackageExtension ) );
}
//Get whether we should fix broken properties that we find.
GetConfigBool(*SectionName, TEXT("bFixBroken"), bFixBroken, GatherTextConfigPath);
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetConfigArray(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Error, TEXT("The GatherTextFromAssets commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
//The main array of files to work from.
TArray< FString > PackageFileNamesToLoad;
TSet< FString > LongPackageNamesToProcess;
TArray<FString> PackageFilesNotInIncludePath;
TArray<FString> PackageFilesInExcludePath;
TArray<FString> PackageFilesExcludedByClass;
//Fill the list of packages to work from.
uint8 PackageFilter = NORMALIZE_DefaultFlags;
TArray<FString> Unused;
for ( int32 PackageFilenameWildcardIdx = 0; PackageFilenameWildcardIdx < PackageExts.Num(); PackageFilenameWildcardIdx++ )
{
const bool IsAssetPackage = PackageExts[PackageFilenameWildcardIdx] == ( FString( TEXT("*") )+ FPackageName::GetAssetPackageExtension() );
TArray<FString> PackageFiles;
if ( !NormalizePackageNames( Unused, PackageFiles, PackageExts[PackageFilenameWildcardIdx], PackageFilter) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("No packages found with extension %i: '%s'"), PackageFilenameWildcardIdx, *PackageExts[PackageFilenameWildcardIdx]);
continue;
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Display, TEXT("Found %i packages with extension %i: '%s'"), PackageFiles.Num(), PackageFilenameWildcardIdx, *PackageExts[PackageFilenameWildcardIdx]);
}
//Run through all the files found and add any that pass the include, exclude and filter constraints to OrderedPackageFilesToLoad
for( int32 PackageFileIdx=0; PackageFileIdx<PackageFiles.Num(); ++PackageFileIdx )
{
bool bExclude = false;
//Ensure it matches the include paths if there are some.
for( int32 IncludePathIdx=0; IncludePathIdx<IncludePaths.Num() ; ++IncludePathIdx )
{
bExclude = true;
if( PackageFiles[PackageFileIdx].MatchesWildcard(IncludePaths[IncludePathIdx]) )
{
bExclude = false;
break;
}
}
if ( bExclude )
{
PackageFilesNotInIncludePath.Add(PackageFiles[PackageFileIdx]);
}
//Ensure it does not match the exclude paths if there are some.
for( int32 ExcludePathIdx=0; !bExclude && ExcludePathIdx<ExcludePaths.Num() ; ++ExcludePathIdx )
{
if( PackageFiles[PackageFileIdx].MatchesWildcard(ExcludePaths[ExcludePathIdx]) )
{
bExclude = true;
PackageFilesInExcludePath.Add(PackageFiles[PackageFileIdx]);
break;
}
}
//Check that this is not on the list of packages that we don't care about e.g. textures.
if ( !bExclude && IsAssetPackage && LongPackageNamesToExclude.Contains( PackageFiles[PackageFileIdx] ) )
{
bExclude = true;
PackageFilesExcludedByClass.Add(PackageFiles[PackageFileIdx]);
}
//If we haven't failed one of the above checks, add it to the array of packages to process.
if(!bExclude)
{
TScopedPointer< FArchive > FileReader( IFileManager::Get().CreateFileReader( *PackageFiles[PackageFileIdx] ) );
if( FileReader )
{
// Read package file summary from the file
FPackageFileSummary PackageSummary;
(*FileReader) << PackageSummary;
// Early out check if the package has been flagged as needing localization gathering
if( PackageSummary.PackageFlags & PKG_RequiresLocalizationGather || PackageSummary.GetFileVersionUE4() < VER_UE4_PACKAGE_REQUIRES_LOCALIZATION_GATHER_FLAGGING )
{
PackageFileNamesToLoad.Add( PackageFiles[PackageFileIdx] );
}
}
}
}
}
if ( PackageFileNamesToLoad.Num() == 0 )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("No files found. Or none passed the include/exclude criteria."));
}
CollectGarbage( RF_Native );
//Now go through the remaining packages in the main array and process them in batches.
int32 PackagesPerBatchCount = 100;
TArray< UPackage* > LoadedPackages;
TArray< FString > LoadedPackageFileNames;
TArray< FString > FailedPackageFileNames;
TArray< UPackage* > PackagesToProcess;
const int32 PackageCount = PackageFileNamesToLoad.Num();
const int32 BatchCount = PackageCount / PackagesPerBatchCount + (PackageCount % PackagesPerBatchCount > 0 ? 1 : 0); // Add an extra batch for any remainder if necessary
if(PackageCount > 0)
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loading %i packages in %i batches of %i."), PackageCount, BatchCount, PackagesPerBatchCount);
}
//Load the packages in batches
int32 PackageIndex = 0;
for( int32 BatchIndex = 0; BatchIndex < BatchCount; ++BatchIndex )
{
int32 PackagesInThisBatch = 0;
for( PackageIndex; PackageIndex < PackageCount && PackagesInThisBatch < PackagesPerBatchCount; ++PackageIndex )
{
FString PackageFileName = PackageFileNamesToLoad[PackageIndex];
UPackage *Package = LoadPackage( NULL, *PackageFileName, LOAD_None );
if( Package )
{
LoadedPackages.Add(Package);
LoadedPackageFileNames.Add(PackageFileName);
// Because packages may not have been resaved after this flagging was implemented, we may have added packages to load that weren't flagged - potential false positives.
// The loading process should have reflagged said packages so that only true positives will have this flag.
if( Package->RequiresLocalizationGather() )
{
PackagesToProcess.Add( Package );
}
}
else
{
FailedPackageFileNames.Add( PackageFileName );
continue;
}
++PackagesInThisBatch;
}
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Loaded %i packages in batch %i of %i."), PackagesInThisBatch, BatchIndex + 1, BatchCount);
ProcessPackages(PackagesToProcess);
PackagesToProcess.Empty(PackagesPerBatchCount);
if( bFixBroken )
{
for( int32 LoadedPackageIndex=0; LoadedPackageIndex < LoadedPackages.Num() ; ++LoadedPackageIndex )
{
UPackage *Package = LoadedPackages[LoadedPackageIndex];
const FString PackageName = LoadedPackageFileNames[LoadedPackageIndex];
//Todo - link with source control.
if( Package )
{
if( Package->IsDirty() )
{
if( SavePackageHelper( Package, *PackageName ) )
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Saved Package %s."),*PackageName);
}
else
{
//TODO - Work out how to integrate with source control. The code from the source gatherer doesn't work.
UE_LOG(LogGatherTextFromAssetsCommandlet, Log, TEXT("Could not save package %s. Probably due to source control. "),*PackageName);
}
}
}
else
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Failed to find one of the loaded packages."));
}
}
}
CollectGarbage( RF_Native );
LoadedPackages.Empty(PackagesPerBatchCount);
LoadedPackageFileNames.Empty(PackagesPerBatchCount);
}
for(auto i = ConflictTracker.Namespaces.CreateConstIterator(); i; ++i)
{
const FString& NamespaceName = i.Key();
const FConflictTracker::FKeyTable& KeyTable = i.Value();
for(auto j = KeyTable.CreateConstIterator(); j; ++j)
{
const FString& KeyName = j.Key();
const FConflictTracker::FEntryArray& EntryArray = j.Value();
for(int k = 0; k < EntryArray.Num(); ++k)
{
const FConflictTracker::FEntry& Entry = EntryArray[k];
switch(Entry.Status)
{
case EAssetTextGatherStatus::MissingKey:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Detected missing key on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::MissingKey_Resolved:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Fixed missing key on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::IdentityConflict:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Detected duplicate identity with differing source on asset \"%s\"."), *Entry.ObjectPath);
}
break;
case EAssetTextGatherStatus::IdentityConflict_Resolved:
{
UE_LOG(LogGatherTextFromAssetsCommandlet, Warning, TEXT("Fixed duplicate identity with differing source on asset \"%s\"."), *Entry.ObjectPath);
}
break;
}
}
}
}
return 0;
}
int32 UGatherTextFromMetaDataCommandlet::Main( const FString& Params )
{
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Include paths
TArray<FString> IncludePaths;
GetConfigArray(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num() == 0)
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("No include paths in section %s"), *SectionName);
return -1;
}
//Exclude paths
TArray<FString> ExcludePaths;
GetConfigArray(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
//Required module names
TArray<FString> RequiredModuleNames;
GetConfigArray(*SectionName, TEXT("RequiredModuleNames"), ExcludePaths, GatherTextConfigPath);
// Pre-load all required modules so that UFields from those modules can be guaranteed a chance to have their metadata gathered.
for(const FString& RequiredModuleName : RequiredModuleNames)
{
FModuleManager::Get().LoadModule(*RequiredModuleName);
}
// Execute gather.
GatherTextFromUObjects(IncludePaths, ExcludePaths);
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetConfigArray(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromMetaDataCommandlet, Error, TEXT("The GatherTextFromMetaData commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
return 0;
}
int32 UGatherTextFromSourceCommandlet::Main( const FString& Params )
{
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config section specified."));
return -1;
}
//Include paths
TArray<FString> IncludePaths;
GetConfigArray(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
if (IncludePaths.Num() == 0)
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No include paths in section %s"), *SectionName);
return -1;
}
//Exclude paths
TArray<FString> ExcludePaths;
GetConfigArray(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
// Check the config section for source filters. e.g. *.cpp
TArray<FString> SourceFileSearchFilters;
GetConfigArray(*SectionName, TEXT("SourceFileSearchFilters"), SourceFileSearchFilters, GatherTextConfigPath);
if (SourceFileSearchFilters.Num() == 0)
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No source filters in section %s"), *SectionName);
return -1;
}
//Ensure all filters are unique.
TArray<FString> UniqueSourceFileSearchFilters;
for (int32 Idx=0; Idx<SourceFileSearchFilters.Num(); Idx++)
{
UniqueSourceFileSearchFilters.AddUnique(SourceFileSearchFilters[Idx]);
}
// Search in the root folder for each of the wildcard filters specified and build a list of files
TArray<FString> AllFoundFiles;
for(int32 i = 0; i < IncludePaths.Num(); i++)
{
for (int32 SourceFileSearchIdx=0; SourceFileSearchIdx < UniqueSourceFileSearchFilters.Num(); SourceFileSearchIdx++)
{
TArray<FString> RootSourceFiles;
IFileManager::Get().FindFilesRecursive(RootSourceFiles, *(FPaths::RootDir() + IncludePaths[i]), *UniqueSourceFileSearchFilters[SourceFileSearchIdx], true, false,false);
AllFoundFiles.Append(RootSourceFiles);
}
}
TArray<FString> FilesToProcess;
TArray<FString> RemovedList;
//Run through all the files found and add any that pass the include, exclude and filter constraints to PackageFilesToProcess
for( int32 FileIdx=0; FileIdx < AllFoundFiles.Num(); ++FileIdx )
{
bool bExclude = false;
//Ensure it does not match the exclude paths if there are some.
for( int32 ExcludePathIdx=0; ExcludePathIdx < ExcludePaths.Num() ; ++ExcludePathIdx )
{
if( AllFoundFiles[FileIdx].MatchesWildcard( ExcludePaths[ExcludePathIdx] ) )
{
bExclude = true;
RemovedList.Add( AllFoundFiles[FileIdx] );
break;
}
}
//If we haven't failed any checks, add it to the array of files to process.
if( !bExclude )
{
FilesToProcess.Add( AllFoundFiles[FileIdx] );
}
}
// Return if no source files were found
if( FilesToProcess.Num() == 0 )
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("The GatherTextFromSource commandlet couldn't find any source files in the specified directories."));
return -1;
}
// Add any manifest dependencies if they were provided
TArray<FString> ManifestDependenciesList;
GetConfigArray(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);
if( !ManifestInfo->AddManifestDependencies( ManifestDependenciesList ) )
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("The GatherTextFromSource commandlet couldn't find all the specified manifest dependencies."));
return -1;
}
// Get the loc macros and their syntax
TArray<FParsableDescriptor*> Parsables;
Parsables.Add(new FDefineDescriptor());
Parsables.Add(new FUndefDescriptor());
Parsables.Add(new FCommandMacroDescriptor());
// New Localization System with Namespace as literal argument.
Parsables.Add(new FStringMacroDescriptor( FString(TEXT("NSLOCTEXT")),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Namespace, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Identifier, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_SourceText, true)));
// New Localization System with Namespace as preprocessor define.
Parsables.Add(new FStringMacroDescriptor( FString(TEXT("LOCTEXT")),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_Identifier, true),
FMacroDescriptor::FMacroArg(FMacroDescriptor::MAS_SourceText, true)));
Parsables.Add(new FIniNamespaceDescriptor());
// Init a parse context to track the state of the file parsing
FSourceFileParseContext ParseCtxt;
ParseCtxt.ManifestInfo = ManifestInfo;
// Parse all source files for macros and add entries to SourceParsedEntries
for (int32 SourceFileIdx=0; SourceFileIdx < FilesToProcess.Num(); SourceFileIdx++)
{
ParseCtxt.Filename = FilesToProcess[SourceFileIdx];
ParseCtxt.LineNumber = 0;
ParseCtxt.ExcludedRegion = false;
ParseCtxt.WithinBlockComment = false;
ParseCtxt.WithinLineComment = false;
FString SourceFileText;
if (!FFileHelper::LoadFileToString(SourceFileText, *ParseCtxt.Filename))
{
UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("GatherTextSource failed to open file %s"), *ParseCtxt.Filename);
}
else
{
if (!ParseSourceText(SourceFileText, Parsables, ParseCtxt))
{
UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("GatherTextSource error(s) parsing source file %s"), *ParseCtxt.Filename);
}
}
}
// Clear parsables list safely
for (int32 i=0; i<Parsables.Num(); i++)
{
delete Parsables[i];
}
return 0;
}
int32 UInternationalizationConditioningCommandlet::Main( const FString& Params )
{
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogInternationalizationConditioningCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogInternationalizationConditioningCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// Common settings
FString SourcePath; // Source path to the root folder that manifest/archive files live in
FString DestinationPath; // Destination path that we will write conditioned archive and manifest files to. Language specific info will be appended to this path for archives.
FString PrimaryLangExt;
TArray<FString> LanguagesToProcess;
// Settings for generating/appending to archive files from legacy localization files
bool bGenerateArchiveFromLocIni = false;
// Settings for generating or appending entries to manifest from legacy localization files
bool bGenerateManifestFromLocIni = false;
// Get the common settings from config
GetConfigString( *SectionName, TEXT("SourcePath"), SourcePath, GatherTextConfigPath );
GetConfigString( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath );
GetConfigString( *SectionName, TEXT("PrimaryLanguage"), PrimaryLangExt, GatherTextConfigPath );
GetConfigArray( *SectionName, TEXT("ProcessLanguage"), LanguagesToProcess, GatherTextConfigPath );
GetConfigBool( *SectionName, TEXT("bGenerateManifestFromLocIni"), bGenerateManifestFromLocIni, GatherTextConfigPath );
GetConfigBool( *SectionName, TEXT("bGenerateArchiveFromLocIni"), bGenerateArchiveFromLocIni, GatherTextConfigPath );
// Load legacy localization files.
LoadLegacyLocalizationFiles(SourcePath, PrimaryLangExt, LanguagesToProcess);
// If features are enabled, we'll do those in order here
if( bGenerateManifestFromLocIni )
{
// Add to or create a manifest if desired
if( !ProcessManifest( PrimaryLangExt, SourcePath, DestinationPath ) )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Error, TEXT("Failed to generate manifest file from ini files."));
return -1;
}
}
if( bGenerateArchiveFromLocIni )
{
if( !ProcessArchive( PrimaryLangExt, SourcePath, DestinationPath ) )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Error, TEXT("Failed to generate manifest file from ini files."));
return -1;
}
}
return 0;
}
bool UInternationalizationConditioningCommandlet::ProcessArchive( const FString& PrimaryLangExt, const FString& SourcePath, const FString& DestinationPath )
{
FString ArchiveName = TEXT("Archive.txt");
TArray<FString> LanguagesToProcess;
TArray<FString> TargetCultures;
bool bAppendToExistingArchive = true;
GetConfigString( *SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath );
GetConfigArray( *SectionName, TEXT("ProcessLanguage"), LanguagesToProcess, GatherTextConfigPath );
GetConfigArray( *SectionName, TEXT("TargetCulture"), TargetCultures, GatherTextConfigPath );
GetConfigBool( *SectionName, TEXT("bAppendToExistingArchive"), bAppendToExistingArchive, GatherTextConfigPath );
// Build info about the primary language
TArray<FString> PrimaryFilenames;
TArray<FString> PathPrimaryFilenames;
FString PrimaryLocDirectory = SourcePath / PrimaryLangExt + TEXT("/");
FString PrimaryWildcardName = PrimaryLocDirectory + TEXT("*.") + PrimaryLangExt;
// Grab the list of primary language loc files
IFileManager::Get().FindFiles(PathPrimaryFilenames, *PrimaryWildcardName, true, false);
for ( int32 FileIndex = 0; FileIndex < PathPrimaryFilenames.Num(); FileIndex++ )
{
FString* CompleteFilename = new(PrimaryFilenames) FString(PrimaryLocDirectory + PathPrimaryFilenames[FileIndex]);
}
if ( PrimaryFilenames.Num() == 0 )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("No primary language(%s) loc files found!"), *PrimaryLangExt);
return false;
}
for( int32 LanguageIndex = 0; LanguageIndex < LanguagesToProcess.Num(); LanguageIndex++ )
{
FString ForeignLangExt = LanguagesToProcess[LanguageIndex];
TArray<FString> ForeignFilenames;
TArray<FString> PathForeignFilenames;
FString ForeignLocDirectory = SourcePath / ForeignLangExt + TEXT("/");
FString ForeignWildcardName = ForeignLocDirectory + TEXT("*.") + ForeignLangExt;
FString TargetSubfolder = TargetCultures.Num() > LanguageIndex ? TargetCultures[LanguageIndex] : ForeignLangExt;
// Get a list of foreign loc files
IFileManager::Get().FindFiles(PathForeignFilenames, *ForeignWildcardName, true, false);
for ( int32 FileIndex = 0; FileIndex < PathForeignFilenames.Num(); FileIndex++ )
{
FString* CompleteFilename = new(ForeignFilenames) FString(ForeignLocDirectory + PathForeignFilenames[FileIndex]);
}
if ( ForeignFilenames.Num() == 0 )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("No foreign loc files found using language extension '%s'"), *ForeignLangExt);
continue;
}
ReadLocFiles(PrimaryFilenames, ForeignFilenames);
TArray<FLocalizationFileEntry> ArchiveProperties;
// FSor each file in the list,
for ( int32 i = 0; i < LocPairs.Num(); i++ )
{
FLocalizationFilePair& Pair = LocPairs[i];
Pair.CompareFiles();
Pair.GetTranslatedProperties( ArchiveProperties );
Pair.GetIdenticalProperties( ArchiveProperties );
}
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
FInternationalizationArchiveJsonSerializer ArchiveSerializer;
const FString DestinationArchiveFileName = DestinationPath / TargetSubfolder / ArchiveName;
// If we want to append to an existing archive, we first read it into our data structure
if( bAppendToExistingArchive )
{
FString ExistingArchiveFileName = DestinationArchiveFileName;
if( FPaths::FileExists(ExistingArchiveFileName) )
{
TSharedPtr< FJsonObject > ExistingArchiveJsonObject = ReadJSONTextFile( ExistingArchiveFileName );
if( ExistingArchiveJsonObject.IsValid() )
{
ArchiveSerializer.DeserializeArchive( ExistingArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
}
}
}
for( int PropIndex = 0; PropIndex < ArchiveProperties.Num(); PropIndex++ )
{
FLocalizationFileEntry& Prop = ArchiveProperties[PropIndex];
FString NewNamespace = Prop.Namespace;
FLocItem Source( Prop.SourceText );
FLocItem Translation( Prop.TranslatedText );
if( !InternationalizationArchive->AddEntry( NewNamespace, Source, Translation, NULL, false ) )
{
TSharedPtr<FArchiveEntry> ExistingConflictEntry = InternationalizationArchive->FindEntryBySource( NewNamespace, Source, NULL );
if( !ExistingConflictEntry.IsValid() )
{
// Looks like we failed to add for a reason beyond conflicting translation, display an error and continue.
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("Failed to add entry to archive Namespace [%s]: (DEFAULT TEXT): %s (EXISTING TRANSLATION): %s"),
*NewNamespace, *Prop.SourceText, *ExistingConflictEntry->Translation.Text );
continue;
}
// If we can't add the entry, we find the existing conflicting entry and see if the translation is empty. If it is empty we will
// just overwrite the translation. If it is not empty we will display info about the conflict.
if( ExistingConflictEntry->Translation.Text.IsEmpty() )
{
ExistingConflictEntry->Translation.Text = Prop.TranslatedText;
}
else
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("Conflicting translation ignored in Namespace [%s]: (DEFAULT TEXT): %s (EXISTING TRANSLATION): %s (REJECTED TRANSLATION): %s"),
*NewNamespace, *Prop.SourceText, *ExistingConflictEntry->Translation.Text, *Prop.TranslatedText );
}
}
}
TSharedRef<FJsonObject> FinalArchiveJsonObj = MakeShareable( new FJsonObject );
ArchiveSerializer.SerializeArchive( InternationalizationArchive, FinalArchiveJsonObj );
WriteJSONToTextFile( FinalArchiveJsonObj, DestinationArchiveFileName, SourceControlInfo );
LocPairs.Empty();
}
return true;
}