bool FManifestInfo::AddManifestDependency( const FString& InManifestFilePath )
{
if( ManifestDependenciesFilePaths.Contains( InManifestFilePath ) )
{
return true;
}
bool bSuccess = false;
if( FPaths::FileExists( InManifestFilePath ) )
{
FJsonInternationalizationManifestSerializer ManifestSerializer;
TSharedPtr< FJsonObject > LoadedManifestDependencyJsonObject = UGatherTextCommandletBase::ReadJSONTextFile( InManifestFilePath );
TSharedRef< FInternationalizationManifest > LoadedManifestDependency = MakeShareable( new FInternationalizationManifest );
bSuccess = ManifestSerializer.DeserializeManifest( LoadedManifestDependencyJsonObject.ToSharedRef(), LoadedManifestDependency );
if( bSuccess )
{
ManifestDependencies.Add( LoadedManifestDependency );
ManifestDependenciesFilePaths.Add( InManifestFilePath );
}
else
{
UE_LOG(LogGatherTextCommandletBase, Warning, TEXT("Could not load manifest dependency file %s"), *InManifestFilePath);
}
}
else
{
bSuccess = false;
UE_LOG(LogGatherTextCommandletBase, Warning, TEXT("Could not find manifest dependency file %s"), *InManifestFilePath);
}
return bSuccess;
}
bool UGenerateGatherManifestCommandlet::WriteManifest( const TSharedPtr<FInternationalizationManifest>& InManifest, const FString& OutputFilePath )
{
// We can not continue if the provided manifest is not valid
if( !InManifest.IsValid() )
{
return false;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
TSharedRef<FJsonObject> JsonManifestObj = MakeShareable( new FJsonObject );
bool bSuccess = ManifestSerializer.SerializeManifest( InManifest.ToSharedRef(), JsonManifestObj );
if( bSuccess )
{
bSuccess = WriteJSONToTextFile( JsonManifestObj, OutputFilePath, SourceControlInfo );
}
return bSuccess;
}
int32 UGenerateGatherArchiveCommandlet::Main( const FString& Params )
{
FInternationalization& I18N = FInternationalization::Get();
// Parse command line - we're interested in the param vals
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// Get manifest name.
FString ManifestName;
if( !GetConfigString( *SectionName, TEXT("ManifestName"), ManifestName, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error, TEXT("No manifest name specified.") );
return -1;
}
// Get source culture.
FString SourceCulture;
if( GetConfigString( *SectionName, TEXT("SourceCulture"), SourceCulture, GatherTextConfigPath ) )
{
if( I18N.GetCulture( SourceCulture ).IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(SourceCulture) );
}
}
// Get cultures to generate.
TArray<FString> CulturesToGenerate;
GetConfigArray(*SectionName, TEXT("CulturesToGenerate"), CulturesToGenerate, GatherTextConfigPath);
if( CulturesToGenerate.Num() == 0 )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No cultures specified for generation."));
return -1;
}
for(int32 i = 0; i < CulturesToGenerate.Num(); ++i)
{
if( I18N.GetCulture( CulturesToGenerate[i] ).IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(CulturesToGenerate[i]) );
}
}
// Get destination path.
FString DestinationPath;
if( !GetConfigString( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
if (FPaths::IsRelative(DestinationPath))
{
if (!FPaths::GameDir().IsEmpty())
{
DestinationPath = FPaths::Combine( *( FPaths::GameDir() ), *DestinationPath );
}
else
{
DestinationPath = FPaths::Combine( *( FPaths::EngineDir() ), *DestinationPath );
}
}
// Get archive name.
FString ArchiveName;
if( !( GetConfigString(* SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath ) ) )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("No archive name specified."));
return -1;
}
// Get bPurgeOldEmptyEntries option.
bool ShouldPurgeOldEmptyEntries;
if ( !GetConfigBool( *SectionName, TEXT("bPurgeOldEmptyEntries"), ShouldPurgeOldEmptyEntries, GatherTextConfigPath) )
{
ShouldPurgeOldEmptyEntries = false;
}
FString ManifestFilePath = DestinationPath / ManifestName;
TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogGenerateArchiveCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return -1;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
BuildArchiveFromManifest( InternationalizationManifest, InternationalizationArchive, SourceCulture, CulturesToGenerate[Culture] );
const FString CulturePath = DestinationPath / CulturesToGenerate[Culture];
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
TSharedRef< FInternationalizationArchive > OutputInternationalizationArchive = MakeShareable( new FInternationalizationArchive );
// Read in any existing archive for this culture.
FString ExistingArchiveFileName = CulturePath / ArchiveName;
TSharedPtr< FJsonObject > ExistingArchiveJsonObject = NULL;
if( FPaths::FileExists(ExistingArchiveFileName) )
{
ExistingArchiveJsonObject = ReadJSONTextFile( ExistingArchiveFileName );
// Some of the existing archives were saved out with an "Unnamed" namespace for the root instead of the empty string. We try to fix that here.
if( ExistingArchiveJsonObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE ) )
{
FString RootNamespace = ExistingArchiveJsonObject->GetStringField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE );
if( RootNamespace == TEXT("Unnamed") )
{
ExistingArchiveJsonObject->RemoveField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE );
ExistingArchiveJsonObject->SetStringField( FJsonInternationalizationArchiveSerializer::TAG_NAMESPACE, TEXT("") );
}
}
struct Local
{
// Purges this JSONObject of an entries with no translated text and purges empty namespaces.
// Returns true if the object was modified, false if not.
static bool PurgeNamespaceOfEmptyEntries(const TSharedPtr<FJsonObject>& JSONObject)
{
bool ModifiedChildrenArray = false;
if( JSONObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_CHILDREN ) )
{
TArray<TSharedPtr<FJsonValue>> ChildrenArray = JSONObject->GetArrayField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN);
for( int32 ChildIndex = ChildrenArray.Num() - 1; ChildIndex >= 0; --ChildIndex )
{
TSharedPtr<FJsonObject> Child = ChildrenArray[ ChildIndex ]->AsObject();
TSharedPtr<FJsonObject> TranslationObject = Child->GetObjectField(FJsonInternationalizationArchiveSerializer::TAG_TRANSLATION);
const FString& TranslatedText = TranslationObject->GetStringField(FJsonInternationalizationArchiveSerializer::TAG_TRANSLATION_TEXT);
if(TranslatedText.IsEmpty())
{
ChildrenArray.RemoveAt( ChildIndex );
Child = NULL;
ModifiedChildrenArray = true;
}
}
if(ModifiedChildrenArray)
{
JSONObject->RemoveField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN);
if(ChildrenArray.Num())
{
JSONObject->SetArrayField(FJsonInternationalizationArchiveSerializer::TAG_CHILDREN, ChildrenArray);
}
}
}
bool ModifiedSubnamespaceArray = false;
if( JSONObject->HasField( FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES ) )
{
TArray<TSharedPtr<FJsonValue>> SubnamespaceArray = JSONObject->GetArrayField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES);
for( int32 Index = SubnamespaceArray.Num() - 1; Index >= 0; --Index )
{
TSharedPtr<FJsonObject> Subnamespace = SubnamespaceArray[ Index ]->AsObject();
ModifiedSubnamespaceArray = PurgeNamespaceOfEmptyEntries(Subnamespace);
bool HasChildren = Subnamespace->HasField( FJsonInternationalizationArchiveSerializer::TAG_CHILDREN );
bool HasSubnamespaces = Subnamespace->HasField( FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES );
if(!HasChildren && !HasSubnamespaces)
{
SubnamespaceArray.RemoveAt( Index );
Subnamespace = NULL;
ModifiedSubnamespaceArray = true;
}
}
if(ModifiedSubnamespaceArray)
{
JSONObject->RemoveField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES);
if(SubnamespaceArray.Num())
{
JSONObject->SetArrayField(FJsonInternationalizationArchiveSerializer::TAG_SUBNAMESPACES, SubnamespaceArray);
}
}
}
return ModifiedChildrenArray || ModifiedSubnamespaceArray;
}
};
if(ShouldPurgeOldEmptyEntries)
{
// Remove entries lacking translations from pre-existing archive.
// If they are absent in the source manifest, we save on not translating non-existent text.
// If they are present in the source manifest, then the newly generated entries will contain the empty text again.
Local::PurgeNamespaceOfEmptyEntries(ExistingArchiveJsonObject);
}
ArchiveSerializer.DeserializeArchive( ExistingArchiveJsonObject.ToSharedRef(), OutputInternationalizationArchive );
}
if (InternationalizationArchive->GetFormatVersion() < FInternationalizationArchive::EFormatVersion::Latest)
{
UE_LOG( LogGenerateArchiveCommandlet, Error,TEXT("Archive version is out of date. Repair the archives or manually set the version to %d."), static_cast<int32>(FInternationalizationArchive::EFormatVersion::Latest));
return -1;
}
// Combine the generated gather archive with the contents of the archive structure we will write out.
AppendArchiveData( InternationalizationArchive, OutputInternationalizationArchive );
InternationalizationArchive->SetFormatVersion(FInternationalizationArchive::EFormatVersion::Latest);
TSharedRef< FJsonObject > OutputArchiveJsonObj = MakeShareable( new FJsonObject );
ArchiveSerializer.SerializeArchive( OutputInternationalizationArchive, OutputArchiveJsonObj );
if( !WriteArchiveToFile( OutputArchiveJsonObj, DestinationPath, *CulturesToGenerate[Culture], *ArchiveName ) )
{
UE_LOG( LogGenerateArchiveCommandlet, Error,TEXT("Failed to write archive to %s."), *DestinationPath );
return -1;
}
}
return 0;
}
bool UInternationalizationExportCommandlet::DoImport(const FString& SourcePath, const FString& DestinationPath, const FString& Filename)
{
// Get manifest name.
FString ManifestName;
if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No manifest name specified.") );
return false;
}
// Get archive name.
FString ArchiveName;
if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, ConfigPath ) ) )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No archive name specified."));
return false;
}
// Get culture directory setting, default to true if not specified (used to allow picking of import directory with file open dialog from Translation Editor)
bool bUseCultureDirectory = true;
if (!(GetBoolFromConfig(*SectionName, TEXT("bUseCultureDirectory"), bUseCultureDirectory, ConfigPath)))
{
bUseCultureDirectory = true;
}
// Process the desired cultures
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
// Load the Portable Object file if found
const FString CultureName = CulturesToGenerate[Culture];
FString POFilePath = "";
if (bUseCultureDirectory)
{
POFilePath = SourcePath / CultureName / Filename;
}
else
{
POFilePath = SourcePath / Filename;
}
FPortableObjectFormatDOM PortableObject;
const bool HasLoadedPOFile = LoadPOFile(POFilePath, PortableObject);
if (!HasLoadedPOFile)
{
continue;
}
if (ShouldPersistComments)
{
PreserveExtractedCommentsForPersistence(PortableObject);
}
if (PortableObject.GetProjectName() != ManifestName.Replace(TEXT(".manifest"), TEXT("")))
{
UE_LOG(LogInternationalizationExportCommandlet, Log, TEXT("The project name (%s) in the file (%s) did not match the target manifest project (%s)."), *POFilePath, *PortableObject.GetProjectName(), *ManifestName.Replace(TEXT(".manifest"), TEXT("")));
}
const FString ManifestFileName = DestinationPath / ManifestName;
TSharedPtr< FJsonObject > ManifestJsonObject = NULL;
ManifestJsonObject = ReadJSONTextFile( ManifestFileName );
FJsonInternationalizationManifestSerializer ManifestSerializer;
TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
if( !FPaths::FileExists(ManifestFileName) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to find manifest %s."), *ManifestFileName);
continue;
}
const FString DestinationCulturePath = DestinationPath / CultureName;
const FString ArchiveFileName = DestinationCulturePath / ArchiveName;
if( !FPaths::FileExists(ArchiveFileName) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to find destination archive %s."), *ArchiveFileName);
continue;
}
TSharedPtr< FJsonObject > ArchiveJsonObject = NULL;
ArchiveJsonObject = ReadJSONTextFile( ArchiveFileName );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
bool bModifiedArchive = false;
{
for( auto EntryIter = PortableObject.GetEntriesIterator(); EntryIter; ++EntryIter )
{
auto POEntry = *EntryIter;
if( POEntry->MsgId.IsEmpty() || POEntry->MsgStr.Num() == 0 || POEntry->MsgStr[0].Trim().IsEmpty() )
{
// We ignore the header entry or entries with no translation.
continue;
}
// Some warning messages for data we don't process at the moment
if( !POEntry->MsgIdPlural.IsEmpty() || POEntry->MsgStr.Num() > 1 )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Portable Object entry has plural form we did not process. File: %s MsgCtxt: %s MsgId: %s"), *POFilePath, *POEntry->MsgCtxt, *POEntry->MsgId );
}
FString Key;
FString Namespace;
ParsePOMsgCtxtForIdentity(POEntry->MsgCtxt, Namespace, Key);
const FString& SourceText = ConditionPoStringForArchive(POEntry->MsgId);
const FString& Translation = ConditionPoStringForArchive(POEntry->MsgStr[0]);
TSharedPtr<FLocMetadataObject> KeyMetaDataObject;
// Get key metadata from the manifest, using the namespace and key.
if (!Key.IsEmpty())
{
// Find manifest entry by namespace
for (auto ManifestEntryIterator = InternationalizationManifest->GetEntriesByContextIdIterator(); ManifestEntryIterator; ++ManifestEntryIterator)
{
const FString& ManifestEntryNamespace = ManifestEntryIterator->Key;
const TSharedRef<FManifestEntry>& ManifestEntry = ManifestEntryIterator->Value;
if (ManifestEntry->Namespace == Namespace)
{
FContext* const MatchingContext = ManifestEntry->Contexts.FindByPredicate([&](FContext& Context) -> bool
{
return Context.Key == Key;
});
if (MatchingContext)
{
KeyMetaDataObject = MatchingContext->KeyMetadataObj;
}
}
}
}
//@TODO: Take into account optional entries and entries that differ by keymetadata. Ex. Each optional entry needs a unique msgCtxt
const TSharedPtr< FArchiveEntry > FoundEntry = InternationalizationArchive->FindEntryBySource( Namespace, SourceText, KeyMetaDataObject );
if( !FoundEntry.IsValid() )
{
UE_LOG(LogInternationalizationExportCommandlet, Warning, TEXT("Could not find corresponding archive entry for PO entry. File: %s MsgCtxt: %s MsgId: %s"), *POFilePath, *POEntry->MsgCtxt, *POEntry->MsgId );
continue;
}
if( FoundEntry->Translation != Translation )
{
FoundEntry->Translation = Translation;
bModifiedArchive = true;
}
}
}
if( bModifiedArchive )
{
TSharedRef<FJsonObject> FinalArchiveJsonObj = MakeShareable( new FJsonObject );
ArchiveSerializer.SerializeArchive( InternationalizationArchive, FinalArchiveJsonObj );
if( !WriteJSONToTextFile(FinalArchiveJsonObj, ArchiveFileName, SourceControlInfo ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to write archive to %s."), *ArchiveFileName );
return false;
}
}
}
return true;
}
/**
* UInternationalizationExportCommandlet
*/
bool UInternationalizationExportCommandlet::DoExport( const FString& SourcePath, const FString& DestinationPath, const FString& Filename )
{
// Get native culture.
FString NativeCultureName;
if( !GetStringFromConfig( *SectionName, TEXT("NativeCulture"), NativeCultureName, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No native culture specified.") );
return false;
}
// Get manifest name.
FString ManifestName;
if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No manifest name specified.") );
return false;
}
// Get archive name.
FString ArchiveName;
if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, ConfigPath ) ) )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No archive name specified."));
return false;
}
// Get culture directory setting, default to true if not specified (used to allow picking of export directory with windows file dialog from Translation Editor)
bool bUseCultureDirectory = true;
if (!(GetBoolFromConfig(*SectionName, TEXT("bUseCultureDirectory"), bUseCultureDirectory, ConfigPath)))
{
bUseCultureDirectory = true;
}
bool ShouldAddSourceLocationsAsComments = true;
GetBoolFromConfig(*SectionName, TEXT("ShouldAddSourceLocationsAsComments"), ShouldAddSourceLocationsAsComments, ConfigPath);
TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
// Load the manifest info
{
FString ManifestFilePath = SourcePath / ManifestName;
if( !FPaths::FileExists(ManifestFilePath) )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Could not find manifest file %s."), *ManifestFilePath);
return false;
}
TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return false;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
}
TArray< TSharedPtr<FInternationalizationArchive> > NativeArchives;
{
const FString NativeCulturePath = SourcePath / *(NativeCultureName);
TArray<FString> NativeArchiveFileNames;
IFileManager::Get().FindFiles(NativeArchiveFileNames, *(NativeCulturePath / TEXT("*.archive")), true, false);
for (const FString& NativeArchiveFileName : NativeArchiveFileNames)
{
// Read each archive file from the culture-named directory in the source path.
FString ArchiveFilePath = NativeCulturePath / NativeArchiveFileName;
ArchiveFilePath = FPaths::ConvertRelativePathToFull(ArchiveFilePath);
TSharedRef<FInternationalizationArchive> InternationalizationArchive = MakeShareable(new FInternationalizationArchive);
TSharedPtr< FJsonObject > ArchiveJsonObject = ReadJSONTextFile( ArchiveFilePath );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
NativeArchives.Add(InternationalizationArchive);
}
}
// Process the desired cultures
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
// Load the archive
const FString CultureName = CulturesToGenerate[Culture];
const FString CulturePath = SourcePath / CultureName;
FString ArchiveFileName = CulturePath / ArchiveName;
TSharedPtr< FJsonObject > ArchiveJsonObject = NULL;
if( FPaths::FileExists(ArchiveFileName) )
{
ArchiveJsonObject = ReadJSONTextFile( ArchiveFileName );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
{
FPortableObjectFormatDOM NewPortableObject;
FString LocLang;
if( !NewPortableObject.SetLanguage( CultureName ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Skipping export of loc language %s because it is not recognized."), *LocLang );
continue;
}
NewPortableObject.SetProjectName( FPaths::GetBaseFilename( ManifestName ) );
NewPortableObject.CreateNewHeader();
{
for(TManifestEntryBySourceTextContainer::TConstIterator ManifestIter = InternationalizationManifest->GetEntriesBySourceTextIterator(); ManifestIter; ++ManifestIter)
{
// Gather relevant info from manifest entry.
const TSharedRef<FManifestEntry>& ManifestEntry = ManifestIter.Value();
const FString& Namespace = ManifestEntry->Namespace;
const FLocItem& Source = ManifestEntry->Source;
// For each context, we may need to create a different or even multiple PO entries.
for( auto ContextIter = ManifestEntry->Contexts.CreateConstIterator(); ContextIter; ++ContextIter )
{
const FContext& Context = *ContextIter;
// Create the typical PO entry from the archive entry which matches the exact same namespace, source, and key metadata, if it exists.
{
const TSharedPtr<FArchiveEntry> ArchiveEntry = InternationalizationArchive->FindEntryBySource( Namespace, Source, Context.KeyMetadataObj );
if( ArchiveEntry.IsValid() )
{
const FString ConditionedArchiveSource = ConditionArchiveStrForPo(ArchiveEntry->Source.Text);
const FString ConditionedArchiveTranslation = ConditionArchiveStrForPo(ArchiveEntry->Translation.Text);
TSharedRef<FPortableObjectEntry> PoEntry = MakeShareable( new FPortableObjectEntry );
//@TODO: We support additional metadata entries that can be translated. How do those fit in the PO file format? Ex: isMature
PoEntry->MsgId = ConditionedArchiveSource;
PoEntry->MsgCtxt = ConditionIdentityForPOMsgCtxt(Namespace, Context.Key, Context.KeyMetadataObj);
PoEntry->MsgStr.Add( ConditionedArchiveTranslation );
const FString PORefString = ConvertSrcLocationToPORef( Context.SourceLocation );
PoEntry->AddReference(PORefString); // Source location.
PoEntry->AddExtractedComment( GetConditionedKeyForExtractedComment(Context.Key) ); // "Notes from Programmer" in the form of the Key.
if (ShouldAddSourceLocationsAsComments)
{
PoEntry->AddExtractedComment(GetConditionedReferenceForExtractedComment(PORefString)); // "Notes from Programmer" in the form of the Source Location, since this comes in handy too and OneSky doesn't properly show references, only comments.
}
TArray<FString> InfoMetaDataStrings;
if (Context.InfoMetadataObj.IsValid())
{
for (auto InfoMetaDataPair : Context.InfoMetadataObj->Values)
{
const FString KeyName = InfoMetaDataPair.Key;
const TSharedPtr<FLocMetadataValue> Value = InfoMetaDataPair.Value;
InfoMetaDataStrings.Add(GetConditionedInfoMetaDataForExtractedComment(KeyName, Value->AsString()));
}
}
if (InfoMetaDataStrings.Num())
{
PoEntry->AddExtractedComments(InfoMetaDataStrings);
}
NewPortableObject.AddEntry( PoEntry );
}
}
// If we're exporting for something other than the native culture, we'll need to create PO entries for archive entries based on the native archive's translation.
if (CultureName != NativeCultureName)
{
TSharedPtr<FArchiveEntry> NativeArchiveEntry;
// Find the native archive entry which matches the exact same namespace, source, and key metadata, if it exists.
for (const auto& NativeArchive : NativeArchives)
{
const TSharedPtr<FArchiveEntry> PotentialNativeArchiveEntry = NativeArchive->FindEntryBySource( Namespace, Source, Context.KeyMetadataObj );
if (PotentialNativeArchiveEntry.IsValid())
{
NativeArchiveEntry = PotentialNativeArchiveEntry;
break;
}
}
if (NativeArchiveEntry.IsValid())
{
// Only need to create this PO entry if the native archive entry's translation differs from its source, in which case we need to find the our translation of the native translation.
if (!NativeArchiveEntry->Source.IsExactMatch(NativeArchiveEntry->Translation))
{
const TSharedPtr<FArchiveEntry> ArchiveEntry = InternationalizationArchive->FindEntryBySource( Namespace, NativeArchiveEntry->Translation, NativeArchiveEntry->KeyMetadataObj );
if (ArchiveEntry.IsValid())
{
const FString ConditionedArchiveSource = ConditionArchiveStrForPo(ArchiveEntry->Source.Text);
const FString ConditionedArchiveTranslation = ConditionArchiveStrForPo(ArchiveEntry->Translation.Text);
TSharedRef<FPortableObjectEntry> PoEntry = MakeShareable( new FPortableObjectEntry );
//@TODO: We support additional metadata entries that can be translated. How do those fit in the PO file format? Ex: isMature
PoEntry->MsgId = ConditionedArchiveSource;
PoEntry->MsgCtxt = ConditionIdentityForPOMsgCtxt(Namespace, Context.Key, Context.KeyMetadataObj);
PoEntry->MsgStr.Add( ConditionedArchiveTranslation );
const FString PORefString = ConvertSrcLocationToPORef( Context.SourceLocation );
PoEntry->AddReference( PORefString ); // Source location.
PoEntry->AddExtractedComment( FString::Printf(TEXT("Key:\t%s"), *Context.Key) ); // "Notes from Programmer" in the form of the Key.
PoEntry->AddExtractedComment( FString::Printf(TEXT("SourceLocation:\t%s"), *PORefString) ); // "Notes from Programmer" in the form of the Source Location, since this comes in handy too and OneSky doesn't properly show references, only comments.
TArray<FString> InfoMetaDataStrings;
if (Context.InfoMetadataObj.IsValid())
{
for (auto InfoMetaDataPair : Context.InfoMetadataObj->Values)
{
const FString KeyName = InfoMetaDataPair.Key;
const TSharedPtr<FLocMetadataValue> Value = InfoMetaDataPair.Value;
InfoMetaDataStrings.Add(FString::Printf(TEXT("InfoMetaData:\t\"%s\" : \"%s\""), *KeyName, *Value->AsString()));
}
}
if (InfoMetaDataStrings.Num())
{
PoEntry->AddExtractedComments(InfoMetaDataStrings);
}
NewPortableObject.AddEntry( PoEntry );
}
}
}
}
}
}
}
// Write out the Portable Object to .po file.
{
FString OutputFileName;
if (bUseCultureDirectory)
{
OutputFileName = DestinationPath / CultureName / Filename;
}
else
{
OutputFileName = DestinationPath / Filename;
}
// Persist comments if requested.
if (ShouldPersistComments)
{
// Preserve comments from the specified file now, if they haven't already been.
if (!HasPreservedComments)
{
FPortableObjectFormatDOM ExistingPortableObject;
const bool HasLoadedPOFile = LoadPOFile(OutputFileName, ExistingPortableObject);
if (!HasLoadedPOFile)
{
return false;
}
PreserveExtractedCommentsForPersistence(ExistingPortableObject);
}
// Persist the comments into the new portable object we're going to be saving.
for (const auto& Pair : POEntryToCommentMap)
{
const TSharedPtr<FPortableObjectEntry> FoundEntry = NewPortableObject.FindEntry(Pair.Key.MsgId, Pair.Key.MsgIdPlural, Pair.Key.MsgCtxt);
if (FoundEntry.IsValid())
{
FoundEntry->AddExtractedComments(Pair.Value);
}
}
}
NewPortableObject.SortEntries();
if( SourceControlInfo.IsValid() )
{
FText SCCErrorText;
if (!SourceControlInfo->CheckOutFile(OutputFileName, SCCErrorText))
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Check out of file %s failed: %s"), *OutputFileName, *SCCErrorText.ToString());
return false;
}
}
//@TODO We force UTF8 at the moment but we want this to be based on the format found in the header info.
const FString OutputString = NewPortableObject.ToString();
if (!FFileHelper::SaveStringToFile(OutputString, *OutputFileName, FFileHelper::EEncodingOptions::ForceUTF8))
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Could not write file %s"), *OutputFileName );
return false;
}
}
}
}
}
return true;
}
bool UInternationalizationExportCommandlet::DoExport( const FString& SourcePath, const FString& DestinationPath, const FString& Filename )
{
// Get native culture.
FString NativeCultureName;
if( !GetStringFromConfig( *SectionName, TEXT("NativeCulture"), NativeCultureName, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No native culture specified.") );
return false;
}
// Get manifest name.
FString ManifestName;
if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, ConfigPath ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No manifest name specified.") );
return false;
}
// Get archive name.
FString ArchiveName;
if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, ConfigPath ) ) )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No archive name specified."));
return false;
}
// Get culture directory setting, default to true if not specified (used to allow picking of export directory with windows file dialog from Translation Editor)
bool bUseCultureDirectory = true;
if (!(GetBoolFromConfig(*SectionName, TEXT("bUseCultureDirectory"), bUseCultureDirectory, ConfigPath)))
{
bUseCultureDirectory = true;
}
TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
// Load the manifest info
{
FString ManifestFilePath = SourcePath / ManifestName;
if( !FPaths::FileExists(ManifestFilePath) )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Could not find manifest file %s."), *ManifestFilePath);
return false;
}
TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return false;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
}
TArray< TSharedPtr<FInternationalizationArchive> > NativeArchives;
{
const FString NativeCulturePath = DestinationPath / *(NativeCultureName);
TArray<FString> NativeArchiveFileNames;
IFileManager::Get().FindFiles(NativeArchiveFileNames, *(NativeCulturePath / TEXT("*.archive")), true, false);
for (const FString& NativeArchiveFileName : NativeArchiveFileNames)
{
// Read each archive file from the culture-named directory in the source path.
FString ArchiveFilePath = NativeCulturePath / NativeArchiveFileName;
ArchiveFilePath = FPaths::ConvertRelativePathToFull(ArchiveFilePath);
TSharedRef<FInternationalizationArchive> InternationalizationArchive = MakeShareable(new FInternationalizationArchive);
TSharedPtr< FJsonObject > ArchiveJsonObject = ReadJSONTextFile( ArchiveFilePath );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
NativeArchives.Add(InternationalizationArchive);
}
}
// Process the desired cultures
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
// Load the archive
const FString CultureName = CulturesToGenerate[Culture];
const FString CulturePath = SourcePath / CultureName;
FString ArchiveFileName = CulturePath / ArchiveName;
TSharedPtr< FJsonObject > ArchiveJsonObject = NULL;
if( FPaths::FileExists(ArchiveFileName) )
{
ArchiveJsonObject = ReadJSONTextFile( ArchiveFileName );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
{
FPortableObjectFormatDOM PortableObj;
FString LocLang;
if( !PortableObj.SetLanguage( CultureName ) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Skipping export of loc language %s because it is not recognized."), *LocLang );
continue;
}
PortableObj.SetProjectName( FPaths::GetBaseFilename( ManifestName ) );
PortableObj.CreateNewHeader();
{
for(TManifestEntryBySourceTextContainer::TConstIterator ManifestIter = InternationalizationManifest->GetEntriesBySourceTextIterator(); ManifestIter; ++ManifestIter)
{
// Gather relevant info from manifest entry.
const TSharedRef<FManifestEntry>& ManifestEntry = ManifestIter.Value();
const FString& Namespace = ManifestEntry->Namespace;
const FLocItem& Source = ManifestEntry->Source;
for( auto ContextIter = ManifestEntry->Contexts.CreateConstIterator(); ContextIter; ++ContextIter )
{
{
const TSharedPtr<FArchiveEntry> ArchiveEntry = InternationalizationArchive->FindEntryBySource( Namespace, Source, ContextIter->KeyMetadataObj );
if( ArchiveEntry.IsValid() )
{
const FString ConditionedArchiveSource = ConditionArchiveStrForPo(ArchiveEntry->Source.Text);
const FString ConditionedArchiveTranslation = ConditionArchiveStrForPo(ArchiveEntry->Translation.Text);
TSharedRef<FPortableObjectEntry> PoEntry = MakeShareable( new FPortableObjectEntry );
//@TODO: We support additional metadata entries that can be translated. How do those fit in the PO file format? Ex: isMature
PoEntry->MsgId = ConditionedArchiveSource;
//@TODO: Take into account optional entries and entries that differ by keymetadata. Ex. Each optional entry needs a unique msgCtxt
PoEntry->MsgCtxt = Namespace;
PoEntry->MsgStr.Add( ConditionedArchiveTranslation );
FString PORefString = ConvertSrcLocationToPORef( ContextIter->SourceLocation );
PoEntry->AddReference( PORefString ); // Source location.
PoEntry->AddExtractedComment( ContextIter->Key ); // "Notes from Programmer" in the form of the Key.
PoEntry->AddExtractedComment( PORefString ); // "Notes from Programmer" in the form of the Source Location, since this comes in handy too and OneSky doesn't properly show references, only comments.
PortableObj.AddEntry( PoEntry );
}
}
if (CultureName != NativeCultureName)
{
TSharedPtr<FArchiveEntry> NativeArchiveEntry;
for (const auto& NativeArchive : NativeArchives)
{
const TSharedPtr<FArchiveEntry> PotentialNativeArchiveEntry = NativeArchive->FindEntryBySource( Namespace, Source, ContextIter->KeyMetadataObj );
if (PotentialNativeArchiveEntry.IsValid())
{
NativeArchiveEntry = PotentialNativeArchiveEntry;
break;
}
}
if (NativeArchiveEntry.IsValid())
{
if (!NativeArchiveEntry->Source.IsExactMatch(NativeArchiveEntry->Translation))
{
const TSharedPtr<FArchiveEntry> ArchiveEntry = InternationalizationArchive->FindEntryBySource( Namespace, NativeArchiveEntry->Translation, NativeArchiveEntry->KeyMetadataObj );
const FString ConditionedArchiveSource = ConditionArchiveStrForPo(ArchiveEntry->Source.Text);
const FString ConditionedArchiveTranslation = ConditionArchiveStrForPo(ArchiveEntry->Translation.Text);
TSharedRef<FPortableObjectEntry> PoEntry = MakeShareable( new FPortableObjectEntry );
//@TODO: We support additional metadata entries that can be translated. How do those fit in the PO file format? Ex: isMature
PoEntry->MsgId = ConditionedArchiveSource;
//@TODO: Take into account optional entries and entries that differ by keymetadata. Ex. Each optional entry needs a unique msgCtxt
PoEntry->MsgCtxt = Namespace;
PoEntry->MsgStr.Add( ConditionedArchiveTranslation );
FString PORefString = ConvertSrcLocationToPORef( ContextIter->SourceLocation );
PoEntry->AddReference( PORefString ); // Source location.
PoEntry->AddExtractedComment( ContextIter->Key ); // "Notes from Programmer" in the form of the Key.
PoEntry->AddExtractedComment( PORefString ); // "Notes from Programmer" in the form of the Source Location, since this comes in handy too and OneSky doesn't properly show references, only comments.
PortableObj.AddEntry( PoEntry );
}
}
}
}
}
}
// Write out the Portable Object to .po file.
{
PortableObj.SortEntries();
FString OutputString = PortableObj.ToString();
FString OutputFileName = "";
if (bUseCultureDirectory)
{
OutputFileName = DestinationPath / CultureName / Filename;
}
else
{
OutputFileName = DestinationPath / Filename;
}
if( SourceControlInfo.IsValid() )
{
FText SCCErrorText;
if (!SourceControlInfo->CheckOutFile(OutputFileName, SCCErrorText))
{
UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("Check out of file %s failed: %s"), *OutputFileName, *SCCErrorText.ToString());
return false;
}
}
//@TODO We force UTF8 at the moment but we want this to be based on the format found in the header info.
if( !FFileHelper::SaveStringToFile(OutputString, *OutputFileName, FFileHelper::EEncodingOptions::ForceUTF8) )
{
UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Could not write file %s"), *OutputFileName );
return false;
}
}
}
}
}
return true;
}
int32 URepairLocalizationDataCommandlet::Main(const FString& Params)
{
FInternationalization& I18N = FInternationalization::Get();
// Parse command line.
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
//Set config file
const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
FString GatherTextConfigPath;
if ( ParamVal )
{
GatherTextConfigPath = *ParamVal;
}
else
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No config specified."));
return -1;
}
//Set config section
ParamVal = ParamVals.Find(FString(TEXT("Section")));
FString SectionName;
if ( ParamVal )
{
SectionName = *ParamVal;
}
else
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No config section specified."));
return -1;
}
// Get destination path.
FString DestinationPath;
if( !GetPathFromConfig( *SectionName, TEXT("DestinationPath"), DestinationPath, GatherTextConfigPath ) )
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error, TEXT("No destination path specified.") );
return -1;
}
// Get manifest name.
FString ManifestName;
if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, GatherTextConfigPath ) )
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error, TEXT("No manifest name specified.") );
return -1;
}
// Get archive name.
FString ArchiveName;
if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath ) ) )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No archive name specified."));
return -1;
}
// Get cultures to generate.
TArray<FString> CulturesToGenerate;
GetStringArrayFromConfig(*SectionName, TEXT("CulturesToGenerate"), CulturesToGenerate, GatherTextConfigPath);
if( CulturesToGenerate.Num() == 0 )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("No cultures specified for generation."));
return -1;
}
for(int32 i = 0; i < CulturesToGenerate.Num(); ++i)
{
if( I18N.GetCulture( CulturesToGenerate[i] ).IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Verbose, TEXT("Specified culture is not a valid runtime culture, but may be a valid base language: %s"), *(CulturesToGenerate[i]) );
}
}
//////////////////////////////////////////////////////////////////////////
// Read the damaged manifest.
const FString ManifestFilePath = DestinationPath / ManifestName;
const TSharedPtr<FJsonObject> ManifestJsonObject = ReadJSONTextFile( ManifestFilePath );
if( !ManifestJsonObject.IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("Could not read manifest file %s."), *ManifestFilePath);
return -1;
}
FJsonInternationalizationManifestSerializer ManifestSerializer;
const TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );
// Read the damaged archives.
TArray< TSharedRef<FInternationalizationArchive> > InternationalizationArchives;
TArray<FString> ArchiveCultures;
for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
{
// Read in any existing archive for this culture.
const FString CulturePath = DestinationPath / CulturesToGenerate[Culture];
const FString ArchiveFilePath = CulturePath / ArchiveName;
if( FPaths::FileExists(ArchiveFilePath) )
{
TSharedPtr< FJsonObject > ArchiveJsonObject = ReadJSONTextFile( ArchiveFilePath );
if( !ArchiveJsonObject.IsValid() )
{
UE_LOG(LogRepairLocalizationDataCommandlet, Error, TEXT("Could not read archive file %s."), *ArchiveFilePath);
return -1;
}
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
const TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
InternationalizationArchives.Add(InternationalizationArchive);
ArchiveCultures.Add(CulturesToGenerate[Culture]);
}
}
// Repair.
RepairManifestAndArchives(InternationalizationManifest, InternationalizationArchives);
// Write the repaired manifests.
TSharedRef< FJsonObject > OutputManifestJsonObj = MakeShareable( new FJsonObject );
ManifestSerializer.SerializeManifest( InternationalizationManifest, OutputManifestJsonObj );
if (!WriteJSONToTextFile( OutputManifestJsonObj, DestinationPath / ManifestName, SourceControlInfo ))
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error,TEXT("Failed to write manifest to %s."), *DestinationPath );
return -1;
}
// Write the repaired archives.
{
int Index = 0;
for (const TSharedRef<FInternationalizationArchive>& InternationalizationArchive : InternationalizationArchives)
{
TSharedRef< FJsonObject > OutputArchiveJsonObj = MakeShareable( new FJsonObject );
FJsonInternationalizationArchiveSerializer ArchiveSerializer;
ArchiveSerializer.SerializeArchive( InternationalizationArchive, OutputArchiveJsonObj );
int32 Culture = 0;
if (!WriteJSONToTextFile( OutputArchiveJsonObj, (DestinationPath / *ArchiveCultures[Index] / ArchiveName), SourceControlInfo ))
{
UE_LOG( LogRepairLocalizationDataCommandlet, Error,TEXT("Failed to write archive to %s."), *DestinationPath );
return -1;
}
++Index;
}
}
return 0;
}