void UGenerateGatherArchiveCommandlet::BuildArchiveFromManifest( TSharedRef< const FInternationalizationManifest > InManifest, TSharedRef< FInternationalizationArchive > Archive, const FString& SourceCulture, const FString& TargetCulture )
{
for(TManifestEntryByContextIdContainer::TConstIterator It( InManifest->GetEntriesByContextIdIterator() ); It; ++It)
{
const TSharedRef<FManifestEntry> UnstructuredManifestEntry = It.Value();
const FString& Namespace = UnstructuredManifestEntry->Namespace;
const FLocItem& Source = UnstructuredManifestEntry->Source;
for( auto ContextIter = UnstructuredManifestEntry->Contexts.CreateConstIterator(); ContextIter; ++ContextIter )
{
const FContext& Context = *ContextIter;
// We only add the non-optional entries
if( !Context.bIsOptional )
{
FLocItem Translation = Source;
if( SourceCulture != TargetCulture)
{
// We want to process the translation before adding it to the archive
ConditionTranslation( Translation );
}
// We also condition the source object
FLocItem ConditionedSource = Source;
ConditionSource( ConditionedSource );
Archive->AddEntry(Namespace, ConditionedSource, Translation, Context.KeyMetadataObj, Context.bIsOptional );
}
}
}
}
void UGenerateGatherArchiveCommandlet::AppendArchiveData( TSharedRef< const FInternationalizationArchive > InArchiveToAppend, TSharedRef< FInternationalizationArchive > ArchiveCombined )
{
for(TArchiveEntryContainer::TConstIterator It( InArchiveToAppend->GetEntryIterator() ); It; ++It)
{
const TSharedRef<FArchiveEntry> EntryToAppend = It.Value();
ArchiveCombined->AddEntry( EntryToAppend );
}
}
bool FInternationalizationArchiveJsonSerializer::JsonObjToArchive( TSharedRef< FJsonObject > InJsonObj, FString ParentNamespace, TSharedRef< FInternationalizationArchive > InternationalizationArchive )
{
bool bConvertSuccess = true;
FString AccumulatedNamespace = ParentNamespace;
if( InJsonObj->HasField( TAG_NAMESPACE) )
{
if( !( AccumulatedNamespace.IsEmpty() ) )
{
AccumulatedNamespace += NAMESPACE_DELIMITER;
}
AccumulatedNamespace += InJsonObj->GetStringField( TAG_NAMESPACE );
}
else
{
UE_LOG( LogInternationalizationArchiveSerializer, Warning,TEXT("Encountered an object with a missing namespace while converting to Internationalization archive.") );
bConvertSuccess = false;
}
// Process all the child objects
if( bConvertSuccess && InJsonObj->HasField( TAG_CHILDREN ) )
{
const TArray< TSharedPtr<FJsonValue> > ChildrenArray = InJsonObj->GetArrayField( TAG_CHILDREN );
for( TArray< TSharedPtr< FJsonValue > >::TConstIterator ChildIter( ChildrenArray.CreateConstIterator() ); ChildIter; ++ChildIter )
{
const TSharedPtr< FJsonValue > ChildEntry = *ChildIter;
const TSharedPtr< FJsonObject > ChildJSONObject = ChildEntry->AsObject();
FString SourceText;
TSharedPtr< FLocMetadataObject > SourceMetadata;
if( ChildJSONObject->HasTypedField< EJson::String >( TAG_DEPRECATED_DEFAULTTEXT ) )
{
SourceText = ChildJSONObject->GetStringField( TAG_DEPRECATED_DEFAULTTEXT );
}
else if( ChildJSONObject->HasTypedField< EJson::Object >( TAG_SOURCE ) )
{
const TSharedPtr< FJsonObject > SourceJSONObject = ChildJSONObject->GetObjectField( TAG_SOURCE );
if( SourceJSONObject->HasTypedField< EJson::String >( TAG_SOURCE_TEXT ) )
{
SourceText = SourceJSONObject->GetStringField( TAG_SOURCE_TEXT );
// Source meta data is mixed in with the source text, we'll process metadata if the source json object has more than one entry
if( SourceJSONObject->Values.Num() > 1 )
{
// We load in the entire source object as metadata and just remove the source text.
FInternationalizationMetaDataJsonSerializer::DeserializeMetadata( SourceJSONObject.ToSharedRef(), SourceMetadata );
if( SourceMetadata.IsValid() )
{
SourceMetadata->Values.Remove( TAG_SOURCE_TEXT );
}
}
}
else
{
bConvertSuccess = false;
}
}
else
{
bConvertSuccess = false;
}
FString TranslationText;
TSharedPtr< FLocMetadataObject > TranslationMetadata;
if( ChildJSONObject->HasTypedField< EJson::String >( TAG_DEPRECATED_TRANSLATEDTEXT ) )
{
TranslationText = ChildJSONObject->GetStringField( TAG_DEPRECATED_TRANSLATEDTEXT );
}
else if( ChildJSONObject->HasTypedField< EJson::Object >( TAG_TRANSLATION ) )
{
const TSharedPtr< FJsonObject > TranslationJSONObject = ChildJSONObject->GetObjectField( TAG_TRANSLATION );
if( TranslationJSONObject->HasTypedField< EJson::String >( TAG_TRANSLATION_TEXT ) )
{
TranslationText = TranslationJSONObject->GetStringField( TAG_TRANSLATION_TEXT );
// Source meta data is mixed in with the source text, we'll process metadata if the source json object has more than one entry
if( TranslationJSONObject->Values.Num() > 1 )
{
// We load in the entire source object as metadata and remove the source text
FInternationalizationMetaDataJsonSerializer::DeserializeMetadata( TranslationJSONObject.ToSharedRef(), TranslationMetadata );
if( TranslationJSONObject.IsValid() )
{
TranslationJSONObject->Values.Remove( TAG_TRANSLATION_TEXT );
}
}
}
else
{
bConvertSuccess = false;
}
}
else
{
bConvertSuccess = false;
}
if( bConvertSuccess )
{
FLocItem Source( SourceText );
Source.MetadataObj = SourceMetadata;
FLocItem Translation( TranslationText );
Translation.MetadataObj = TranslationMetadata;
bool bIsOptional = false;
if( ChildJSONObject->HasTypedField< EJson::Boolean >( TAG_OPTIONAL ) )
{
bIsOptional = ChildJSONObject->GetBoolField( TAG_OPTIONAL );
}
TSharedPtr< FLocMetadataObject > MetadataNode;
if( ChildJSONObject->HasTypedField< EJson::Object >( TAG_METADATA_KEY ) )
{
const TSharedPtr< FJsonObject > MetaDataKeyJSONObject = ChildJSONObject->GetObjectField( TAG_METADATA_KEY );
FInternationalizationMetaDataJsonSerializer::DeserializeMetadata( MetaDataKeyJSONObject.ToSharedRef(), MetadataNode );
}
bool bAddSuccessful = InternationalizationArchive->AddEntry( AccumulatedNamespace, Source, Translation, MetadataNode, bIsOptional );
if( !bAddSuccessful )
{
UE_LOG( LogInternationalizationArchiveSerializer, Warning,TEXT("Could not add JSON entry to the Internationalization archive: Namespace:%s DefaultText:%s"), *AccumulatedNamespace, *SourceText );
}
}
}
}
if( bConvertSuccess && InJsonObj->HasField( TAG_SUBNAMESPACES ) )
{
const TArray< TSharedPtr<FJsonValue> > SubnamespaceArray = InJsonObj->GetArrayField( TAG_SUBNAMESPACES );
for(TArray< TSharedPtr< FJsonValue > >::TConstIterator SubnamespaceIter( SubnamespaceArray.CreateConstIterator() ); SubnamespaceIter; ++SubnamespaceIter )
{
const TSharedPtr< FJsonValue > SubnamespaceEntry = *SubnamespaceIter;
const TSharedPtr< FJsonObject > SubnamespaceJSONObject = SubnamespaceEntry->AsObject();
if( !JsonObjToArchive( SubnamespaceJSONObject.ToSharedRef(), AccumulatedNamespace, InternationalizationArchive ) )
{
bConvertSuccess = false;
break;
}
}
}
return bConvertSuccess;
}
bool FJsonInternationalizationArchiveSerializer::JsonObjToArchive(TSharedRef<FJsonObject> InJsonObj, const FString& ParentNamespace, TSharedRef<FInternationalizationArchive> InArchive, TSharedPtr<const FInternationalizationManifest> InManifest, TSharedPtr<const FInternationalizationArchive> InNativeArchive)
{
bool bConvertSuccess = true;
FString AccumulatedNamespace = ParentNamespace;
if (InJsonObj->HasField(TAG_NAMESPACE))
{
if (!(AccumulatedNamespace.IsEmpty()))
{
AccumulatedNamespace += NAMESPACE_DELIMITER;
}
AccumulatedNamespace += InJsonObj->GetStringField(TAG_NAMESPACE);
}
else
{
UE_LOG(LogInternationalizationArchiveSerializer, Warning, TEXT("Encountered an object with a missing namespace while converting to Internationalization archive."));
bConvertSuccess = false;
}
// Process all the child objects
if (bConvertSuccess && InJsonObj->HasField(TAG_CHILDREN))
{
const TArray< TSharedPtr<FJsonValue> > ChildrenArray = InJsonObj->GetArrayField(TAG_CHILDREN);
for (TArray< TSharedPtr< FJsonValue > >::TConstIterator ChildIter(ChildrenArray.CreateConstIterator()); ChildIter; ++ChildIter)
{
const TSharedPtr< FJsonValue > ChildEntry = *ChildIter;
const TSharedPtr< FJsonObject > ChildJSONObject = ChildEntry->AsObject();
FString SourceText;
TSharedPtr< FLocMetadataObject > SourceMetadata;
if (ChildJSONObject->HasTypedField< EJson::String >(TAG_DEPRECATED_DEFAULTTEXT))
{
SourceText = ChildJSONObject->GetStringField(TAG_DEPRECATED_DEFAULTTEXT);
}
else if (ChildJSONObject->HasTypedField< EJson::Object >(TAG_SOURCE))
{
const TSharedPtr< FJsonObject > SourceJSONObject = ChildJSONObject->GetObjectField(TAG_SOURCE);
if (SourceJSONObject->HasTypedField< EJson::String >(TAG_SOURCE_TEXT))
{
SourceText = SourceJSONObject->GetStringField(TAG_SOURCE_TEXT);
// Source meta data is mixed in with the source text, we'll process metadata if the source json object has more than one entry
if (SourceJSONObject->Values.Num() > 1)
{
// We load in the entire source object as metadata and just remove the source text.
FJsonInternationalizationMetaDataSerializer::DeserializeMetadata(SourceJSONObject.ToSharedRef(), SourceMetadata);
if (SourceMetadata.IsValid())
{
SourceMetadata->Values.Remove(TAG_SOURCE_TEXT);
}
}
}
else
{
bConvertSuccess = false;
}
}
else
{
bConvertSuccess = false;
}
FString TranslationText;
TSharedPtr< FLocMetadataObject > TranslationMetadata;
if (ChildJSONObject->HasTypedField< EJson::String >(TAG_DEPRECATED_TRANSLATEDTEXT))
{
TranslationText = ChildJSONObject->GetStringField(TAG_DEPRECATED_TRANSLATEDTEXT);
}
else if (ChildJSONObject->HasTypedField< EJson::Object >(TAG_TRANSLATION))
{
const TSharedPtr< FJsonObject > TranslationJSONObject = ChildJSONObject->GetObjectField(TAG_TRANSLATION);
if (TranslationJSONObject->HasTypedField< EJson::String >(TAG_TRANSLATION_TEXT))
{
TranslationText = TranslationJSONObject->GetStringField(TAG_TRANSLATION_TEXT);
// Source meta data is mixed in with the source text, we'll process metadata if the source json object has more than one entry
if (TranslationJSONObject->Values.Num() > 1)
{
// We load in the entire source object as metadata and remove the source text
FJsonInternationalizationMetaDataSerializer::DeserializeMetadata(TranslationJSONObject.ToSharedRef(), TranslationMetadata);
if (TranslationJSONObject.IsValid())
{
TranslationJSONObject->Values.Remove(TAG_TRANSLATION_TEXT);
}
}
}
else
{
bConvertSuccess = false;
}
}
else
{
bConvertSuccess = false;
}
if (bConvertSuccess)
{
FLocItem Source(SourceText);
Source.MetadataObj = SourceMetadata;
FLocItem Translation(TranslationText);
Translation.MetadataObj = TranslationMetadata;
bool bIsOptional = false;
if (ChildJSONObject->HasTypedField< EJson::Boolean >(TAG_OPTIONAL))
{
bIsOptional = ChildJSONObject->GetBoolField(TAG_OPTIONAL);
}
TArray<FString> Keys;
TSharedPtr< FLocMetadataObject > KeyMetadataNode;
if (InArchive->GetFormatVersion() < FInternationalizationArchive::EFormatVersion::AddedKeys)
{
// We used to store the key meta-data as a top-level value, rather than within a "MetaData" object
if (ChildJSONObject->HasTypedField< EJson::Object >(TAG_METADATA_KEY))
{
const TSharedPtr< FJsonObject > MetaDataKeyJSONObject = ChildJSONObject->GetObjectField(TAG_METADATA_KEY);
FJsonInternationalizationMetaDataSerializer::DeserializeMetadata(MetaDataKeyJSONObject.ToSharedRef(), KeyMetadataNode);
}
if (InManifest.IsValid())
{
// We have no key in the archive data, so we must try and infer it from the manifest
FLocTextHelper::FindKeysForLegacyTranslation(InManifest.ToSharedRef(), InNativeArchive, AccumulatedNamespace, SourceText, KeyMetadataNode, Keys);
}
}
else
{
if (ChildJSONObject->HasTypedField< EJson::String >(TAG_KEY))
{
Keys.Add(ChildJSONObject->GetStringField(TAG_KEY));
}
if (ChildJSONObject->HasTypedField< EJson::Object >(TAG_METADATA))
{
const TSharedPtr< FJsonObject > MetaDataJSONObject = ChildJSONObject->GetObjectField(TAG_METADATA);
if (MetaDataJSONObject->HasTypedField< EJson::Object >(TAG_METADATA_KEY))
{
const TSharedPtr< FJsonObject > MetaDataKeyJSONObject = MetaDataJSONObject->GetObjectField(TAG_METADATA_KEY);
FJsonInternationalizationMetaDataSerializer::DeserializeMetadata(MetaDataKeyJSONObject.ToSharedRef(), KeyMetadataNode);
}
}
}
for (const FString& Key : Keys)
{
const bool bAddSuccessful = InArchive->AddEntry(AccumulatedNamespace, Key, Source, Translation, KeyMetadataNode, bIsOptional);
if (!bAddSuccessful)
{
UE_LOG(LogInternationalizationArchiveSerializer, Warning, TEXT("Could not add JSON entry to the Internationalization archive: Namespace:%s Key:%s DefaultText:%s"), *AccumulatedNamespace, *Key, *SourceText);
}
}
}
}
}
if (bConvertSuccess && InJsonObj->HasField(TAG_SUBNAMESPACES))
{
const TArray< TSharedPtr<FJsonValue> > SubnamespaceArray = InJsonObj->GetArrayField(TAG_SUBNAMESPACES);
for (TArray< TSharedPtr< FJsonValue > >::TConstIterator SubnamespaceIter(SubnamespaceArray.CreateConstIterator()); SubnamespaceIter; ++SubnamespaceIter)
{
const TSharedPtr< FJsonValue > SubnamespaceEntry = *SubnamespaceIter;
const TSharedPtr< FJsonObject > SubnamespaceJSONObject = SubnamespaceEntry->AsObject();
if (!JsonObjToArchive(SubnamespaceJSONObject.ToSharedRef(), AccumulatedNamespace, InArchive, InManifest, InNativeArchive))
{
bConvertSuccess = false;
break;
}
}
}
return bConvertSuccess;
}
bool UInternationalizationConditioningCommandlet::ProcessArchive( const FString& PrimaryLangExt, const FString& SourcePath, const FString& DestinationPath )
{
FString ArchiveName = TEXT("Archive.txt");
TArray<FString> LanguagesToProcess;
TArray<FString> TargetCultures;
bool bAppendToExistingArchive = true;
GetConfigString( *SectionName, TEXT("ArchiveName"), ArchiveName, GatherTextConfigPath );
GetConfigArray( *SectionName, TEXT("ProcessLanguage"), LanguagesToProcess, GatherTextConfigPath );
GetConfigArray( *SectionName, TEXT("TargetCulture"), TargetCultures, GatherTextConfigPath );
GetConfigBool( *SectionName, TEXT("bAppendToExistingArchive"), bAppendToExistingArchive, GatherTextConfigPath );
// Build info about the primary language
TArray<FString> PrimaryFilenames;
TArray<FString> PathPrimaryFilenames;
FString PrimaryLocDirectory = SourcePath / PrimaryLangExt + TEXT("/");
FString PrimaryWildcardName = PrimaryLocDirectory + TEXT("*.") + PrimaryLangExt;
// Grab the list of primary language loc files
IFileManager::Get().FindFiles(PathPrimaryFilenames, *PrimaryWildcardName, true, false);
for ( int32 FileIndex = 0; FileIndex < PathPrimaryFilenames.Num(); FileIndex++ )
{
FString* CompleteFilename = new(PrimaryFilenames) FString(PrimaryLocDirectory + PathPrimaryFilenames[FileIndex]);
}
if ( PrimaryFilenames.Num() == 0 )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("No primary language(%s) loc files found!"), *PrimaryLangExt);
return false;
}
for( int32 LanguageIndex = 0; LanguageIndex < LanguagesToProcess.Num(); LanguageIndex++ )
{
FString ForeignLangExt = LanguagesToProcess[LanguageIndex];
TArray<FString> ForeignFilenames;
TArray<FString> PathForeignFilenames;
FString ForeignLocDirectory = SourcePath / ForeignLangExt + TEXT("/");
FString ForeignWildcardName = ForeignLocDirectory + TEXT("*.") + ForeignLangExt;
FString TargetSubfolder = TargetCultures.Num() > LanguageIndex ? TargetCultures[LanguageIndex] : ForeignLangExt;
// Get a list of foreign loc files
IFileManager::Get().FindFiles(PathForeignFilenames, *ForeignWildcardName, true, false);
for ( int32 FileIndex = 0; FileIndex < PathForeignFilenames.Num(); FileIndex++ )
{
FString* CompleteFilename = new(ForeignFilenames) FString(ForeignLocDirectory + PathForeignFilenames[FileIndex]);
}
if ( ForeignFilenames.Num() == 0 )
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("No foreign loc files found using language extension '%s'"), *ForeignLangExt);
continue;
}
ReadLocFiles(PrimaryFilenames, ForeignFilenames);
TArray<FLocalizationFileEntry> ArchiveProperties;
// FSor each file in the list,
for ( int32 i = 0; i < LocPairs.Num(); i++ )
{
FLocalizationFilePair& Pair = LocPairs[i];
Pair.CompareFiles();
Pair.GetTranslatedProperties( ArchiveProperties );
Pair.GetIdenticalProperties( ArchiveProperties );
}
TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
FInternationalizationArchiveJsonSerializer ArchiveSerializer;
const FString DestinationArchiveFileName = DestinationPath / TargetSubfolder / ArchiveName;
// If we want to append to an existing archive, we first read it into our data structure
if( bAppendToExistingArchive )
{
FString ExistingArchiveFileName = DestinationArchiveFileName;
if( FPaths::FileExists(ExistingArchiveFileName) )
{
TSharedPtr< FJsonObject > ExistingArchiveJsonObject = ReadJSONTextFile( ExistingArchiveFileName );
if( ExistingArchiveJsonObject.IsValid() )
{
ArchiveSerializer.DeserializeArchive( ExistingArchiveJsonObject.ToSharedRef(), InternationalizationArchive );
}
}
}
for( int PropIndex = 0; PropIndex < ArchiveProperties.Num(); PropIndex++ )
{
FLocalizationFileEntry& Prop = ArchiveProperties[PropIndex];
FString NewNamespace = Prop.Namespace;
FLocItem Source( Prop.SourceText );
FLocItem Translation( Prop.TranslatedText );
if( !InternationalizationArchive->AddEntry( NewNamespace, Source, Translation, NULL, false ) )
{
TSharedPtr<FArchiveEntry> ExistingConflictEntry = InternationalizationArchive->FindEntryBySource( NewNamespace, Source, NULL );
if( !ExistingConflictEntry.IsValid() )
{
// Looks like we failed to add for a reason beyond conflicting translation, display an error and continue.
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("Failed to add entry to archive Namespace [%s]: (DEFAULT TEXT): %s (EXISTING TRANSLATION): %s"),
*NewNamespace, *Prop.SourceText, *ExistingConflictEntry->Translation.Text );
continue;
}
// If we can't add the entry, we find the existing conflicting entry and see if the translation is empty. If it is empty we will
// just overwrite the translation. If it is not empty we will display info about the conflict.
if( ExistingConflictEntry->Translation.Text.IsEmpty() )
{
ExistingConflictEntry->Translation.Text = Prop.TranslatedText;
}
else
{
UE_LOG(LogInternationalizationConditioningCommandlet, Warning, TEXT("Conflicting translation ignored in Namespace [%s]: (DEFAULT TEXT): %s (EXISTING TRANSLATION): %s (REJECTED TRANSLATION): %s"),
*NewNamespace, *Prop.SourceText, *ExistingConflictEntry->Translation.Text, *Prop.TranslatedText );
}
}
}
TSharedRef<FJsonObject> FinalArchiveJsonObj = MakeShareable( new FJsonObject );
ArchiveSerializer.SerializeArchive( InternationalizationArchive, FinalArchiveJsonObj );
WriteJSONToTextFile( FinalArchiveJsonObj, DestinationArchiveFileName, SourceControlInfo );
LocPairs.Empty();
}
return true;
}
