void UGenerateGatherArchiveCommandlet::BuildArchiveFromManifest( TSharedRef< const FInternationalizationManifest > InManifest, TSharedRef< FInternationalizationArchive > Archive, const FString& SourceCulture, const FString& TargetCulture )
{
    for(TManifestEntryByContextIdContainer::TConstIterator It( InManifest->GetEntriesByContextIdIterator() ); It; ++It)
    {
        const TSharedRef<FManifestEntry> UnstructuredManifestEntry = It.Value();
        const FString& Namespace = UnstructuredManifestEntry->Namespace;
        const FLocItem& Source = UnstructuredManifestEntry->Source;

        for( auto ContextIter = UnstructuredManifestEntry->Contexts.CreateConstIterator(); ContextIter; ++ContextIter )
        {
            const FContext& Context = *ContextIter;

            // We only add the non-optional entries
            if( !Context.bIsOptional )
            {
                FLocItem Translation = Source;

                if( SourceCulture != TargetCulture)
                {
                    // We want to process the translation before adding it to the archive
                    ConditionTranslation( Translation );
                }

                // We also condition the source object
                FLocItem ConditionedSource = Source;
                ConditionSource( ConditionedSource );

                Archive->AddEntry(Namespace, ConditionedSource, Translation, Context.KeyMetadataObj, Context.bIsOptional );

            }
        }
    }
}
void FInternationalizationManifestJsonSerializer::GenerateStructuredData( TSharedRef< const FInternationalizationManifest > InManifest, TSharedPtr< FStructuredEntry > RootElement )
{
	//Loop through all the unstructured manifest entries and build up our structured hierarchy
	for( TManifestEntryByContextIdContainer::TConstIterator It( InManifest->GetEntriesByContextIdIterator() ); It; ++It )
	{
		const TSharedRef< FManifestEntry > UnstructuredManifestEntry = It.Value();

		TArray< FString > NamespaceTokens;

		// Tokenize the namespace by using '.' as a delimiter
		int32 NamespaceTokenCount = UnstructuredManifestEntry->Namespace.ParseIntoArray( &NamespaceTokens, *NAMESPACE_DELIMITER, true );

		TSharedPtr< FStructuredEntry > StructuredManifestEntry = RootElement;
		//Loop through all the namespace tokens and find the appropriate structured entry, if it does not exist add it.  At the end StructuredManifestEntry
		//  will point to the correct hierarchy entry for a given namespace
		for( int32 TokenIndex = 0; TokenIndex < NamespaceTokenCount; ++TokenIndex )
		{
			TSharedPtr< FStructuredEntry > FoundNamespaceEntry;
			for( int SubNamespaceIndex = 0; SubNamespaceIndex < StructuredManifestEntry->SubNamespaces.Num(); SubNamespaceIndex++ )
			{
				if(  StructuredManifestEntry->SubNamespaces[SubNamespaceIndex]->Namespace == NamespaceTokens[TokenIndex] )
				{
					FoundNamespaceEntry = StructuredManifestEntry->SubNamespaces[SubNamespaceIndex];
					break;
				}
			}

			if( !FoundNamespaceEntry.IsValid() )
			{
				int32 index = StructuredManifestEntry->SubNamespaces.Add( MakeShareable( new FStructuredEntry( NamespaceTokens[TokenIndex] ) ) );
				FoundNamespaceEntry = StructuredManifestEntry->SubNamespaces[index];
			}
			StructuredManifestEntry = FoundNamespaceEntry;
		}

		// We add the unstructured manifest entry to the hierarchy
		StructuredManifestEntry->ManifestEntries.AddUnique( UnstructuredManifestEntry );
	}
}
bool FTextLocalizationResourceGenerator::Generate(const FString& SourcePath, const TSharedRef<FInternationalizationManifest>& InternationalizationManifest, const FString& NativeCulture, const FString& CultureToGenerate, FArchive* const DestinationArchive, IInternationalizationArchiveSerializer& ArchiveSerializer)
{
	FLocalizationEntryTracker LocalizationEntryTracker;

	// Find archives in the native culture-specific folder.
	const FString NativeCulturePath = SourcePath / *(NativeCulture);
	TArray<FString> NativeArchiveFileNames;
	IFileManager::Get().FindFiles(NativeArchiveFileNames, *(NativeCulturePath / TEXT("*.archive")), true, false);

	// Find archives in the culture-specific folder.
	const FString CulturePath = SourcePath / *(CultureToGenerate);
	TArray<FString> ArchiveFileNames;
	IFileManager::Get().FindFiles(ArchiveFileNames, *(CulturePath / TEXT("*.archive")), true, false);

	if(NativeArchiveFileNames.Num() == 0)
	{
		FString Message = FString::Printf(TEXT("No archives were found for native culture %s."), *(CultureToGenerate));
		UE_LOG(LogTextLocalizationResourceGenerator, Warning, TEXT("%s"), *Message);
	}

	if(ArchiveFileNames.Num() == 0)
	{
		FString Message = FString::Printf(TEXT("No archives were found for culture %s."), *(CultureToGenerate));
		UE_LOG(LogTextLocalizationResourceGenerator, Warning, TEXT("%s"), *Message);
	}

	TArray< TSharedPtr<FInternationalizationArchive> > NativeArchives;
	for (const FString& NativeArchiveFileName : NativeArchiveFileNames)
	{
		// Read each archive file from the culture-named directory in the source path.
		FString ArchiveFilePath = NativeCulturePath / NativeArchiveFileName;
		ArchiveFilePath = FPaths::ConvertRelativePathToFull(ArchiveFilePath);
		TSharedRef<FInternationalizationArchive> InternationalizationArchive = MakeShareable(new FInternationalizationArchive);
#if 0 // @todo Json: Serializing from FArchive is currently broken
		FArchive* ArchiveFile = IFileManager::Get().CreateFileReader(*ArchiveFilePath);

		if (ArchiveFile == nullptr)
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("No archive found at %s."), *ArchiveFilePath);
			continue;
		}

		ArchiveSerializer.DeserializeArchive(*ArchiveFile, InternationalizationArchive);
#else
		FString ArchiveContent;

		if (!FFileHelper::LoadFileToString(ArchiveContent, *ArchiveFilePath))
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("Failed to load file %s."), *ArchiveFilePath);
			continue;
		}

		if (!ArchiveSerializer.DeserializeArchive(ArchiveContent, InternationalizationArchive))
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("Failed to serialize archive from file %s."), *ArchiveFilePath);
			continue;
		}
#endif

		NativeArchives.Add(InternationalizationArchive);
	}

	// For each archive:
	for (const FString& ArchiveFileName : ArchiveFileNames)
	{
		// Read each archive file from the culture-named directory in the source path.
		FString ArchiveFilePath = CulturePath / ArchiveFileName;
		ArchiveFilePath = FPaths::ConvertRelativePathToFull(ArchiveFilePath);
		TSharedRef<FInternationalizationArchive> InternationalizationArchive = MakeShareable(new FInternationalizationArchive);

#if 0 // @todo Json: Serializing from FArchive is currently broken
		FArchive* ArchiveFile = IFileManager::Get().CreateFileReader(*ArchiveFilePath);

		if (ArchiveFile == nullptr)
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("No archive found at %s."), *ArchiveFilePath);
			continue;
		}
			
		ArchiveSerializer.DeserializeArchive(*ArchiveFile, InternationalizationArchive);
#else
		FString ArchiveContent;

		if (!FFileHelper::LoadFileToString(ArchiveContent, *ArchiveFilePath))
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("Failed to load file %s."), *ArchiveFilePath);
			continue;
		}

		if (!ArchiveSerializer.DeserializeArchive(ArchiveContent, InternationalizationArchive))
		{
			UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("Failed to serialize archive from file %s."), *ArchiveFilePath);
			continue;
		}
#endif

		// Generate text localization resource from manifest and archive entries.
		for(TManifestEntryByContextIdContainer::TConstIterator i = InternationalizationManifest->GetEntriesByContextIdIterator(); i; ++i)
		{
			// Gather relevant info from manifest entry.
			const TSharedRef<FManifestEntry>& ManifestEntry = i.Value();
			const FString& Namespace = ManifestEntry->Namespace;
			const FLocItem& Source = ManifestEntry->Source;
			const FString& SourceString = Source.Text;
			const FString UnescapedSourceString = SourceString;
			const uint32 SourceStringHash = FCrc::StrCrc32(*UnescapedSourceString);

			FLocalizationEntryTracker::FKeyTable& KeyTable = LocalizationEntryTracker.Namespaces.FindOrAdd(*Namespace);

			// Keeps track of the key strings of non-optional manifest entries that are missing a corresponding archive entry.
			FString MissingArchiveEntryKeyStrings;

			// Keeps track of the key strings of non-optional manifest entries that are missing a translation
			FString MissingArchiveTranslationKeyStrings;

			// Create a localization entry for each namespace and key combination.
			for(auto ContextIt = ManifestEntry->Contexts.CreateConstIterator(); ContextIt; ++ContextIt)
			{
				const FString& Key = ContextIt->Key;

				// Get proper source string to use - if the native source is different than the native translation, the native translation becomes the source.
				FLocItem SourceToUse = Source;
				if (CultureToGenerate != NativeCulture)
				{
					for (const auto& NativeArchive : NativeArchives)
					{
						if (NativeArchive.IsValid())
						{
							const TSharedPtr<FArchiveEntry> NativeArchiveEntry = NativeArchive->FindEntryBySource(Namespace, Source, ContextIt->KeyMetadataObj);
							if (NativeArchiveEntry.IsValid() && !NativeArchiveEntry->Source.IsExactMatch(NativeArchiveEntry->Translation))
							{
								SourceToUse = NativeArchiveEntry->Translation;
								break;
							}
						}
					}
				}
				// Find matching archive entry.
				TSharedPtr<FArchiveEntry> ArchiveEntry = InternationalizationArchive->FindEntryBySource(Namespace, SourceToUse, ContextIt->KeyMetadataObj);

				if(ContextIt->bIsOptional && (!ArchiveEntry.IsValid() || (ArchiveEntry.IsValid() && ArchiveEntry->Translation.Text.IsEmpty())))
				{
					// Skip any optional manifest entries that do not have a matching archive entry or if the matching archive entry does not have a translation
					continue;
				}

				FString UnescapedTranslatedString;
				if( ArchiveEntry.IsValid() )
				{
					UnescapedTranslatedString = ArchiveEntry->Translation.Text;
					
					if(UnescapedTranslatedString.IsEmpty())
					{
						if ( !MissingArchiveTranslationKeyStrings.IsEmpty() )
						{
							MissingArchiveTranslationKeyStrings += TEXT(", ");
						}
						MissingArchiveTranslationKeyStrings += Key;
					}
				}
				else
				{
					// If we're not using the actual source string, then embed the fake source string as the foreign translation when we have no actual foreign translation.
					if (!Source.IsExactMatch(SourceToUse))
					{
						UnescapedTranslatedString = SourceToUse.Text;
					}

					if ( !MissingArchiveEntryKeyStrings.IsEmpty() )
					{
						MissingArchiveEntryKeyStrings += TEXT(", ");
					}
					MissingArchiveEntryKeyStrings += Key;
				}

				FLocalizationEntryTracker::FEntryArray& EntryArray = KeyTable.FindOrAdd(*Key);
				if(ArchiveEntry.IsValid())
				{
					FLocalizationEntryTracker::FEntry NewEntry;
					NewEntry.ArchiveName = ArchiveFilePath;
					NewEntry.LocalizedString = UnescapedTranslatedString;
					NewEntry.SourceStringHash = SourceStringHash;
					EntryArray.Add(NewEntry);
				}
			}

			if(!MissingArchiveEntryKeyStrings.IsEmpty())
			{
				FString KeyListString = FString::Printf(TEXT("[%s]"), *MissingArchiveEntryKeyStrings);
				FString Message = FString::Printf( TEXT("Archive (%s) contains no translation for entry (Namespace:%s, Source:%s) for keys: %s."), *ArchiveFilePath, *Namespace, *SourceString, *KeyListString);
				UE_LOG(LogTextLocalizationResourceGenerator, Verbose, TEXT("%s"), *Message);
			}

			if(!MissingArchiveTranslationKeyStrings.IsEmpty())
			{
				FString KeyListString = FString::Printf(TEXT("[%s]"), *MissingArchiveTranslationKeyStrings);
				FString Message = FString::Printf( TEXT("Archive (%s) contains empty translation for entry (Namespace:%s, Source:%s) with keys: %s."), *ArchiveFilePath, *Namespace, *SourceString, *KeyListString);
				UE_LOG(LogTextLocalizationResourceGenerator, Verbose, TEXT("%s"), *Message);
			}
		}
	}

	LocalizationEntryTracker.ReportCollisions();

	// Write resource.
	if( !(LocalizationEntryTracker.WriteToArchive(DestinationArchive)) )
	{
		UE_LOG(LogTextLocalizationResourceGenerator, Error, TEXT("Failed to write localization entries to archive."));
		return false;
	}

	return true;
}
bool UInternationalizationExportCommandlet::DoImport(const FString& SourcePath, const FString& DestinationPath, const FString& Filename)
{
	// Get manifest name.
	FString ManifestName;
	if( !GetStringFromConfig( *SectionName, TEXT("ManifestName"), ManifestName, ConfigPath ) )
	{
		UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("No manifest name specified.") );
		return false;
	}

	// Get archive name.
	FString ArchiveName;
	if( !( GetStringFromConfig(* SectionName, TEXT("ArchiveName"), ArchiveName, ConfigPath ) ) )
	{
		UE_LOG(LogInternationalizationExportCommandlet, Error, TEXT("No archive name specified."));
		return false;
	}

	// Get culture directory setting, default to true if not specified (used to allow picking of import directory with file open dialog from Translation Editor)
	bool bUseCultureDirectory = true;
	if (!(GetBoolFromConfig(*SectionName, TEXT("bUseCultureDirectory"), bUseCultureDirectory, ConfigPath)))
	{
		bUseCultureDirectory = true;
	}

	// Process the desired cultures
	for(int32 Culture = 0; Culture < CulturesToGenerate.Num(); Culture++)
	{
		// Load the Portable Object file if found
		const FString CultureName = CulturesToGenerate[Culture];
		FString POFilePath = "";
		if (bUseCultureDirectory)
		{
			POFilePath = SourcePath / CultureName / Filename;
		}
		else
		{
			POFilePath = SourcePath / Filename;
		}

		FPortableObjectFormatDOM PortableObject;
		const bool HasLoadedPOFile = LoadPOFile(POFilePath, PortableObject);
		if (!HasLoadedPOFile)
		{
			continue;
		}

		if (ShouldPersistComments)
		{
			PreserveExtractedCommentsForPersistence(PortableObject);
		}

		if (PortableObject.GetProjectName() != ManifestName.Replace(TEXT(".manifest"), TEXT("")))
		{
			UE_LOG(LogInternationalizationExportCommandlet, Log, TEXT("The project name (%s) in the file (%s) did not match the target manifest project (%s)."), *POFilePath, *PortableObject.GetProjectName(), *ManifestName.Replace(TEXT(".manifest"), TEXT("")));
		}

		const FString ManifestFileName = DestinationPath / ManifestName;

		TSharedPtr< FJsonObject > ManifestJsonObject = NULL;
		ManifestJsonObject = ReadJSONTextFile( ManifestFileName );

		FJsonInternationalizationManifestSerializer ManifestSerializer;
		TSharedRef< FInternationalizationManifest > InternationalizationManifest = MakeShareable( new FInternationalizationManifest );
		ManifestSerializer.DeserializeManifest( ManifestJsonObject.ToSharedRef(), InternationalizationManifest );

		if( !FPaths::FileExists(ManifestFileName) )
		{
			UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to find manifest %s."), *ManifestFileName);
			continue;
		}

		const FString DestinationCulturePath = DestinationPath / CultureName;
		const FString ArchiveFileName = DestinationCulturePath / ArchiveName;

		if( !FPaths::FileExists(ArchiveFileName) )
		{
			UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to find destination archive %s."), *ArchiveFileName);
			continue;
		}

		TSharedPtr< FJsonObject > ArchiveJsonObject = NULL;
		ArchiveJsonObject = ReadJSONTextFile( ArchiveFileName );

		FJsonInternationalizationArchiveSerializer ArchiveSerializer;
		TSharedRef< FInternationalizationArchive > InternationalizationArchive = MakeShareable( new FInternationalizationArchive );
		ArchiveSerializer.DeserializeArchive( ArchiveJsonObject.ToSharedRef(), InternationalizationArchive );

		bool bModifiedArchive = false;
		{
			for( auto EntryIter = PortableObject.GetEntriesIterator(); EntryIter; ++EntryIter )
			{
				auto POEntry = *EntryIter;
				if( POEntry->MsgId.IsEmpty() || POEntry->MsgStr.Num() == 0 || POEntry->MsgStr[0].Trim().IsEmpty() )
				{
					// We ignore the header entry or entries with no translation.
					continue;
				}

				// Some warning messages for data we don't process at the moment
				if( !POEntry->MsgIdPlural.IsEmpty() || POEntry->MsgStr.Num() > 1 )
				{
					UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Portable Object entry has plural form we did not process.  File: %s  MsgCtxt: %s  MsgId: %s"), *POFilePath, *POEntry->MsgCtxt, *POEntry->MsgId );
				}

				FString Key;
				FString Namespace;
				ParsePOMsgCtxtForIdentity(POEntry->MsgCtxt, Namespace, Key);
				const FString& SourceText = ConditionPoStringForArchive(POEntry->MsgId);
				const FString& Translation = ConditionPoStringForArchive(POEntry->MsgStr[0]);

				TSharedPtr<FLocMetadataObject> KeyMetaDataObject;
				// Get key metadata from the manifest, using the namespace and key.
				if (!Key.IsEmpty())
				{
					// Find manifest entry by namespace
					for (auto ManifestEntryIterator = InternationalizationManifest->GetEntriesByContextIdIterator(); ManifestEntryIterator; ++ManifestEntryIterator)
					{
						const FString& ManifestEntryNamespace = ManifestEntryIterator->Key;
						const TSharedRef<FManifestEntry>& ManifestEntry = ManifestEntryIterator->Value;
						if (ManifestEntry->Namespace == Namespace)
						{
							FContext* const MatchingContext = ManifestEntry->Contexts.FindByPredicate([&](FContext& Context) -> bool
								{
									return Context.Key == Key;
								});
							if (MatchingContext)
							{
								KeyMetaDataObject = MatchingContext->KeyMetadataObj;
							}
						}
					}
				}

				//@TODO: Take into account optional entries and entries that differ by keymetadata.  Ex. Each optional entry needs a unique msgCtxt
				const TSharedPtr< FArchiveEntry > FoundEntry = InternationalizationArchive->FindEntryBySource( Namespace, SourceText, KeyMetaDataObject );
				if( !FoundEntry.IsValid() )
				{
					UE_LOG(LogInternationalizationExportCommandlet, Warning, TEXT("Could not find corresponding archive entry for PO entry.  File: %s  MsgCtxt: %s  MsgId: %s"), *POFilePath, *POEntry->MsgCtxt, *POEntry->MsgId );
					continue;
				}

				if( FoundEntry->Translation != Translation )
				{
					FoundEntry->Translation = Translation;
					bModifiedArchive = true;
				}
			}
		}

		if( bModifiedArchive )
		{
			TSharedRef<FJsonObject> FinalArchiveJsonObj = MakeShareable( new FJsonObject );
			ArchiveSerializer.SerializeArchive( InternationalizationArchive, FinalArchiveJsonObj );

			if( !WriteJSONToTextFile(FinalArchiveJsonObj, ArchiveFileName, SourceControlInfo ) )
			{
				UE_LOG( LogInternationalizationExportCommandlet, Error, TEXT("Failed to write archive to %s."), *ArchiveFileName );				
				return false;
			}
		}
	}

	return true;
}