Exemplo n.º 1
0
int extractBotNames(int argc, char *argv[])
{
	//-------------------------------------------------------------------
	// read the parameters
	for (int i=2; i<argc; ++i)
	{
		string s = argv[i];
		if (s == "-r")
		{
			// active remove mode
			RemoveOlds = true;
		}
		else
		{
			nlwarning("Unknow option '%s'", argv[i]);
			return -1;
		}
	}

	//-------------------------------------------------------------------
	// read the configuration file
	CConfigFile	cf;

	cf.load("bin/translation_tools.cfg");

	//-------------------------------------------------------------------
	// read the vars
	CConfigFile::CVar &paths = cf.getVar("Paths");
	CConfigFile::CVar &filtersVar = cf.getVar("Filters");
	CConfigFile::CVar &ligoClassFile= cf.getVar("LigoClassFile");
	CConfigFile::CVar &georgesPaths= cf.getVar("GeorgesPaths");
	CConfigFile::CVar &pathNoRecurse= cf.getVar("PathsNoRecurse");
	CConfigFile::CVar &workBotNamesFile= cf.getVar("WorkBotNamesFile");
	CConfigFile::CVar &transBotNamesFile= cf.getVar("TransBotNamesFile");
	CConfigFile::CVar &workTitleFile= cf.getVar("WorkTitleFile");

	for (uint i=0; i<paths.size(); ++i)
	{
		CPath::addSearchPath(paths.asString(i), true, false);
	}
	for (uint i=0; i<pathNoRecurse.size(); ++i)
	{
		CPath::addSearchPath(pathNoRecurse.asString(i), false, false);
	}

	for (uint i=0; i<filtersVar.size(); ++i)
	{
		Filters.push_back(filtersVar.asString(i));
	}


	//-------------------------------------------------------------------
	// init the sheets
	CSheetId::init(false);
	const string PACKED_SHEETS_NAME = "bin/translation_tools_creature.packed_sheets";
	loadForm("creature", PACKED_SHEETS_NAME, Creatures, false, false);

	if (Creatures.empty())
	{
		for (uint i=0;i<georgesPaths.size();++i)
			CPath::addSearchPath(georgesPaths.asString(i).c_str(), true, false);

		loadForm("creature", PACKED_SHEETS_NAME, Creatures, true);
	}


	//-------------------------------------------------------------------
	// init ligo config
	string ligoPath = CPath::lookup(ligoClassFile.asString(), true, true);
	LigoConfig.readPrimitiveClass(ligoPath.c_str(), false);
	NLLIGO::Register();

	CPrimitiveContext::instance().CurrentLigoConfig = &LigoConfig;

	//-------------------------------------------------------------------
	// ok, ready the the real work,
	// first, read the primitives files and parse the primitives
	vector<string>	files;
	CPath::getFileList("primitive", files);

	for (uint i=0; i<files.size(); ++i)
	{
		string pathName = files[i];
		pathName = CPath::lookup(pathName);

		// check filters
		uint j=0;
		for (j=0; j<Filters.size(); ++j)
		{
			if (pathName.find(Filters[j]) != string::npos)
				break;
		}
		if (j != Filters.size())
			// skip this file
			continue;

		nlinfo("Loading file '%s'...", CFile::getFilename(pathName).c_str());
		
		CPrimitives primDoc;
		CPrimitiveContext::instance().CurrentPrimitive = &primDoc;
		loadXmlPrimitiveFile(primDoc, pathName, LigoConfig);

		// now parse the file

		// look for group template
		{
			TPrimitiveClassPredicate pred("group_template_npc");
			TPrimitiveSet result;

			CPrimitiveSet<TPrimitiveClassPredicate> ps;
			ps.buildSet(primDoc.RootNode, pred, result);

			for (uint i=0; i<result.size(); ++i)
			{
				string name;
				string countStr;
				string sheetStr;
				result[i]->getPropertyByName("name", name);
				result[i]->getPropertyByName("count", countStr);
				result[i]->getPropertyByName("bot_sheet_look", sheetStr);

				uint32 count;
				NLMISC::fromString(countStr, count);

				if (count != 0)
				{
					if (sheetStr.empty())
					{
						nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
					}
					else
					{
						addGenericName(removeAndStoreFunction(name), sheetStr);
					}
				}
			}
		}
		// look for bot template
		{
			TPrimitiveClassPredicate pred("bot_template_npc");
			TPrimitiveSet result;

			CPrimitiveSet<TPrimitiveClassPredicate> ps;
			ps.buildSet(primDoc.RootNode, pred, result);

			for (uint i=0; i<result.size(); ++i)
			{
				string name;
				string sheetStr;
				result[i]->getPropertyByName("name", name);
				result[i]->getPropertyByName("sheet_look", sheetStr);

				if (sheetStr.empty())
				{
					// take the sheet in the parent
					result[i]->getParent()->getPropertyByName("bot_sheet_look", sheetStr);
				}

				if (sheetStr.empty())
				{
					nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
				}
				else
				{
					addGenericName(removeAndStoreFunction(name), sheetStr);
				}
			}
		}
		// look for npc_group 
		{
			TPrimitiveClassPredicate pred("npc_group");
			TPrimitiveSet result;

			CPrimitiveSet<TPrimitiveClassPredicate> ps;
			ps.buildSet(primDoc.RootNode, pred, result);

			for (uint i=0; i<result.size(); ++i)
			{
				string name;
				string countStr;
				string sheetStr;
				result[i]->getPropertyByName("name", name);
				result[i]->getPropertyByName("count", countStr);
				result[i]->getPropertyByName("bot_sheet_client", sheetStr);

				uint32 count;
				NLMISC::fromString(countStr, count);

				if (count > 0 && sheetStr.empty())
				{
					nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
				}
				else
				{
					if (count == 1)
					{
						addSimpleName(removeAndStoreFunction(name), sheetStr);
					}
					else if (count > 1)
					{
						addGenericName(removeAndStoreFunction(name), sheetStr);
					}
				}
			}
		}
		// look for bot 
		{
			TPrimitiveClassPredicate pred("npc_bot");
			TPrimitiveSet result;

			CPrimitiveSet<TPrimitiveClassPredicate> ps;
			ps.buildSet(primDoc.RootNode, pred, result);

			for (uint i=0; i<result.size(); ++i)
			{
				string name;
				string sheetStr;
				result[i]->getPropertyByName("name", name);
				result[i]->getPropertyByName("sheet_client", sheetStr);

				if (sheetStr.empty())
				{
					// take the sheet in the parent
					result[i]->getParent()->getPropertyByName("bot_sheet_client", sheetStr);
				}

				if (sheetStr.empty())
				{
					nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
				}
				else
				{
					TEntryInfo ei;
					addSimpleName(removeAndStoreFunction(name), sheetStr);
				}
			}
		}
	}

	//-------------------------------------------------------------------
	// step 2 : load the reference file

	nlinfo("Looking for missing translation:");

	TWorksheet			botNames;
	loadExcelSheet(workBotNamesFile.asString(), botNames, true);
	TWorksheet			transBotNames;
	loadExcelSheet(transBotNamesFile.asString(), transBotNames, true);

	TWorksheet			fcts;
	loadExcelSheet(workTitleFile.asString(), fcts, true);


	// add missing element

	uint	nbAddSimpleName = 0;
	uint	nbAddFunction = 0;
	uint	nbAddGenericName = 0;

	uint botIdCol;
	nlverify(botNames.findId(botIdCol));
	uint transIdCol;
	nlverify(transBotNames.findId(transIdCol));
	uint	fctsIdCol;
	nlverify(fcts.findId(fctsIdCol));

	// special treatment to add the sheet_name col
	{
		uint sheetCol;
		if (!botNames.findCol(ucstring("sheet_name"), sheetCol))
		{
			botNames.insertColumn(botNames.ColCount);
			botNames.setData(0, botNames.ColCount-1, ucstring("sheet_name"));
		}
		
		if (!transBotNames.findCol(ucstring("sheet_name"), sheetCol))
		{
			transBotNames.insertColumn(transBotNames.ColCount);
			transBotNames.setData(0, transBotNames.ColCount-1, ucstring("sheet_name"));
		}
	}
	// 1 - simple names
	{
		nlinfo("  Simple names...");


		map<string, TEntryInfo>::iterator first(SimpleNames.begin()), last(SimpleNames.end());
		for (; first != last; ++first)
		{
			uint rowIdx;
			if (!botNames.findRow(botIdCol, first->first, rowIdx))
			{
				// we need to add the entry
				rowIdx = botNames.size();
				botNames.resize(botNames.size()+1);

				botNames.setData(rowIdx, ucstring("bot name"), first->first);
				botNames.setData(rowIdx, ucstring("translated name"), first->first);
				botNames.setData(rowIdx, ucstring("sheet_name"), first->second.SheetName);

				nbAddSimpleName++;
			}
			else
			{
				// set/update the sheet name info
				// try to restore the existing translation
				uint transRowIdx;
				if (transBotNames.findRow(transIdCol, first->first, transRowIdx))
				{
					ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));
					ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));								
					ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
					ucstring ucWkHash;
					uint64 hash = CI18N::makeHash(wkBotName + wkTranslationName +wkSheetName);						
					CI18N::hashToUCString(hash, ucWkHash);
					ucstring trUcHash = transBotNames[transRowIdx][0];
					bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
					bool hashIsValide = std::equal(ucWkHash.begin(), ucWkHash.end(), trUcHash.begin()+1);
					// Hash is equal get the translation
					if (hashIsValide && !isWkTranslationNameAGroupName)
					{
						wkTranslationName = transBotNames.getData(transRowIdx, ucstring("translated name"));
						wkSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));
						botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
						botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
						hash = CI18N::makeHash(wkBotName + wkTranslationName + wkSheetName);						
  						// update the hash code
						CI18N::hashToUCString(hash, transBotNames[transRowIdx][0]);							
					}
					// bots_name.txt has been manually changed. We trust what the Level Designer has done. We don't destroy is work.
					// or it is a simple 
					else
					{
						//use the "translated name" of the manually changed  work/bot_name.txt
						botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
						botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);	
					}					
				}
			}
		}
	}

	// 2 - generic names
	
	{
		nlinfo("  Generic names...");

		set<string>::iterator first(GenericNames.begin()), last(GenericNames.end());
		for (; first != last; ++first)
		{
			string gnName = "gn_" + cleanupName(*first);

			ucstring fctsTitleId;
			ucstring fctsName;
			// add or modify the bot names
			uint rowIdx;
			if (!botNames.findRow(botIdCol, *first, rowIdx)) 
			{
				// we need to add the entry
				rowIdx = botNames.size();
				botNames.resize(botNames.size()+1);

				botNames.setData(rowIdx, ucstring("bot name"), *first);
				botNames.setData(rowIdx, ucstring("translated name"), ucstring("$") + gnName + "$");
				botNames.setData(rowIdx, ucstring("sheet_name"), ucstring());
				fctsTitleId = gnName;
				fctsName = *first;

				nbAddSimpleName++;
			}
			else
			{
				// look in the translated table to remember the translated name to write it in the string file
				ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));				
				ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
				ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));

				
				nlinfo("Bot name:%s\n",wkBotName.toString().c_str());
				bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
				
				if ( isWkTranslationNameAGroupName ) //work name looks like "$gn_***$: do not modify
				{

					//Do not change work/bot_name.txt
					// update work/world_title.txt

					ucstring transName;
					fctsTitleId = makeGroupName(wkTranslationName);
					uint transRowIdx;
					if (transBotNames.findRow(transIdCol, *first, transRowIdx))
					{
						transName = transBotNames.getData(transRowIdx, ucstring("translated name"));

						if (transName.find(ucstring("$")) != ucstring::npos)
						{
							transName = fctsTitleId;
						} 
						
					}
					else
					{
						transName = fctsTitleId;
					}
					//Do not touch anything
					botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
					botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName); 
					// fctsTitleId = makeGroupName(wkTranslationName);
					fctsName = transName;

				}
				else // WkTranslationName != "$gn*$"
				{
						uint transRowIdx;
						ucstring transName;
						ucstring wkSheetName;
						// Get the translation as a simple name.
						if (transBotNames.findRow(transIdCol, *first, transRowIdx))
						{
		
							transName = transBotNames.getData(transRowIdx, ucstring("translated name"));
							ucstring trSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));

							//tr."translation name" is 
							if (transName.find(ucstring("$")) != ucstring::npos)
							{
								//get Translation, update hash
								botNames[rowIdx][1] = transName;
								botNames[rowIdx][2] = trSheetName;		
								fctsTitleId = makeGroupName(transName);
								fctsName = makeGroupName(transName);
								ucstring trNewUcHash;
								uint64 hash = CI18N::makeHash(wkBotName + transName +trSheetName);						
								CI18N::hashToUCString(hash, trNewUcHash);
								transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
							}
							else //botNames."translated name" != $gn_$ && tansName."translated name" != $gn_$
							{

								// get the translation back
								//update work/bot_name.txt
								wkTranslationName = ucstring("$")+gnName+"$";
								botNames[rowIdx][0] = wkBotName;
								botNames[rowIdx][1] = wkTranslationName;
								botNames[rowIdx][2] = wkSheetName;		
									
									//update translated/bot_name.txt

								fctsName = transName;	//transName	
								fctsTitleId = gnName;
								ucstring trNewUcHash;
								uint64 hash = CI18N::makeHash(botNames[rowIdx][0] + botNames[rowIdx][1] +botNames[rowIdx][2]);						
								CI18N::hashToUCString(hash, trNewUcHash);
								transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
							}

						}
						else //There is no translation yet
						{
								fctsName = wkTranslationName;
								wkTranslationName = ucstring("$")+gnName+"$";
								botNames[rowIdx][0] = wkBotName;
								botNames[rowIdx][1] = wkTranslationName;
								botNames[rowIdx][2] = wkSheetName;		
								fctsTitleId = gnName;

				
						}				
				}

			}


			// look for a corresponding entry
			uint gnNameRow;


			if (!fcts.findRow(fctsIdCol, fctsTitleId, gnNameRow))
			{
				
				// not found, add it
				gnNameRow = fcts.size();
				fcts.resize(fcts.size()+1);
				fcts.setData(gnNameRow, ucstring("title_id"), fctsTitleId);
				fcts.setData(gnNameRow, ucstring("name"), fctsName);										
				nbAddGenericName++;
				
			}
			else //Update 
			{
			
			}
		}
	}


	// 3 - functions
	{
		nlinfo("  Functions...");

		set<string>::iterator first(Functions.begin()), last(Functions.end());
		for (; first != last; ++first)
		{
			string fctName = *first;
			// look for a corresponding entry
			uint functionRow;
			if (!fcts.findRow(fctsIdCol, fctName, functionRow))
			{
				// not found, add it
				functionRow = fcts.size();
				fcts.resize(fcts.size()+1);

				fcts.setData(functionRow, ucstring("title_id"), fctName);
				fcts.setData(functionRow, ucstring("name"), *first);

				nbAddFunction++;
			}
		}
	}

	// display resumé
	nlinfo("Adding %u new simple name", nbAddSimpleName);
	nlinfo("Adding %u new generic name", nbAddGenericName);
	nlinfo("Adding %u new function name", nbAddFunction);

	// saving the modified files

	ucstring s = prepareExcelSheet(botNames);
	CI18N::writeTextFile(workBotNamesFile.asString(), s, false);
	s = prepareExcelSheet(transBotNames);
	CI18N::writeTextFile(transBotNamesFile.asString(), s, false);
	s = prepareExcelSheet(fcts);
	CI18N::writeTextFile(workTitleFile.asString(), s, false);

	return 0;
}
Exemplo n.º 2
0
// ***************************************************************************
void	extractNewWords(string workSheetFileName, string columnId, IWordListBuilder &wordListBuilder)
{
	uint	i;

	// **** Load the excel sheet
	// load
	TWorksheet		workSheet;
	if(!loadExcelSheet(workSheetFileName, workSheet, true))
	{
		nlwarning("Error reading '%s'. Aborted", workSheetFileName.c_str());
		return;
	}
	// get the key column index
	uint	keyColIndex = 0;
	
	if(!workSheet.findCol(columnId, keyColIndex))
	{
		nlwarning("Error: Don't find the column '%s'. '%s' Aborted", columnId.c_str(), workSheetFileName.c_str());
		return;
	}
	// get the name column index
	uint	nameColIndex;
	if(!workSheet.findCol(ucstring("name"), nameColIndex))
	{
		nlwarning("Error: Don't find the column 'name'. '%s' Aborted", workSheetFileName.c_str());
		return;
	}
	// Make a copy of this worksheet, with strlwr on the key
	// Yoyo: I prefer not modify the original worksheet (don't know what bad side effect it can have....)
	TWorksheet		workSheetLwr= workSheet;
	for(i=0;i<workSheetLwr.size();i++)
	{
		ucstring	key= workSheetLwr.getData(i, keyColIndex);
		workSheetLwr.setData(i, keyColIndex, toLower(key));
	}
	

	// **** List all words with the builder given
	std::vector<string>		allWords;
	if(!wordListBuilder.buildWordList(allWords, workSheetFileName))
		return;


	// **** Append new one to the worksheet
	uint	nbAdd= 0;
	for(i=0;i<allWords.size();i++)
	{
		string	keyName= allWords[i];
		uint rowIdx;
		// search in the key lowred worksheet (avoid case bugs (they do exist...))
		if (!workSheetLwr.findRow(keyColIndex, keyName, rowIdx))
		{
			// we need to add the entry. Add it to the 2 workSheet to maintain coherence (avoid non unique etc...)
			rowIdx = workSheetLwr.size();
			// add to the workSheetLwr
			workSheetLwr.resize(workSheetLwr.size()+1);
			workSheetLwr.setData(rowIdx, keyColIndex, keyName);
			workSheetLwr.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);
			// add to the workSheet
			workSheet.resize(workSheet.size()+1);
			workSheet.setData(rowIdx, keyColIndex, keyName);
			workSheet.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);
			
			nbAdd++;
		}
	}


	// **** Remove no more present ones (and log)
	uint	nbRemove= 0;
	if(RemoveOlds)
	{
		// Build as a set
		std::set<string>	allWordSet;
		for(i=0;i<allWords.size();i++)
			allWordSet.insert(allWords[i]);
		// For all rows, append to a copy if not erased
		TWorksheet	tmpCopy, tmpCopyLwr;
		nlassert(workSheet.ColCount==workSheetLwr.ColCount);
		nlassert(workSheet.size()==workSheetLwr.size());
		tmpCopy.setColCount(workSheet.ColCount);
		tmpCopy.resize(workSheet.size());
		tmpCopyLwr.setColCount(workSheet.ColCount);
		tmpCopyLwr.resize(workSheet.size());
		uint	dstRowId=0;
		for(i=0;i<workSheet.size();i++)
		{
			string	keyStr= workSheetLwr.getData(i, keyColIndex).toString();
			// if first line, or if the key (lwred) is found in the list of files
			if(i==0 || allWordSet.find(keyStr)!=allWordSet.end())
			{
				tmpCopy.Data[dstRowId]= workSheet.Data[i];
				tmpCopyLwr.Data[dstRowId]= workSheetLwr.Data[i];
				dstRowId++;
			}
			else
			{
				nbRemove++;
				// log
				NLMISC::InfoLog->displayRawNL("'%s': '%s' entry erased at line '%d'.", workSheetFileName.c_str(), 
					keyStr.c_str(), i);
			}
		}
		// resize to correct new size
		tmpCopy.resize(dstRowId);
		tmpCopyLwr.resize(dstRowId);

		// copy back
		workSheet= tmpCopy;
		workSheetLwr= tmpCopyLwr;
	}


	// **** Save
	if(nbAdd==0 && nbRemove==0)
	{
		if(RemoveOlds)
			NLMISC::InfoLog->displayRawNL("'%s': No deprecated entry found.", workSheetFileName.c_str());
		NLMISC::InfoLog->displayRawNL("'%s': No new entry found.", workSheetFileName.c_str());
		// Don't save
	}
	else
	{
		if(RemoveOlds)
			NLMISC::InfoLog->displayRawNL("'%s': %d deprecated entry erased.", workSheetFileName.c_str(), nbRemove);
		NLMISC::InfoLog->displayRawNL("'%s': %d new entry found.", workSheetFileName.c_str(), nbAdd);
		// Save the not lowered worksheet
		ucstring s = prepareExcelSheet(workSheet);
		try
		{
			CI18N::writeTextFile(workSheetFileName.c_str(), s, false);
		}
		catch (const Exception &e)
		{
			nlwarning("cannot save file: '%s'. Reason: %s", workSheetFileName.c_str(), e.what());
		}
	}
}