int extractBotNames(int argc, char *argv[])
{
//-------------------------------------------------------------------
// read the parameters
for (int i=2; i<argc; ++i)
{
string s = argv[i];
if (s == "-r")
{
// active remove mode
RemoveOlds = true;
}
else
{
nlwarning("Unknow option '%s'", argv[i]);
return -1;
}
}
//-------------------------------------------------------------------
// read the configuration file
CConfigFile cf;
cf.load("bin/translation_tools.cfg");
//-------------------------------------------------------------------
// read the vars
CConfigFile::CVar &paths = cf.getVar("Paths");
CConfigFile::CVar &filtersVar = cf.getVar("Filters");
CConfigFile::CVar &ligoClassFile= cf.getVar("LigoClassFile");
CConfigFile::CVar &georgesPaths= cf.getVar("GeorgesPaths");
CConfigFile::CVar &pathNoRecurse= cf.getVar("PathsNoRecurse");
CConfigFile::CVar &workBotNamesFile= cf.getVar("WorkBotNamesFile");
CConfigFile::CVar &transBotNamesFile= cf.getVar("TransBotNamesFile");
CConfigFile::CVar &workTitleFile= cf.getVar("WorkTitleFile");
for (uint i=0; i<paths.size(); ++i)
{
CPath::addSearchPath(paths.asString(i), true, false);
}
for (uint i=0; i<pathNoRecurse.size(); ++i)
{
CPath::addSearchPath(pathNoRecurse.asString(i), false, false);
}
for (uint i=0; i<filtersVar.size(); ++i)
{
Filters.push_back(filtersVar.asString(i));
}
//-------------------------------------------------------------------
// init the sheets
CSheetId::init(false);
const string PACKED_SHEETS_NAME = "bin/translation_tools_creature.packed_sheets";
loadForm("creature", PACKED_SHEETS_NAME, Creatures, false, false);
if (Creatures.empty())
{
for (uint i=0;i<georgesPaths.size();++i)
CPath::addSearchPath(georgesPaths.asString(i).c_str(), true, false);
loadForm("creature", PACKED_SHEETS_NAME, Creatures, true);
}
//-------------------------------------------------------------------
// init ligo config
string ligoPath = CPath::lookup(ligoClassFile.asString(), true, true);
LigoConfig.readPrimitiveClass(ligoPath.c_str(), false);
NLLIGO::Register();
CPrimitiveContext::instance().CurrentLigoConfig = &LigoConfig;
//-------------------------------------------------------------------
// ok, ready the the real work,
// first, read the primitives files and parse the primitives
vector<string> files;
CPath::getFileList("primitive", files);
for (uint i=0; i<files.size(); ++i)
{
string pathName = files[i];
pathName = CPath::lookup(pathName);
// check filters
uint j=0;
for (j=0; j<Filters.size(); ++j)
{
if (pathName.find(Filters[j]) != string::npos)
break;
}
if (j != Filters.size())
// skip this file
continue;
nlinfo("Loading file '%s'...", CFile::getFilename(pathName).c_str());
CPrimitives primDoc;
CPrimitiveContext::instance().CurrentPrimitive = &primDoc;
loadXmlPrimitiveFile(primDoc, pathName, LigoConfig);
// now parse the file
// look for group template
{
TPrimitiveClassPredicate pred("group_template_npc");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string countStr;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("count", countStr);
result[i]->getPropertyByName("bot_sheet_look", sheetStr);
uint32 count;
NLMISC::fromString(countStr, count);
if (count != 0)
{
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
// look for bot template
{
TPrimitiveClassPredicate pred("bot_template_npc");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("sheet_look", sheetStr);
if (sheetStr.empty())
{
// take the sheet in the parent
result[i]->getParent()->getPropertyByName("bot_sheet_look", sheetStr);
}
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
// look for npc_group
{
TPrimitiveClassPredicate pred("npc_group");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string countStr;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("count", countStr);
result[i]->getPropertyByName("bot_sheet_client", sheetStr);
uint32 count;
NLMISC::fromString(countStr, count);
if (count > 0 && sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
if (count == 1)
{
addSimpleName(removeAndStoreFunction(name), sheetStr);
}
else if (count > 1)
{
addGenericName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
// look for bot
{
TPrimitiveClassPredicate pred("npc_bot");
TPrimitiveSet result;
CPrimitiveSet<TPrimitiveClassPredicate> ps;
ps.buildSet(primDoc.RootNode, pred, result);
for (uint i=0; i<result.size(); ++i)
{
string name;
string sheetStr;
result[i]->getPropertyByName("name", name);
result[i]->getPropertyByName("sheet_client", sheetStr);
if (sheetStr.empty())
{
// take the sheet in the parent
result[i]->getParent()->getPropertyByName("bot_sheet_client", sheetStr);
}
if (sheetStr.empty())
{
nlwarning("In '%s', empty sheet !", buildPrimPath(result[i]).c_str());
}
else
{
TEntryInfo ei;
addSimpleName(removeAndStoreFunction(name), sheetStr);
}
}
}
}
//-------------------------------------------------------------------
// step 2 : load the reference file
nlinfo("Looking for missing translation:");
TWorksheet botNames;
loadExcelSheet(workBotNamesFile.asString(), botNames, true);
TWorksheet transBotNames;
loadExcelSheet(transBotNamesFile.asString(), transBotNames, true);
TWorksheet fcts;
loadExcelSheet(workTitleFile.asString(), fcts, true);
// add missing element
uint nbAddSimpleName = 0;
uint nbAddFunction = 0;
uint nbAddGenericName = 0;
uint botIdCol;
nlverify(botNames.findId(botIdCol));
uint transIdCol;
nlverify(transBotNames.findId(transIdCol));
uint fctsIdCol;
nlverify(fcts.findId(fctsIdCol));
// special treatment to add the sheet_name col
{
uint sheetCol;
if (!botNames.findCol(ucstring("sheet_name"), sheetCol))
{
botNames.insertColumn(botNames.ColCount);
botNames.setData(0, botNames.ColCount-1, ucstring("sheet_name"));
}
if (!transBotNames.findCol(ucstring("sheet_name"), sheetCol))
{
transBotNames.insertColumn(transBotNames.ColCount);
transBotNames.setData(0, transBotNames.ColCount-1, ucstring("sheet_name"));
}
}
// 1 - simple names
{
nlinfo(" Simple names...");
map<string, TEntryInfo>::iterator first(SimpleNames.begin()), last(SimpleNames.end());
for (; first != last; ++first)
{
uint rowIdx;
if (!botNames.findRow(botIdCol, first->first, rowIdx))
{
// we need to add the entry
rowIdx = botNames.size();
botNames.resize(botNames.size()+1);
botNames.setData(rowIdx, ucstring("bot name"), first->first);
botNames.setData(rowIdx, ucstring("translated name"), first->first);
botNames.setData(rowIdx, ucstring("sheet_name"), first->second.SheetName);
nbAddSimpleName++;
}
else
{
// set/update the sheet name info
// try to restore the existing translation
uint transRowIdx;
if (transBotNames.findRow(transIdCol, first->first, transRowIdx))
{
ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));
ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));
ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
ucstring ucWkHash;
uint64 hash = CI18N::makeHash(wkBotName + wkTranslationName +wkSheetName);
CI18N::hashToUCString(hash, ucWkHash);
ucstring trUcHash = transBotNames[transRowIdx][0];
bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
bool hashIsValide = std::equal(ucWkHash.begin(), ucWkHash.end(), trUcHash.begin()+1);
// Hash is equal get the translation
if (hashIsValide && !isWkTranslationNameAGroupName)
{
wkTranslationName = transBotNames.getData(transRowIdx, ucstring("translated name"));
wkSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
hash = CI18N::makeHash(wkBotName + wkTranslationName + wkSheetName);
// update the hash code
CI18N::hashToUCString(hash, transBotNames[transRowIdx][0]);
}
// bots_name.txt has been manually changed. We trust what the Level Designer has done. We don't destroy is work.
// or it is a simple
else
{
//use the "translated name" of the manually changed work/bot_name.txt
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
}
}
}
}
}
// 2 - generic names
{
nlinfo(" Generic names...");
set<string>::iterator first(GenericNames.begin()), last(GenericNames.end());
for (; first != last; ++first)
{
string gnName = "gn_" + cleanupName(*first);
ucstring fctsTitleId;
ucstring fctsName;
// add or modify the bot names
uint rowIdx;
if (!botNames.findRow(botIdCol, *first, rowIdx))
{
// we need to add the entry
rowIdx = botNames.size();
botNames.resize(botNames.size()+1);
botNames.setData(rowIdx, ucstring("bot name"), *first);
botNames.setData(rowIdx, ucstring("translated name"), ucstring("$") + gnName + "$");
botNames.setData(rowIdx, ucstring("sheet_name"), ucstring());
fctsTitleId = gnName;
fctsName = *first;
nbAddSimpleName++;
}
else
{
// look in the translated table to remember the translated name to write it in the string file
ucstring wkBotName = botNames.getData(rowIdx, ucstring("bot name"));
ucstring wkTranslationName = botNames.getData(rowIdx, ucstring("translated name"));
ucstring wkSheetName = botNames.getData(rowIdx, ucstring("sheet_name"));
nlinfo("Bot name:%s\n",wkBotName.toString().c_str());
bool isWkTranslationNameAGroupName = wkTranslationName.find(ucstring("$")) != ucstring::npos;
if ( isWkTranslationNameAGroupName ) //work name looks like "$gn_***$: do not modify
{
//Do not change work/bot_name.txt
// update work/world_title.txt
ucstring transName;
fctsTitleId = makeGroupName(wkTranslationName);
uint transRowIdx;
if (transBotNames.findRow(transIdCol, *first, transRowIdx))
{
transName = transBotNames.getData(transRowIdx, ucstring("translated name"));
if (transName.find(ucstring("$")) != ucstring::npos)
{
transName = fctsTitleId;
}
}
else
{
transName = fctsTitleId;
}
//Do not touch anything
botNames.setData(rowIdx, ucstring("translated name"), wkTranslationName);
botNames.setData(rowIdx, ucstring("sheet_name"), wkSheetName);
// fctsTitleId = makeGroupName(wkTranslationName);
fctsName = transName;
}
else // WkTranslationName != "$gn*$"
{
uint transRowIdx;
ucstring transName;
ucstring wkSheetName;
// Get the translation as a simple name.
if (transBotNames.findRow(transIdCol, *first, transRowIdx))
{
transName = transBotNames.getData(transRowIdx, ucstring("translated name"));
ucstring trSheetName = transBotNames.getData(transRowIdx, ucstring("sheet_name"));
//tr."translation name" is
if (transName.find(ucstring("$")) != ucstring::npos)
{
//get Translation, update hash
botNames[rowIdx][1] = transName;
botNames[rowIdx][2] = trSheetName;
fctsTitleId = makeGroupName(transName);
fctsName = makeGroupName(transName);
ucstring trNewUcHash;
uint64 hash = CI18N::makeHash(wkBotName + transName +trSheetName);
CI18N::hashToUCString(hash, trNewUcHash);
transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
}
else //botNames."translated name" != $gn_$ && tansName."translated name" != $gn_$
{
// get the translation back
//update work/bot_name.txt
wkTranslationName = ucstring("$")+gnName+"$";
botNames[rowIdx][0] = wkBotName;
botNames[rowIdx][1] = wkTranslationName;
botNames[rowIdx][2] = wkSheetName;
//update translated/bot_name.txt
fctsName = transName; //transName
fctsTitleId = gnName;
ucstring trNewUcHash;
uint64 hash = CI18N::makeHash(botNames[rowIdx][0] + botNames[rowIdx][1] +botNames[rowIdx][2]);
CI18N::hashToUCString(hash, trNewUcHash);
transBotNames[transRowIdx][0] = ucstring("_") + trNewUcHash;
}
}
else //There is no translation yet
{
fctsName = wkTranslationName;
wkTranslationName = ucstring("$")+gnName+"$";
botNames[rowIdx][0] = wkBotName;
botNames[rowIdx][1] = wkTranslationName;
botNames[rowIdx][2] = wkSheetName;
fctsTitleId = gnName;
}
}
}
// look for a corresponding entry
uint gnNameRow;
if (!fcts.findRow(fctsIdCol, fctsTitleId, gnNameRow))
{
// not found, add it
gnNameRow = fcts.size();
fcts.resize(fcts.size()+1);
fcts.setData(gnNameRow, ucstring("title_id"), fctsTitleId);
fcts.setData(gnNameRow, ucstring("name"), fctsName);
nbAddGenericName++;
}
else //Update
{
}
}
}
// 3 - functions
{
nlinfo(" Functions...");
set<string>::iterator first(Functions.begin()), last(Functions.end());
for (; first != last; ++first)
{
string fctName = *first;
// look for a corresponding entry
uint functionRow;
if (!fcts.findRow(fctsIdCol, fctName, functionRow))
{
// not found, add it
functionRow = fcts.size();
fcts.resize(fcts.size()+1);
fcts.setData(functionRow, ucstring("title_id"), fctName);
fcts.setData(functionRow, ucstring("name"), *first);
nbAddFunction++;
}
}
}
// display resumé
nlinfo("Adding %u new simple name", nbAddSimpleName);
nlinfo("Adding %u new generic name", nbAddGenericName);
nlinfo("Adding %u new function name", nbAddFunction);
// saving the modified files
ucstring s = prepareExcelSheet(botNames);
CI18N::writeTextFile(workBotNamesFile.asString(), s, false);
s = prepareExcelSheet(transBotNames);
CI18N::writeTextFile(transBotNamesFile.asString(), s, false);
s = prepareExcelSheet(fcts);
CI18N::writeTextFile(workTitleFile.asString(), s, false);
return 0;
}
// ***************************************************************************
void extractNewWords(string workSheetFileName, string columnId, IWordListBuilder &wordListBuilder)
{
uint i;
// **** Load the excel sheet
// load
TWorksheet workSheet;
if(!loadExcelSheet(workSheetFileName, workSheet, true))
{
nlwarning("Error reading '%s'. Aborted", workSheetFileName.c_str());
return;
}
// get the key column index
uint keyColIndex = 0;
if(!workSheet.findCol(columnId, keyColIndex))
{
nlwarning("Error: Don't find the column '%s'. '%s' Aborted", columnId.c_str(), workSheetFileName.c_str());
return;
}
// get the name column index
uint nameColIndex;
if(!workSheet.findCol(ucstring("name"), nameColIndex))
{
nlwarning("Error: Don't find the column 'name'. '%s' Aborted", workSheetFileName.c_str());
return;
}
// Make a copy of this worksheet, with strlwr on the key
// Yoyo: I prefer not modify the original worksheet (don't know what bad side effect it can have....)
TWorksheet workSheetLwr= workSheet;
for(i=0;i<workSheetLwr.size();i++)
{
ucstring key= workSheetLwr.getData(i, keyColIndex);
workSheetLwr.setData(i, keyColIndex, toLower(key));
}
// **** List all words with the builder given
std::vector<string> allWords;
if(!wordListBuilder.buildWordList(allWords, workSheetFileName))
return;
// **** Append new one to the worksheet
uint nbAdd= 0;
for(i=0;i<allWords.size();i++)
{
string keyName= allWords[i];
uint rowIdx;
// search in the key lowred worksheet (avoid case bugs (they do exist...))
if (!workSheetLwr.findRow(keyColIndex, keyName, rowIdx))
{
// we need to add the entry. Add it to the 2 workSheet to maintain coherence (avoid non unique etc...)
rowIdx = workSheetLwr.size();
// add to the workSheetLwr
workSheetLwr.resize(workSheetLwr.size()+1);
workSheetLwr.setData(rowIdx, keyColIndex, keyName);
workSheetLwr.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);
// add to the workSheet
workSheet.resize(workSheet.size()+1);
workSheet.setData(rowIdx, keyColIndex, keyName);
workSheet.setData(rowIdx, nameColIndex, string("<GEN>")+keyName);
nbAdd++;
}
}
// **** Remove no more present ones (and log)
uint nbRemove= 0;
if(RemoveOlds)
{
// Build as a set
std::set<string> allWordSet;
for(i=0;i<allWords.size();i++)
allWordSet.insert(allWords[i]);
// For all rows, append to a copy if not erased
TWorksheet tmpCopy, tmpCopyLwr;
nlassert(workSheet.ColCount==workSheetLwr.ColCount);
nlassert(workSheet.size()==workSheetLwr.size());
tmpCopy.setColCount(workSheet.ColCount);
tmpCopy.resize(workSheet.size());
tmpCopyLwr.setColCount(workSheet.ColCount);
tmpCopyLwr.resize(workSheet.size());
uint dstRowId=0;
for(i=0;i<workSheet.size();i++)
{
string keyStr= workSheetLwr.getData(i, keyColIndex).toString();
// if first line, or if the key (lwred) is found in the list of files
if(i==0 || allWordSet.find(keyStr)!=allWordSet.end())
{
tmpCopy.Data[dstRowId]= workSheet.Data[i];
tmpCopyLwr.Data[dstRowId]= workSheetLwr.Data[i];
dstRowId++;
}
else
{
nbRemove++;
// log
NLMISC::InfoLog->displayRawNL("'%s': '%s' entry erased at line '%d'.", workSheetFileName.c_str(),
keyStr.c_str(), i);
}
}
// resize to correct new size
tmpCopy.resize(dstRowId);
tmpCopyLwr.resize(dstRowId);
// copy back
workSheet= tmpCopy;
workSheetLwr= tmpCopyLwr;
}
// **** Save
if(nbAdd==0 && nbRemove==0)
{
if(RemoveOlds)
NLMISC::InfoLog->displayRawNL("'%s': No deprecated entry found.", workSheetFileName.c_str());
NLMISC::InfoLog->displayRawNL("'%s': No new entry found.", workSheetFileName.c_str());
// Don't save
}
else
{
if(RemoveOlds)
NLMISC::InfoLog->displayRawNL("'%s': %d deprecated entry erased.", workSheetFileName.c_str(), nbRemove);
NLMISC::InfoLog->displayRawNL("'%s': %d new entry found.", workSheetFileName.c_str(), nbAdd);
// Save the not lowered worksheet
ucstring s = prepareExcelSheet(workSheet);
try
{
CI18N::writeTextFile(workSheetFileName.c_str(), s, false);
}
catch (const Exception &e)
{
nlwarning("cannot save file: '%s'. Reason: %s", workSheetFileName.c_str(), e.what());
}
}
}