const ship_type *findshiptype(const char *name, const struct locale *lang) { local_names *sn = snames; variant var; while (sn && sn->lang != lang) { sn = sn->next; } if (!sn) { selist *ql; int qi; sn = (local_names *)calloc(1, sizeof(local_names)); if (!sn) abort(); sn->next = snames; sn->lang = lang; for (qi = 0, ql = shiptypes; ql; selist_advance(&ql, &qi, 1)) { ship_type *stype = (ship_type *)selist_get(ql, qi); variant var2; const char *n = LOC(lang, stype->_name); var2.v = (void *)stype; addtoken((struct tnode **)&sn->names, n, var2); } snames = sn; } if (findtoken(sn->names, name, &var) == E_TOK_NOMATCH) return NULL; return (const ship_type *)var.v; }
/* Find the building type for a given localized name (as seen by the user). Useful for parsing * orders. The inverse of locale_string(lang, btype->_name), sort of. */ const building_type *findbuildingtype(const char *name, const struct locale *lang) { variant type; local_names *bn = bnames; while (bn) { if (bn->lang == lang) break; bn = bn->next; } if (!bn) { quicklist *ql = buildingtypes; int qi; bn = (local_names *)calloc(sizeof(local_names), 1); bn->next = bnames; bn->lang = lang; for (qi = 0, ql = buildingtypes; ql; ql_advance(&ql, &qi, 1)) { building_type *btype = (building_type *)ql_get(ql, qi); const char *n = LOC(lang, btype->_name); type.v = (void *)btype; addtoken(&bn->names, n, type); } bnames = bn; } if (findtoken(bn->names, name, &type) == E_TOK_NOMATCH) return NULL; return (const building_type *)type.v; }
const ship_type *findshiptype(const char *name, const struct locale *lang) { local_names *sn = snames; variant var; while (sn) { if (sn->lang == lang) break; sn = sn->next; } if (!sn) { quicklist *ql; int qi; sn = (local_names *)calloc(sizeof(local_names), 1); sn->next = snames; sn->lang = lang; for (qi = 0, ql = shiptypes; ql; ql_advance(&ql, &qi, 1)) { ship_type *stype = (ship_type *)ql_get(ql, qi); variant var2; const char *n = LOC(lang, stype->_name); var2.v = (void *)stype; addtoken(&sn->names, n, var2); } snames = sn; } if (findtoken(sn->names, name, &var) == E_TOK_NOMATCH) return NULL; return (const ship_type *)var.v; }
void TokenList::addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split) { if (str[0] == 0) return; // If token contains # characters, split it up if (split && strstr(str, "##")) { std::string temp; for (unsigned int i = 0; str[i]; ++i) { if (strncmp(&str[i], "##", 2) == 0) { addtoken(temp.c_str(), lineno, fileno, false); temp.clear(); addtoken("##", lineno, fileno, false); ++i; } else temp += str[i]; } addtoken(temp.c_str(), lineno, fileno, false); return; } // Replace hexadecimal value with decimal std::ostringstream str2; if (MathLib::isHex(str) || MathLib::isOct(str) || MathLib::isBin(str)) { str2 << MathLib::toLongNumber(str); } else if (strncmp(str, "_Bool", 5) == 0) { str2 << "bool"; } else { str2 << str; } if (_back) { _back->insertToken(str2.str()); } else { _front = new Token(&_back); _back = _front; _back->str(str2.str()); } _back->linenr(lineno); _back->fileIndex(fileno); }
void register_special_direction(struct locale *lang, const char *name) { const char *token = locale_string(lang, name, false); if (token) { void **tokens = get_translations(lang, UT_SPECDIR); variant var; char *str = strdup(name); var.v = str; addtoken((struct tnode **)tokens, token, var); if (lang == locales) { dir_lookup *dl = malloc(sizeof(dir_lookup)); dl->name = str; dl->oldname = token; dl->next = dir_name_lookup; dir_name_lookup = dl; } } else { log_debug("no translation for spec_direction '%s' in locale '%s'\n", name, locale_name(lang)); } }
bool TokenList::createTokens(std::istream &code, const std::string& file0) { _files.push_back(file0); // line number in parsed code unsigned int lineno = 1; // The current token being parsed std::string CurrentToken; // lineNumbers holds line numbers for files in fileIndexes // every time an include file is completely parsed, last item in the vector // is removed and lineno is set to point to that value. std::stack<unsigned int> lineNumbers; // fileIndexes holds index for _files vector about currently parsed files // every time an include file is completely parsed, last item in the vector // is removed and FileIndex is set to point to that value. std::stack<unsigned int> fileIndexes; // FileIndex. What file in the _files vector is read now? unsigned int FileIndex = 0; bool expandedMacro = false; // Read one byte at a time from code and create tokens for (char ch = (char)code.get(); code.good(); ch = (char)code.get()) { if (ch == Preprocessor::macroChar) { while (code.peek() == Preprocessor::macroChar) code.get(); ch = ' '; expandedMacro = true; } else if (ch == '\n') { expandedMacro = false; } // char/string.. // multiline strings are not handled. The preprocessor should handle that for us. else if (ch == '\'' || ch == '\"') { std::string line; // read char bool special = false; char c = ch; do { // Append token.. line += c; // Special sequence '\.' if (special) special = false; else special = (c == '\\'); // Get next character c = (char)code.get(); } while (code.good() && (special || c != ch)); line += ch; // Handle #file "file.h" if (CurrentToken == "#file") { // Extract the filename line = line.substr(1, line.length() - 2); // Has this file been tokenized already? ++lineno; bool foundOurfile = false; fileIndexes.push(FileIndex); for (unsigned int i = 0; i < _files.size(); ++i) { if (Path::sameFileName(_files[i], line)) { // Use this index foundOurfile = true; FileIndex = i; } } if (!foundOurfile) { // The "_files" vector remembers what files have been tokenized.. _files.push_back(Path::simplifyPath(line.c_str())); FileIndex = static_cast<unsigned int>(_files.size() - 1); } lineNumbers.push(lineno); lineno = 0; } else { // Add previous token addtoken(CurrentToken.c_str(), lineno, FileIndex); if (!CurrentToken.empty()) _back->setExpandedMacro(expandedMacro); // Add content of the string addtoken(line.c_str(), lineno, FileIndex); if (!line.empty()) _back->setExpandedMacro(expandedMacro); } CurrentToken.clear(); continue; } if (ch == '.' && CurrentToken.length() > 0 && std::isdigit(CurrentToken[0])) { // Don't separate doubles "5.4" } else if (strchr("+-", ch) && CurrentToken.length() > 0 && std::isdigit(CurrentToken[0]) && (CurrentToken[CurrentToken.length()-1] == 'e' || CurrentToken[CurrentToken.length()-1] == 'E') && !MathLib::isHex(CurrentToken)) { // Don't separate doubles "4.2e+10" } else if (CurrentToken.empty() && ch == '.' && std::isdigit(code.peek())) { // tokenize .125 into 0.125 CurrentToken = "0"; } else if (strchr("+-*/%&|^?!=<>[](){};:,.~\n ", ch)) { if (CurrentToken == "#file") { // Handle this where strings are handled continue; } else if (CurrentToken == "#endfile") { if (lineNumbers.empty() || fileIndexes.empty()) { // error deallocateTokens(); return false; } lineno = lineNumbers.top(); lineNumbers.pop(); FileIndex = fileIndexes.top(); fileIndexes.pop(); CurrentToken.clear(); continue; } addtoken(CurrentToken.c_str(), lineno, FileIndex, true); if (!CurrentToken.empty()) _back->setExpandedMacro(expandedMacro); CurrentToken.clear(); if (ch == '\n') { ++lineno; continue; } else if (ch == ' ') { continue; } CurrentToken += ch; // Add "++", "--", ">>" or ... token if (strchr("+-<>=:&|", ch) && (code.peek() == ch)) CurrentToken += (char)code.get(); addtoken(CurrentToken.c_str(), lineno, FileIndex); _back->setExpandedMacro(expandedMacro); CurrentToken.clear(); continue; } CurrentToken += ch; } addtoken(CurrentToken.c_str(), lineno, FileIndex, true); if (!CurrentToken.empty()) _back->setExpandedMacro(expandedMacro); _front->assignProgressValues(); for (unsigned int i = 1; i < _files.size(); i++) _files[i] = Path::getRelativePath(_files[i], _settings->_basePaths); return true; }
void addtoken(tnode ** root, const char *str, variant id) { tnode * tk; static const struct replace { ucs4_t ucs; const char str[3]; } replace[] = { /* match lower-case (!) umlauts and others to transcriptions */ { 228, "AE" }, { 246, "OE" }, { 252, "UE" }, { 223, "SS" }, { 230, "AE" }, { 248, "OE" }, { 229, "AA" }, { 0, "" } }; assert(root && str); if (!*root) { tk = *root = mknode(); } else { tk = *root; } assert(tk && tk == *root); if (!*str) { tk->id = id; tk->flags |= LEAF; } else { tref *next; int ret, index, i = 0; ucs4_t ucs, lcs; size_t len; ret = unicode_utf8_to_ucs4(&ucs, str, &len); assert(ret == 0 || !"invalid utf8 string"); lcs = ucs; #if NODEHASHSIZE == 8 index = ucs & 7; #else index = ucs % NODEHASHSIZE; #endif assert(index >= 0); next = tk->next[index]; if (!(tk->flags & LEAF)) tk->id = id; while (next && next->ucs != ucs) next = next->nexthash; if (!next) { tref *ref; tnode *node = mknode(); // TODO: what is the reason for this empty node to exist? if (ucs < 'a' || ucs > 'z') { lcs = towlower((wint_t)ucs); } if (ucs == lcs) { ucs = towupper((wint_t)ucs); } ref = (tref *)malloc(sizeof(tref)); ref->ucs = ucs; ref->node = node; ref->nexthash = tk->next[index]; tk->next[index] = ref; /* try lower/upper casing the character, and try again */ if (ucs != lcs) { #if NODEHASHSIZE == 8 index = lcs & 7; #else index = lcs % NODEHASHSIZE; #endif ref = (tref *)malloc(sizeof(tref)); assert_alloc(ref); ref->ucs = lcs; ref->node = node; ++node->refcount; ref->nexthash = tk->next[index]; tk->next[index] = ref; } next = ref; } else { tnode * next_node = (tnode *)next->node; next_node->flags |= SHARED; if ((next_node->flags & LEAF) == 0) next_node->id.v = NULL; /* why? */ } addtoken(&next->node, str + len, id); while (replace[i].str[0]) { if (lcs == replace[i].ucs) { char zText[1024]; memcpy(zText, replace[i].str, 3); strcpy(zText + 2, (const char *)str + len); addtoken(root, zText, id); break; } ++i; } } }