// scanToken() will always get a new token from the frontier // regardless of whether there are tokens in the lookahead store int Scanner::scanToken(ScannerToken &t, Location &l) { m_token = &t; m_loc = &l; int tokid; for (;;) { tokid = scan(); switch (tokid) { case T_DOC_COMMENT: setDocComment(m_token->text()); /* fall through */ case T_COMMENT: case T_OPEN_TAG: case T_WHITESPACE: if (m_type & ReturnAllTokens) { // m_lastToken holds the last "signficant" token, so // don't update it for comments or whitespace return tokid; } break; default: m_lastToken = tokid; return tokid; } } }
// ----------------------------------------------------------------------------- // Reads in a text definition of a language. See slade.pk3 for // formatting examples // ----------------------------------------------------------------------------- bool TextLanguage::readLanguageDefinition(MemChunk& mc, string_view source) { Tokenizer tz; // Open the given text data if (!tz.openMem(mc, source)) { Log::warning("Unable to open language definition {}", source); return false; } // Parse the definition text ParseTreeNode root; if (!root.parse(tz)) return false; // Get parsed data for (unsigned a = 0; a < root.nChildren(); a++) { auto node = root.childPTN(a); // Create language auto lang = new TextLanguage(node->name()); // Check for inheritance if (!node->inherit().empty()) { auto inherit = fromId(node->inherit()); if (inherit) inherit->copyTo(lang); else Log::warning("Warning: Language {} inherits from undefined language {}", node->name(), node->inherit()); } // Parse language info for (unsigned c = 0; c < node->nChildren(); c++) { auto child = node->childPTN(c); auto pn_lower = StrUtil::lower(child->name()); // Language name if (pn_lower == "name") lang->setName(child->stringValue()); // Comment begin else if (pn_lower == "comment_begin") { lang->setCommentBeginList(child->stringValues()); } // Comment end else if (pn_lower == "comment_end") { lang->setCommentEndList(child->stringValues()); } // Line comment else if (pn_lower == "comment_line") { lang->setLineCommentList(child->stringValues()); } // Preprocessor else if (pn_lower == "preprocessor") lang->setPreprocessor(child->stringValue()); // Case sensitive else if (pn_lower == "case_sensitive") lang->setCaseSensitive(child->boolValue()); // Doc comment else if (pn_lower == "comment_doc") lang->setDocComment(child->stringValue()); // Keyword lookup link else if (pn_lower == "keyword_link") lang->word_lists_[WordType::Keyword].lookup_url = child->stringValue(); // Constant lookup link else if (pn_lower == "constant_link") lang->word_lists_[WordType::Constant].lookup_url = child->stringValue(); // Function lookup link else if (pn_lower == "function_link") lang->f_lookup_url_ = child->stringValue(); // Jump blocks else if (pn_lower == "blocks") { for (unsigned v = 0; v < child->nValues(); v++) lang->jump_blocks_.push_back(child->stringValue(v)); } else if (pn_lower == "blocks_ignore") { for (unsigned v = 0; v < child->nValues(); v++) lang->jb_ignore_.push_back(child->stringValue(v)); } // Block begin else if (pn_lower == "block_begin") lang->block_begin_ = child->stringValue(); // Block end else if (pn_lower == "block_end") lang->block_end_ = child->stringValue(); // Preprocessor block begin else if (pn_lower == "pp_block_begin") { for (unsigned v = 0; v < child->nValues(); v++) lang->pp_block_begin_.push_back(child->stringValue(v)); } // Preprocessor block end else if (pn_lower == "pp_block_end") { for (unsigned v = 0; v < child->nValues(); v++) lang->pp_block_end_.push_back(child->stringValue(v)); } // Word block begin else if (pn_lower == "word_block_begin") { for (unsigned v = 0; v < child->nValues(); v++) lang->word_block_begin_.push_back(child->stringValue(v)); } // Word block end else if (pn_lower == "word_block_end") { for (unsigned v = 0; v < child->nValues(); v++) lang->word_block_end_.push_back(child->stringValue(v)); } // Keywords else if (pn_lower == "keywords") { // Go through values for (unsigned v = 0; v < child->nValues(); v++) { auto val = child->stringValue(v); // Check for '$override' if (StrUtil::equalCI(val, "$override")) { // Clear any inherited keywords lang->clearWordList(WordType::Keyword); } // Not a special symbol, add as keyword else lang->addWord(WordType::Keyword, val); } } // Constants else if (pn_lower == "constants") { // Go through values for (unsigned v = 0; v < child->nValues(); v++) { auto val = child->stringValue(v); // Check for '$override' if (StrUtil::equalCI(val, "$override")) { // Clear any inherited constants lang->clearWordList(WordType::Constant); } // Not a special symbol, add as constant else lang->addWord(WordType::Constant, val); } } // Types else if (pn_lower == "types") { // Go through values for (unsigned v = 0; v < child->nValues(); v++) { auto val = child->stringValue(v); // Check for '$override' if (StrUtil::equalCI(val, "$override")) { // Clear any inherited constants lang->clearWordList(WordType::Type); } // Not a special symbol, add as constant else lang->addWord(WordType::Type, val); } } // Properties else if (pn_lower == "properties") { // Go through values for (unsigned v = 0; v < child->nValues(); v++) { auto val = child->stringValue(v); // Check for '$override' if (StrUtil::equalCI(val, "$override")) { // Clear any inherited constants lang->clearWordList(WordType::Property); } // Not a special symbol, add as constant else lang->addWord(WordType::Property, val); } } // Functions else if (pn_lower == "functions") { bool lang_has_void = lang->isWord(Keyword, "void") || lang->isWord(Type, "void"); if (lang->id_ != "zscript") { // Go through children (functions) for (unsigned f = 0; f < child->nChildren(); f++) { auto child_func = child->childPTN(f); string params; // Simple definition if (child_func->nChildren() == 0) { if (child_func->stringValue(0).empty()) { if (lang_has_void) params = "void"; else params = ""; } else { params = child_func->stringValue(0); } // Add function lang->addFunction( child_func->name(), params, "", "", !StrUtil::contains(child_func->name(), '.'), child_func->type()); // Add args for (unsigned v = 1; v < child_func->nValues(); v++) lang->addFunction(child_func->name(), child_func->stringValue(v)); } // Full definition else { string name = child_func->name(); vector<string> args; string desc; string deprecated; for (unsigned p = 0; p < child_func->nChildren(); p++) { auto child_prop = child_func->childPTN(p); if (child_prop->name() == "args") { for (unsigned v = 0; v < child_prop->nValues(); v++) args.push_back(child_prop->stringValue(v)); } else if (child_prop->name() == "description") desc = child_prop->stringValue(); else if (child_prop->name() == "deprecated") deprecated = child_prop->stringValue(); } if (args.empty() && lang_has_void) args.emplace_back("void"); for (unsigned as = 0; as < args.size(); as++) lang->addFunction(name, args[as], desc, deprecated, as == 0, child_func->type()); } } } // ZScript function info which cannot be parsed from (g)zdoom.pk3 else { ZFuncExProp ex_prop; for (unsigned f = 0; f < child->nChildren(); f++) { auto child_func = child->childPTN(f); for (unsigned p = 0; p < child_func->nChildren(); ++p) { auto child_prop = child_func->childPTN(p); if (child_prop->name() == "description") ex_prop.description = child_prop->stringValue(); else if (child_prop->name() == "deprecated_f") ex_prop.deprecated_f = child_prop->stringValue(); } lang->zfuncs_ex_props_.emplace(child_func->name(), ex_prop); } } } } } return true; }