// scanToken() will always get a new token from the frontier
// regardless of whether there are tokens in the lookahead store
int Scanner::scanToken(ScannerToken &t, Location &l) {
m_token = &t;
m_loc = &l;
int tokid;
for (;;) {
tokid = scan();
switch (tokid) {
case T_DOC_COMMENT:
setDocComment(m_token->text());
/* fall through */
case T_COMMENT:
case T_OPEN_TAG:
case T_WHITESPACE:
if (m_type & ReturnAllTokens) {
// m_lastToken holds the last "signficant" token, so
// don't update it for comments or whitespace
return tokid;
}
break;
default:
m_lastToken = tokid;
return tokid;
}
}
}
// -----------------------------------------------------------------------------
// Reads in a text definition of a language. See slade.pk3 for
// formatting examples
// -----------------------------------------------------------------------------
bool TextLanguage::readLanguageDefinition(MemChunk& mc, string_view source)
{
Tokenizer tz;
// Open the given text data
if (!tz.openMem(mc, source))
{
Log::warning("Unable to open language definition {}", source);
return false;
}
// Parse the definition text
ParseTreeNode root;
if (!root.parse(tz))
return false;
// Get parsed data
for (unsigned a = 0; a < root.nChildren(); a++)
{
auto node = root.childPTN(a);
// Create language
auto lang = new TextLanguage(node->name());
// Check for inheritance
if (!node->inherit().empty())
{
auto inherit = fromId(node->inherit());
if (inherit)
inherit->copyTo(lang);
else
Log::warning("Warning: Language {} inherits from undefined language {}", node->name(), node->inherit());
}
// Parse language info
for (unsigned c = 0; c < node->nChildren(); c++)
{
auto child = node->childPTN(c);
auto pn_lower = StrUtil::lower(child->name());
// Language name
if (pn_lower == "name")
lang->setName(child->stringValue());
// Comment begin
else if (pn_lower == "comment_begin")
{
lang->setCommentBeginList(child->stringValues());
}
// Comment end
else if (pn_lower == "comment_end")
{
lang->setCommentEndList(child->stringValues());
}
// Line comment
else if (pn_lower == "comment_line")
{
lang->setLineCommentList(child->stringValues());
}
// Preprocessor
else if (pn_lower == "preprocessor")
lang->setPreprocessor(child->stringValue());
// Case sensitive
else if (pn_lower == "case_sensitive")
lang->setCaseSensitive(child->boolValue());
// Doc comment
else if (pn_lower == "comment_doc")
lang->setDocComment(child->stringValue());
// Keyword lookup link
else if (pn_lower == "keyword_link")
lang->word_lists_[WordType::Keyword].lookup_url = child->stringValue();
// Constant lookup link
else if (pn_lower == "constant_link")
lang->word_lists_[WordType::Constant].lookup_url = child->stringValue();
// Function lookup link
else if (pn_lower == "function_link")
lang->f_lookup_url_ = child->stringValue();
// Jump blocks
else if (pn_lower == "blocks")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->jump_blocks_.push_back(child->stringValue(v));
}
else if (pn_lower == "blocks_ignore")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->jb_ignore_.push_back(child->stringValue(v));
}
// Block begin
else if (pn_lower == "block_begin")
lang->block_begin_ = child->stringValue();
// Block end
else if (pn_lower == "block_end")
lang->block_end_ = child->stringValue();
// Preprocessor block begin
else if (pn_lower == "pp_block_begin")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->pp_block_begin_.push_back(child->stringValue(v));
}
// Preprocessor block end
else if (pn_lower == "pp_block_end")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->pp_block_end_.push_back(child->stringValue(v));
}
// Word block begin
else if (pn_lower == "word_block_begin")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->word_block_begin_.push_back(child->stringValue(v));
}
// Word block end
else if (pn_lower == "word_block_end")
{
for (unsigned v = 0; v < child->nValues(); v++)
lang->word_block_end_.push_back(child->stringValue(v));
}
// Keywords
else if (pn_lower == "keywords")
{
// Go through values
for (unsigned v = 0; v < child->nValues(); v++)
{
auto val = child->stringValue(v);
// Check for '$override'
if (StrUtil::equalCI(val, "$override"))
{
// Clear any inherited keywords
lang->clearWordList(WordType::Keyword);
}
// Not a special symbol, add as keyword
else
lang->addWord(WordType::Keyword, val);
}
}
// Constants
else if (pn_lower == "constants")
{
// Go through values
for (unsigned v = 0; v < child->nValues(); v++)
{
auto val = child->stringValue(v);
// Check for '$override'
if (StrUtil::equalCI(val, "$override"))
{
// Clear any inherited constants
lang->clearWordList(WordType::Constant);
}
// Not a special symbol, add as constant
else
lang->addWord(WordType::Constant, val);
}
}
// Types
else if (pn_lower == "types")
{
// Go through values
for (unsigned v = 0; v < child->nValues(); v++)
{
auto val = child->stringValue(v);
// Check for '$override'
if (StrUtil::equalCI(val, "$override"))
{
// Clear any inherited constants
lang->clearWordList(WordType::Type);
}
// Not a special symbol, add as constant
else
lang->addWord(WordType::Type, val);
}
}
// Properties
else if (pn_lower == "properties")
{
// Go through values
for (unsigned v = 0; v < child->nValues(); v++)
{
auto val = child->stringValue(v);
// Check for '$override'
if (StrUtil::equalCI(val, "$override"))
{
// Clear any inherited constants
lang->clearWordList(WordType::Property);
}
// Not a special symbol, add as constant
else
lang->addWord(WordType::Property, val);
}
}
// Functions
else if (pn_lower == "functions")
{
bool lang_has_void = lang->isWord(Keyword, "void") || lang->isWord(Type, "void");
if (lang->id_ != "zscript")
{
// Go through children (functions)
for (unsigned f = 0; f < child->nChildren(); f++)
{
auto child_func = child->childPTN(f);
string params;
// Simple definition
if (child_func->nChildren() == 0)
{
if (child_func->stringValue(0).empty())
{
if (lang_has_void)
params = "void";
else
params = "";
}
else
{
params = child_func->stringValue(0);
}
// Add function
lang->addFunction(
child_func->name(),
params,
"",
"",
!StrUtil::contains(child_func->name(), '.'),
child_func->type());
// Add args
for (unsigned v = 1; v < child_func->nValues(); v++)
lang->addFunction(child_func->name(), child_func->stringValue(v));
}
// Full definition
else
{
string name = child_func->name();
vector<string> args;
string desc;
string deprecated;
for (unsigned p = 0; p < child_func->nChildren(); p++)
{
auto child_prop = child_func->childPTN(p);
if (child_prop->name() == "args")
{
for (unsigned v = 0; v < child_prop->nValues(); v++)
args.push_back(child_prop->stringValue(v));
}
else if (child_prop->name() == "description")
desc = child_prop->stringValue();
else if (child_prop->name() == "deprecated")
deprecated = child_prop->stringValue();
}
if (args.empty() && lang_has_void)
args.emplace_back("void");
for (unsigned as = 0; as < args.size(); as++)
lang->addFunction(name, args[as], desc, deprecated, as == 0, child_func->type());
}
}
}
// ZScript function info which cannot be parsed from (g)zdoom.pk3
else
{
ZFuncExProp ex_prop;
for (unsigned f = 0; f < child->nChildren(); f++)
{
auto child_func = child->childPTN(f);
for (unsigned p = 0; p < child_func->nChildren(); ++p)
{
auto child_prop = child_func->childPTN(p);
if (child_prop->name() == "description")
ex_prop.description = child_prop->stringValue();
else if (child_prop->name() == "deprecated_f")
ex_prop.deprecated_f = child_prop->stringValue();
}
lang->zfuncs_ex_props_.emplace(child_func->name(), ex_prop);
}
}
}
}
}
return true;
}
