// ---------------------------------------------------------------------------- // TLFunction::addContext // // Adds a [context] of the function // ---------------------------------------------------------------------------- void TLFunction::addContext( const string& context, const string& args, const string& return_type, string description, const string& deprecated_f) { contexts_.push_back(Context{ context, {}, return_type, description, "", "", "" }); auto& ctx = contexts_.back(); // Parse args Tokenizer tz; tz.setSpecialCharacters("[],"); tz.openString(args); vector<string> arg_tokens; while (true) { while (!tz.check(",")) { arg_tokens.push_back(tz.current().text); if (tz.atEnd()) break; tz.adv(); } ctx.params.push_back({}); ctx.params.back().parse(arg_tokens); arg_tokens.clear(); if (tz.atEnd()) break; tz.adv(); } if (!deprecated_f.empty()) { // Parse deprecated string tz.openString(deprecated_f); for (unsigned t = 0; t < 2; t++) { while (tz.check(",")) tz.adv(); bool is_replacement = true; for (unsigned c = 0; c < tz.current().text.size(); c++) { char chr = tz.current().text[c]; if (isdigit(chr) || chr == '.') { is_replacement = false; break; } } if (is_replacement) ctx.deprecated_f = tz.current().text; else ctx.deprecated_v = tz.current().text; if (tz.atEnd()) break; tz.adv(); } } }
// ---------------------------------------------------------------------------- // ParseTreeNode::parse // // Parses formatted text data. Current valid formatting is: // (type) child = value; // (type) child = value1, value2, ...; // (type) child = { value1, value2, ... } // (type) child { grandchild = value; etc... } // (type) child : inherited { ... } // // All values are read as strings, but can be retrieved as string, int, bool // or float. // ---------------------------------------------------------------------------- bool ParseTreeNode::parse(Tokenizer& tz) { // Keep parsing until final } is reached (or end of file) string name, type; while (!tz.atEnd() && tz.current() != '}') { // Check for preprocessor stuff if (parser_ && tz.current()[0] == '#') { if (!parsePreprocessor(tz)) return false; tz.advToNextLine(); continue; } // If it's a special character (ie not a valid name), parsing fails if (tz.isSpecialCharacter(tz.current().text[0])) { logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text))); return false; } // So we have either a node or property name name = tz.current().text; type.Empty(); if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } // Check for type+name pair if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':') { type = name; name = tz.next().text; if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } } //Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text))); // Assignment if (tz.advIfNext('=', 2)) { if (!parseAssignment(tz, addChildPTN(name, type))) return false; } // Child node else if (tz.advIfNext('{', 2)) { // Parse child node if (!addChildPTN(name, type)->parse(tz)) return false; } // Child node (with no values/children) else if (tz.advIfNext(';', 2)) { // Add child node addChildPTN(name, type); continue; } // Child node + inheritance else if (tz.advIfNext(':', 2)) { // Check for opening brace if (tz.checkNext('{')) { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip { tz.adv(2); // Parse child node if (!child->parse(tz)) return false; } else if (tz.checkNext(';')) // Empty child node { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip ; tz.adv(2); continue; } else { logError(tz, S_FMT("Expecting \"{\" or \";\", got \"%s\"", CHR(tz.next().text))); return false; } } // Unexpected token else { logError(tz, S_FMT("Unexpected token \"%s\"", CHR(tz.next().text))); return false; } // Continue parsing tz.adv(); } // Success return true; }
/* MapSpecials::processACSScripts * Process 'OPEN' ACS scripts for various specials - sector colours, * slopes, etc. *******************************************************************/ void MapSpecials::processACSScripts(ArchiveEntry* entry) { sector_colours.clear(); sector_fadecolours.clear(); if (!entry || entry->getSize() == 0) return; Tokenizer tz; tz.setSpecialCharacters(";,:|={}/()"); tz.openMem(entry->getMCData(), "ACS Scripts"); while (!tz.atEnd()) { if (tz.checkNC("script")) { LOG_MESSAGE(3, "script found"); tz.adv(2); // Skip script # // Check for open script if (tz.checkNC("OPEN")) { LOG_MESSAGE(3, "script is OPEN"); // Skip to opening brace while (!tz.check("{")) tz.adv(); // Parse script while (!tz.checkOrEnd("}")) { // --- Sector_SetColor --- if (tz.checkNC("Sector_SetColor")) { // Get parameters auto parameters = tz.getTokensUntil(")"); // Parse parameters long val; int tag = -1; int r = -1; int g = -1; int b = -1; for (unsigned a = 0; a < parameters.size(); a++) { if (parameters[a].text.ToLong(&val)) { if (tag < 0) tag = val; else if (r < 0) r = val; else if (g < 0) g = val; else if (b < 0) b = val; } } // Check everything is set if (b < 0) { LOG_MESSAGE(2, "Invalid Sector_SetColor parameters"); } else { sector_colour_t sc; sc.tag = tag; sc.colour.set(r, g, b, 255); LOG_MESSAGE(3, "Sector tag %d, colour %d,%d,%d", tag, r, g, b); sector_colours.push_back(sc); } } // --- Sector_SetFade --- else if (tz.checkNC("Sector_SetFade")) { // Get parameters auto parameters = tz.getTokensUntil(")"); // Parse parameters long val; int tag = -1; int r = -1; int g = -1; int b = -1; for (unsigned a = 0; a < parameters.size(); a++) { if (parameters[a].text.ToLong(&val)) { if (tag < 0) tag = val; else if (r < 0) r = val; else if (g < 0) g = val; else if (b < 0) b = val; } } // Check everything is set if (b < 0) { LOG_MESSAGE(2, "Invalid Sector_SetFade parameters"); } else { sector_colour_t sc; sc.tag = tag; sc.colour.set(r, g, b, 0); LOG_MESSAGE(3, "Sector tag %d, fade colour %d,%d,%d", tag, r, g, b); sector_fadecolours.push_back(sc); } } tz.adv(); } } } tz.adv(); } }
// ---------------------------------------------------------------------------- // ParseTreeNode::parsePreprocessor // // Parses a preprocessor directive at [tz]'s current token // ---------------------------------------------------------------------------- bool ParseTreeNode::parsePreprocessor(Tokenizer& tz) { //Log::debug(S_FMT("Preprocessor %s", CHR(tz.current().text))); // #define if (tz.current() == "#define") parser_->define(tz.next().text); // #if(n)def else if (tz.current() == "#ifdef" || tz.current() == "#ifndef") { // Continue if condition succeeds bool test = true; if (tz.current() == "#ifndef") test = false; string define = tz.next().text; if (parser_->defined(define) == test) return true; // Failed condition, skip section int skip = 0; while (true) { auto& token = tz.next(); if (token == "#endif") skip--; else if (token == "#ifdef") skip++; else if (token == "#ifndef") skip++; // TODO: #else if (skip < 0) break; } } // #include else if (tz.current() == "#include") { // Include entry at the given path if we have an archive dir set if (archive_dir_) { // Get entry to include auto inc_path = tz.next().text; auto archive = archive_dir_->archive(); auto inc_entry = archive->entryAtPath(archive_dir_->getPath() + inc_path); if (!inc_entry) // Try absolute path inc_entry = archive->entryAtPath(inc_path); //Log::debug(S_FMT("Include %s", CHR(inc_path))); if (inc_entry) { // Save the current dir and set it to the included entry's dir auto orig_dir = archive_dir_; archive_dir_ = inc_entry->getParentDir(); // Parse text in the entry Tokenizer inc_tz; inc_tz.openMem(inc_entry->getMCData(), inc_entry->getName()); bool ok = parse(inc_tz); // Reset dir and abort if parsing failed archive_dir_ = orig_dir; if (!ok) return false; } else logError(tz, S_FMT("Include entry %s not found", CHR(inc_path))); } else tz.adv(); // Skip include path } // #endif (ignore) else if (tz.current() == "#endif") return true; // TODO: #else // Unrecognised else logError(tz, S_FMT("Unrecognised preprocessor directive \"%s\"", CHR(tz.current().text))); return true; }
// ---------------------------------------------------------------------------- // ParseTreeNode::parseAssignment // // Parses an assignment operation at [tz]'s current token to [child] // ---------------------------------------------------------------------------- bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const { // Check type of assignment list char list_end = ';'; if (tz.current() == '{' && !tz.current().quoted_string) { list_end = '}'; tz.adv(); } // Parse until ; or } while (true) { auto& token = tz.current(); // Check for list end if (token == list_end && !token.quoted_string) break; // Setup value Property value; // Detect value type if (token.quoted_string) // Quoted string value = token.text; else if (token == "true") // Boolean (true) value = true; else if (token == "false") // Boolean (false) value = false; else if (token.isInteger()) // Integer { long val; token.text.ToLong(&val); value = (int)val; } else if (token.isHex()) // Hex (0xXXXXXX) { long val; token.text.ToLong(&val, 0); value = (int)val; } else if (token.isFloat()) // Floating point { double val; token.text.ToDouble(&val); value = val; } else // Unknown, just treat as string value = token.text; // Add value child->values_.push_back(value); // Check for , if (tz.peek() == ',') tz.adv(); // Skip it else if (tz.peek() != list_end) { logError( tz, S_FMT("Expected \",\" or \"%c\", got \"%s\"", list_end, CHR(tz.peek().text)) ); return false; } tz.adv(); } return true; }
// ----------------------------------------------------------------------------- // Parses ZMAPINFO-format definitions in [entry] // ----------------------------------------------------------------------------- bool MapInfo::parseZMapInfo(ArchiveEntry* entry) { Tokenizer tz; tz.setReadLowerCase(true); tz.openMem(entry->data(), entry->name()); while (!tz.atEnd()) { // Include if (tz.check("include")) { // Get entry at include path auto include_entry = entry->parent()->entryAtPath(tz.next().text); if (!include_entry) { Log::warning( "Warning - Parsing ZMapInfo \"{}\": Unable to include \"{}\" at line {}", entry->name(), tz.current().text, tz.lineNo()); } else if (!parseZMapInfo(include_entry)) return false; } // Map else if (tz.check("map") || tz.check("defaultmap") || tz.check("adddefaultmap")) { if (!parseZMap(tz, tz.current().text)) return false; } // DoomEdNums else if (tz.check("doomednums")) { if (!parseDoomEdNums(tz)) return false; } // Unknown block (skip it) else if (tz.check("{")) { Log::warning(2, "Warning - Parsing ZMapInfo \"{}\": Skipping {{}} block", entry->name()); tz.adv(); tz.skipSection("{", "}"); continue; } // Unknown else { Log::warning(2, R"(Warning - Parsing ZMapInfo "{}": Unknown token "{}")", entry->name(), tz.current().text); } tz.adv(); } Log::info(2, "Parsed ZMapInfo entry {} successfully", entry->name()); return true; } // ----------------------------------------------------------------------------- // Parses a ZMAPINFO map definition of [type] beginning at the current token in // tokenizer [tz] // ----------------------------------------------------------------------------- bool MapInfo::parseZMap(Tokenizer& tz, std::string_view type) { // TODO: Handle adddefaultmap auto map = default_map_; // Normal map, get lump/name/etc tz.adv(); if (type == "map") { // Entry name should be just after map keyword map.entry_name = tz.current().text; // Parse map name tz.adv(); if (tz.check("lookup")) { map.lookup_name = true; map.name = tz.next().text; } else { map.lookup_name = false; map.name = tz.current().text; } tz.adv(); } if (!tz.advIf("{")) { Log::error(R"(Error Parsing ZMapInfo: Expecting "{{", got "{}" at line {})", tz.current().text, tz.lineNo()); return false; } while (!tz.checkOrEnd("}")) { // Block (skip it) if (tz.advIf("{")) tz.skipSection("{", "}"); // LevelNum else if (tz.check("levelnum")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // Parse number // TODO: Checks tz.next().toInt(map.level_num); } // Sky1 else if (tz.check("sky1")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky1 = tz.next().text; // Scroll speed // TODO: Checks if (tz.advIfNext(",")) tz.next().toFloat(map.sky1_scroll_speed); } // Sky2 else if (tz.check("sky2")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky2 = tz.next().text; // Scroll speed // TODO: Checks if (tz.advIfNext(",")) tz.next().toFloat(map.sky2_scroll_speed); } // Skybox else if (tz.check("skybox")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky1 = tz.next().text; } // DoubleSky else if (tz.check("doublesky")) map.sky_double = true; // ForceNoSkyStretch else if (tz.check("forcenoskystretch")) map.sky_force_no_stretch = true; // SkyStretch else if (tz.check("skystretch")) map.sky_stretch = true; // Fade else if (tz.check("fade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; if (!strToCol(tz.next().text, map.fade)) return false; } // OutsideFog else if (tz.check("outsidefog")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; if (!strToCol(tz.next().text, map.fade_outside)) return false; } // EvenLighting else if (tz.check("evenlighting")) { map.lighting_wallshade_h = 0; map.lighting_wallshade_v = 0; } // SmoothLighting else if (tz.check("smoothlighting")) map.lighting_smooth = true; // VertWallShade else if (tz.check("vertwallshade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // TODO: Checks tz.next().toInt(map.lighting_wallshade_v); } // HorzWallShade else if (tz.check("horzwallshade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // TODO: Checks tz.next().toInt(map.lighting_wallshade_h); } // ForceFakeContrast else if (tz.check("forcefakecontrast")) map.force_fake_contrast = true; tz.adv(); }
// ----------------------------------------------------------------------------- // Reads and parses the SLADE configuration file // ----------------------------------------------------------------------------- void readConfigFile() { // Open SLADE.cfg Tokenizer tz; if (!tz.openFile(App::path("slade3.cfg", App::Dir::User))) return; // Go through the file with the tokenizer while (!tz.atEnd()) { // If we come across a 'cvars' token, read in the cvars section if (tz.advIf("cvars", 2)) { // Keep reading name/value pairs until we hit the ending '}' while (!tz.checkOrEnd("}")) { CVar::set(tz.current().text, tz.peek().text); tz.adv(2); } tz.adv(); // Skip ending } } // Read base resource archive paths if (tz.advIf("base_resource_paths", 2)) { while (!tz.checkOrEnd("}")) { archive_manager.addBaseResourcePath(tz.current().text); tz.adv(); } tz.adv(); // Skip ending } } // Read recent files list if (tz.advIf("recent_files", 2)) { while (!tz.checkOrEnd("}")) { archive_manager.addRecentFile(tz.current().text); tz.adv(); } tz.adv(); // Skip ending } } // Read keybinds if (tz.advIf("keys", 2)) KeyBind::readBinds(tz); // Read nodebuilder paths if (tz.advIf("nodebuilder_paths", 2)) { while (!tz.checkOrEnd("}")) { NodeBuilders::addBuilderPath(tz.current().text, tz.peek().text); tz.adv(2); } tz.adv(); // Skip ending } } // Read game exe paths if (tz.advIf("executable_paths", 2)) { while (!tz.checkOrEnd("}")) { Executables::setGameExePath(tz.current().text, tz.peek().text); tz.adv(2); } tz.adv(); // Skip ending } } // Read window size/position info if (tz.advIf("window_info", 2)) Misc::readWindowInfo(tz); // Next token tz.adv(); } }
// ----------------------------------------------------------------------------- // Parses an assignment operation at [tz]'s current token to [child] // ----------------------------------------------------------------------------- bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const { // Check type of assignment list char list_end = ';'; if (tz.current() == '{' && !tz.current().quoted_string) { list_end = '}'; tz.adv(); } // Parse until ; or } while (true) { auto& token = tz.current(); // Check for list end if (token == list_end && !token.quoted_string) break; // Setup value Property value; // Detect value type if (token.quoted_string) // Quoted string value = token.text; else if (token == "true") // Boolean (true) value = true; else if (token == "false") // Boolean (false) value = false; else if (token.isInteger()) // Integer value = token.asInt(); else if (token.isHex()) // Hex (0xXXXXXX) value = token.asInt(); else if (token.isFloat()) // Floating point value = token.asFloat(); else // Unknown, just treat as string value = token.text; // Add value child->values_.push_back(value); // Check for , if (tz.peek() == ',') tz.adv(); // Skip it else if (tz.peek() != list_end) { logError(tz, fmt::format(R"(Expected "," or "{}", got "{}")", list_end, tz.peek().text)); return false; } tz.adv(); } return true; } // ----------------------------------------------------------------------------- // Parses formatted text data. Current valid formatting is: // (type) child = value; // (type) child = value1, value2, ...; // (type) child = { value1, value2, ... } // (type) child { grandchild = value; etc... } // (type) child : inherited { ... } // // All values are read as strings, but can be retrieved as string, int, bool // or float. // ----------------------------------------------------------------------------- bool ParseTreeNode::parse(Tokenizer& tz) { // Keep parsing until final } is reached (or end of file) string name, type; while (!tz.atEnd() && tz.current() != '}') { // Check for preprocessor stuff if (parser_ && tz.current()[0] == '#') { if (!parsePreprocessor(tz)) return false; tz.advToNextLine(); continue; } // If it's a special character (ie not a valid name), parsing fails if (tz.isSpecialCharacter(tz.current().text[0])) { logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text)); return false; } // So we have either a node or property name name = tz.current().text; type.clear(); if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } // Check for type+name pair if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':') { type = name; name = tz.next().text; if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } } // Log::debug(wxString::Format("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text))); // Assignment if (tz.advIfNext('=', 2)) { if (!parseAssignment(tz, addChildPTN(name, type))) return false; } // Child node else if (tz.advIfNext('{', 2)) { // Parse child node if (!addChildPTN(name, type)->parse(tz)) return false; } // Child node (with no values/children) else if (tz.advIfNext(';', 2)) { // Add child node addChildPTN(name, type); continue; } // Child node + inheritance else if (tz.advIfNext(':', 2)) { // Check for opening brace if (tz.checkNext('{')) { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip { tz.adv(2); // Parse child node if (!child->parse(tz)) return false; } else if (tz.checkNext(';')) // Empty child node { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip ; tz.adv(2); continue; } else { logError(tz, fmt::format(R"(Expecting "{{" or ";", got "{}")", tz.next().text)); return false; } } // Unexpected token else { logError(tz, fmt::format("Unexpected token \"{}\"", tz.next().text)); return false; } // Continue parsing tz.adv(); } // Success return true; }
// ----------------------------------------------------------------------------- // Parses a ZScript 'statement'. This isn't technically correct but suits our // purposes well enough // // tokens // { // block[0].tokens // { // block[0].block[0].tokens; // ... // } // // block[1].tokens; // ... // } // ----------------------------------------------------------------------------- bool ParsedStatement::parse(Tokenizer& tz) { // Check for unexpected token if (tz.check('}')) { tz.adv(); return false; } line = tz.lineNo(); // Tokens bool in_initializer = false; while (true) { // End of statement (;) if (tz.advIf(';')) return true; // DB comment if (tz.current().text.StartsWith(db_comment)) { tokens.push_back(tz.current().text); tokens.push_back(tz.getLine()); return true; } if (tz.check('}')) { // End of array initializer if (in_initializer) { in_initializer = false; tokens.emplace_back("}"); tz.adv(); continue; } // End of statement return true; } if (tz.atEnd()) { Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line)); return false; } // Beginning of block if (tz.advIf('{')) break; // Array initializer: ... = { ... } if (tz.current().text.Cmp("=") == 0 && tz.peek() == '{') { tokens.emplace_back("="); tokens.emplace_back("{"); tz.adv(2); in_initializer = true; continue; } tokens.push_back(tz.current().text); tz.adv(); } // Block while (true) { if (tz.advIf('}')) return true; if (tz.atEnd()) { Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line)); return false; } block.push_back({}); block.back().entry = entry; if (!block.back().parse(tz) || block.back().tokens.empty()) block.pop_back(); } }
// ----------------------------------------------------------------------------- // Reads in a ZDoom-format TEXTURES entry. // Returns true on success, false otherwise // ----------------------------------------------------------------------------- bool TextureXList::readTEXTURESData(ArchiveEntry* textures) { // Check for empty entry if (!textures) { Global::error = "Attempt to read texture data from NULL entry"; return false; } if (textures->size() == 0) { txformat_ = Format::Textures; return true; } // Get text to parse Tokenizer tz; tz.openMem(textures->data(), textures->name()); // Parsing gogo while (!tz.atEnd()) { // Texture definition if (tz.checkNC("Texture")) { auto tex = std::make_unique<CTexture>(); if (tex->parse(tz, "Texture")) addTexture(std::move(tex)); } // Sprite definition else if (tz.checkNC("Sprite")) { auto tex = std::make_unique<CTexture>(); if (tex->parse(tz, "Sprite")) addTexture(std::move(tex)); } // Graphic definition else if (tz.checkNC("Graphic")) { auto tex = std::make_unique<CTexture>(); if (tex->parse(tz, "Graphic")) addTexture(std::move(tex)); } // WallTexture definition else if (tz.checkNC("WallTexture")) { auto tex = std::make_unique<CTexture>(); if (tex->parse(tz, "WallTexture")) addTexture(std::move(tex)); } // Flat definition else if (tz.checkNC("Flat")) { auto tex = std::make_unique<CTexture>(); if (tex->parse(tz, "Flat")) addTexture(std::move(tex)); } // Old HIRESTEX "Define" else if (tz.checkNC("Define")) { auto tex = std::make_unique<CTexture>(); if (tex->parseDefine(tz)) addTexture(std::move(tex)); } tz.adv(); } txformat_ = Format::Textures; return true; }