//Sets the interaction mode based on command sent from tablet //Echoes a confirmation message back to tablet void Terrain::handleMessages(){ std::string msg; msg = server->parseData(); ostringstream ss; ss << "Handling Message: " << msg << std::endl; if(!msg.empty()){ OutputDebugString(ss.str().c_str()); } //Tokenize message Tokenizer *tok = new Tokenizer(msg, ","); std::string token = tok->next(); //Prepare confirmation reply ostringstream replyss; replyss << "CFM," << msg; std::string reply = replyss.str(); //Selecting type of interaction if(token == "placepoi"){ token = tok->next(); poiCube->setText(token); setInteractionMode(POI_MODE); //server->sendData("placepoi\n"); server->sendData(reply.c_str()); } else if(msg == "placepath\n"){ setInteractionMode(ENDPOINT_DRAWING_MODE); //server->sendData("placepath\n"); server->sendData(reply.c_str()); } else if(msg == "drawpath\n"){ setInteractionMode(DIRECT_DRAWING_MODE); //server->sendData("drawpath\n"); server->sendData(reply.c_str()); } else if(msg == "placeroi\n"){ setInteractionMode(ROI_MODE); //server->sendData("placeroi\n"); server->sendData(reply.c_str()); } else if(msg == "placepoi\n"){ setInteractionMode(POI_MODE); //server->sendData("placepoi\n"); server->sendData(reply.c_str()); } else if(msg == "filterheight\n"){ setInteractionMode(HEIGHT_FILTERING_MODE); server->sendData(reply.c_str()); } else if(msg == "selectheight\n"){ setInteractionMode(HEIGHT_SELECT_MODE); //server->sendData("selectheight\n"); server->sendData(reply.c_str()); } }
void QSGShaderSourceBuilder::removeVersion() { Tokenizer tok; const char *input = m_source.constData(); tok.initialize(input); // First find #version beginning and end (if present) const char *versionStartPos = 0; const char *versionEndPos = 0; bool inSingleLineComment = false; bool inMultiLineComment = false; bool foundVersionStart = false; Tokenizer::Token lt = Tokenizer::Token_Unspecified; Tokenizer::Token t = tok.next(); while (t != Tokenizer::Token_EOF) { // Handle comment blocks if (t == Tokenizer::Token_MultiLineCommentStart ) inMultiLineComment = true; if (t == Tokenizer::Token_MultiLineCommentEnd) inMultiLineComment = false; if (t == Tokenizer::Token_SingleLineComment) inSingleLineComment = true; if (t == Tokenizer::Token_NewLine && inSingleLineComment && !inMultiLineComment) inSingleLineComment = false; // Have we found #version, #extension or void main()? if (t == Tokenizer::Token_Version && !inSingleLineComment && !inMultiLineComment) { versionStartPos = tok.pos - 1; foundVersionStart = true; } else if (foundVersionStart && t == Tokenizer::Token_NewLine) { versionEndPos = tok.pos; break; } else if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) { if (qstrncmp("main", tok.identifier, 4) == 0) break; } // Scan to next token lt = t; t = tok.next(); } if (versionStartPos == 0) return; // Construct a new shader string, inserting the definition QByteArray newSource; newSource.reserve(m_source.size() - (versionEndPos - versionStartPos)); newSource += QByteArray::fromRawData(input, versionStartPos - input); newSource += QByteArray::fromRawData(versionEndPos, m_source.size() - (versionEndPos - versionStartPos)); m_source = newSource; }
void Runtime::accept( Token::Code token ) { if ( expect( token ) ) { zdebug() << "ACCEPTED TOKEN" << token; tokenizer.next(); } }
void Memory::loadFile(string fileName){ ifstream file(fileName.c_str(), std::ifstream::in); //check if file is ready for reading if(file.good()){ string line; string tkn; Tokenizer tokenizer; int cursor = 0; //read all lines while(getline(file, line)){ tokenizer.set(line); tkn = tokenizer.next(); if(tkn == ".1000") cursor = 1000; else{ MemVector[cursor] = atoi(tkn.c_str()); cursor++; } } file.close(); }else cout << "Could not open file " << fileName << endl; }
// ----------------------------------------------------------------------------- // Returns true if the next token in [tz] is '='. If not, logs an error message // ----------------------------------------------------------------------------- bool MapInfo::checkEqualsToken(Tokenizer& tz, std::string_view parsing) const { if (tz.next() != "=") { Log::error("Error Parsing {}: Expected \"=\", got \"{}\" at line {}", parsing, tz.current().text, tz.lineNo()); return false; } return true; }
// ----------------------------------------------------------------------------- // Parses all statements/blocks in [entry], adding them to [parsed] // ----------------------------------------------------------------------------- void parseBlocks(ArchiveEntry* entry, vector<ParsedStatement>& parsed) { Tokenizer tz; tz.setSpecialCharacters(CHR(Tokenizer::DEFAULT_SPECIAL_CHARACTERS + "()+-[]&!?.")); tz.enableDecorate(true); tz.setCommentTypes(Tokenizer::CommentTypes::CPPStyle | Tokenizer::CommentTypes::CStyle); tz.openMem(entry->getMCData(), "ZScript"); // Log::info(2, S_FMT("Parsing ZScript entry \"%s\"", entry->getPath(true))); while (!tz.atEnd()) { // Preprocessor if (tz.current().text.StartsWith("#")) { if (tz.checkNC("#include")) { auto inc_entry = entry->relativeEntry(tz.next().text); // Check #include path could be resolved if (!inc_entry) { Log::warning(S_FMT( "Warning parsing ZScript entry %s: " "Unable to find #included entry \"%s\" at line %u, skipping", CHR(entry->getName()), CHR(tz.current().text), tz.current().line_no)); } else parseBlocks(inc_entry, parsed); } tz.advToNextLine(); continue; } // Version else if (tz.checkNC("version")) { tz.advToNextLine(); continue; } // ZScript parsed.push_back({}); parsed.back().entry = entry; if (!parsed.back().parse(tz)) parsed.pop_back(); } // Set entry type if (etype_zscript && entry->getType() != etype_zscript) entry->setType(etype_zscript); }
bool Texture :: loadCubemapFromNames ( const char * fileNames ) { const char * seps = ",;"; Tokenizer tok ( fileNames, seps ); string names [6]; for ( int i = 0; i < 6; i++ ) { names [i] = tok.next (); if ( names [i].empty () ) return false; } return loadCubemap ( names [0].c_str (), names [1].c_str (), names [2].c_str (), names [3].c_str (), names [4].c_str (), names [5].c_str () ); }
void QSGShaderSourceBuilder::addDefinition(const QByteArray &definition) { if (definition.isEmpty()) return; Tokenizer tok; const char *input = m_source.constData(); tok.initialize(input); // First find #version, #extension's and "void main() { ... " const char *versionPos = 0; const char *extensionPos = 0; bool inSingleLineComment = false; bool inMultiLineComment = false; bool foundVersionStart = false; bool foundExtensionStart = false; Tokenizer::Token lt = Tokenizer::Token_Unspecified; Tokenizer::Token t = tok.next(); while (t != Tokenizer::Token_EOF) { // Handle comment blocks if (t == Tokenizer::Token_MultiLineCommentStart ) inMultiLineComment = true; if (t == Tokenizer::Token_MultiLineCommentEnd) inMultiLineComment = false; if (t == Tokenizer::Token_SingleLineComment) inSingleLineComment = true; if (t == Tokenizer::Token_NewLine && inSingleLineComment && !inMultiLineComment) inSingleLineComment = false; // Have we found #version, #extension or void main()? if (t == Tokenizer::Token_Version && !inSingleLineComment && !inMultiLineComment) foundVersionStart = true; if (t == Tokenizer::Token_Extension && !inSingleLineComment && !inMultiLineComment) foundExtensionStart = true; if (foundVersionStart && t == Tokenizer::Token_NewLine) { versionPos = tok.pos; foundVersionStart = false; } else if (foundExtensionStart && t == Tokenizer::Token_NewLine) { extensionPos = tok.pos; foundExtensionStart = false; } else if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) { if (qstrncmp("main", tok.identifier, 4) == 0) break; } // Scan to next token lt = t; t = tok.next(); } // Determine where to insert the definition. // If we found #extension directives, insert after last one, // else, if we found #version insert after #version // otherwise, insert at beginning. const char *insertionPos = extensionPos ? extensionPos : (versionPos ? versionPos : input); // Construct a new shader string, inserting the definition QByteArray newSource; newSource.reserve(m_source.size() + definition.size() + 9); newSource += QByteArray::fromRawData(input, insertionPos - input); newSource += QByteArrayLiteral("#define ") + definition + QByteArrayLiteral("\n"); newSource += QByteArray::fromRawData(insertionPos, m_source.size() - (insertionPos - input)); m_source = newSource; }
// ---------------------------------------------------------------------------- // ParseTreeNode::parse // // Parses formatted text data. Current valid formatting is: // (type) child = value; // (type) child = value1, value2, ...; // (type) child = { value1, value2, ... } // (type) child { grandchild = value; etc... } // (type) child : inherited { ... } // // All values are read as strings, but can be retrieved as string, int, bool // or float. // ---------------------------------------------------------------------------- bool ParseTreeNode::parse(Tokenizer& tz) { // Keep parsing until final } is reached (or end of file) string name, type; while (!tz.atEnd() && tz.current() != '}') { // Check for preprocessor stuff if (parser_ && tz.current()[0] == '#') { if (!parsePreprocessor(tz)) return false; tz.advToNextLine(); continue; } // If it's a special character (ie not a valid name), parsing fails if (tz.isSpecialCharacter(tz.current().text[0])) { logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text))); return false; } // So we have either a node or property name name = tz.current().text; type.Empty(); if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } // Check for type+name pair if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':') { type = name; name = tz.next().text; if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } } //Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text))); // Assignment if (tz.advIfNext('=', 2)) { if (!parseAssignment(tz, addChildPTN(name, type))) return false; } // Child node else if (tz.advIfNext('{', 2)) { // Parse child node if (!addChildPTN(name, type)->parse(tz)) return false; } // Child node (with no values/children) else if (tz.advIfNext(';', 2)) { // Add child node addChildPTN(name, type); continue; } // Child node + inheritance else if (tz.advIfNext(':', 2)) { // Check for opening brace if (tz.checkNext('{')) { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip { tz.adv(2); // Parse child node if (!child->parse(tz)) return false; } else if (tz.checkNext(';')) // Empty child node { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip ; tz.adv(2); continue; } else { logError(tz, S_FMT("Expecting \"{\" or \";\", got \"%s\"", CHR(tz.next().text))); return false; } } // Unexpected token else { logError(tz, S_FMT("Unexpected token \"%s\"", CHR(tz.next().text))); return false; } // Continue parsing tz.adv(); } // Success return true; }
// ---------------------------------------------------------------------------- // ParseTreeNode::parsePreprocessor // // Parses a preprocessor directive at [tz]'s current token // ---------------------------------------------------------------------------- bool ParseTreeNode::parsePreprocessor(Tokenizer& tz) { //Log::debug(S_FMT("Preprocessor %s", CHR(tz.current().text))); // #define if (tz.current() == "#define") parser_->define(tz.next().text); // #if(n)def else if (tz.current() == "#ifdef" || tz.current() == "#ifndef") { // Continue if condition succeeds bool test = true; if (tz.current() == "#ifndef") test = false; string define = tz.next().text; if (parser_->defined(define) == test) return true; // Failed condition, skip section int skip = 0; while (true) { auto& token = tz.next(); if (token == "#endif") skip--; else if (token == "#ifdef") skip++; else if (token == "#ifndef") skip++; // TODO: #else if (skip < 0) break; } } // #include else if (tz.current() == "#include") { // Include entry at the given path if we have an archive dir set if (archive_dir_) { // Get entry to include auto inc_path = tz.next().text; auto archive = archive_dir_->archive(); auto inc_entry = archive->entryAtPath(archive_dir_->getPath() + inc_path); if (!inc_entry) // Try absolute path inc_entry = archive->entryAtPath(inc_path); //Log::debug(S_FMT("Include %s", CHR(inc_path))); if (inc_entry) { // Save the current dir and set it to the included entry's dir auto orig_dir = archive_dir_; archive_dir_ = inc_entry->getParentDir(); // Parse text in the entry Tokenizer inc_tz; inc_tz.openMem(inc_entry->getMCData(), inc_entry->getName()); bool ok = parse(inc_tz); // Reset dir and abort if parsing failed archive_dir_ = orig_dir; if (!ok) return false; } else logError(tz, S_FMT("Include entry %s not found", CHR(inc_path))); } else tz.adv(); // Skip include path } // #endif (ignore) else if (tz.current() == "#endif") return true; // TODO: #else // Unrecognised else logError(tz, S_FMT("Unrecognised preprocessor directive \"%s\"", CHR(tz.current().text))); return true; }
* This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "tokenizer.hpp" #include <sstream> #include "catch.hpp" TEST_CASE( "Parsing Tokens", "[Tokenizer::parse]" ) { Tokenizer tok; std::stringstream test; test << "test_project {" << std::endl << "test=1," << std::endl << " meep = 39" << std::endl << "}" << std::endl; tok = Tokenizer(test); CHECK(tok.next() == "test_project"); Token token = tok.getToken(); CHECK_FALSE(token.getType() == ERROR); CHECK(token.getType() == PROJECT); CHECK(token.getValue() == "test_project"); }
// ----------------------------------------------------------------------------- // Parses ZMAPINFO-format definitions in [entry] // ----------------------------------------------------------------------------- bool MapInfo::parseZMapInfo(ArchiveEntry* entry) { Tokenizer tz; tz.setReadLowerCase(true); tz.openMem(entry->data(), entry->name()); while (!tz.atEnd()) { // Include if (tz.check("include")) { // Get entry at include path auto include_entry = entry->parent()->entryAtPath(tz.next().text); if (!include_entry) { Log::warning( "Warning - Parsing ZMapInfo \"{}\": Unable to include \"{}\" at line {}", entry->name(), tz.current().text, tz.lineNo()); } else if (!parseZMapInfo(include_entry)) return false; } // Map else if (tz.check("map") || tz.check("defaultmap") || tz.check("adddefaultmap")) { if (!parseZMap(tz, tz.current().text)) return false; } // DoomEdNums else if (tz.check("doomednums")) { if (!parseDoomEdNums(tz)) return false; } // Unknown block (skip it) else if (tz.check("{")) { Log::warning(2, "Warning - Parsing ZMapInfo \"{}\": Skipping {{}} block", entry->name()); tz.adv(); tz.skipSection("{", "}"); continue; } // Unknown else { Log::warning(2, R"(Warning - Parsing ZMapInfo "{}": Unknown token "{}")", entry->name(), tz.current().text); } tz.adv(); } Log::info(2, "Parsed ZMapInfo entry {} successfully", entry->name()); return true; } // ----------------------------------------------------------------------------- // Parses a ZMAPINFO map definition of [type] beginning at the current token in // tokenizer [tz] // ----------------------------------------------------------------------------- bool MapInfo::parseZMap(Tokenizer& tz, std::string_view type) { // TODO: Handle adddefaultmap auto map = default_map_; // Normal map, get lump/name/etc tz.adv(); if (type == "map") { // Entry name should be just after map keyword map.entry_name = tz.current().text; // Parse map name tz.adv(); if (tz.check("lookup")) { map.lookup_name = true; map.name = tz.next().text; } else { map.lookup_name = false; map.name = tz.current().text; } tz.adv(); } if (!tz.advIf("{")) { Log::error(R"(Error Parsing ZMapInfo: Expecting "{{", got "{}" at line {})", tz.current().text, tz.lineNo()); return false; } while (!tz.checkOrEnd("}")) { // Block (skip it) if (tz.advIf("{")) tz.skipSection("{", "}"); // LevelNum else if (tz.check("levelnum")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // Parse number // TODO: Checks tz.next().toInt(map.level_num); } // Sky1 else if (tz.check("sky1")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky1 = tz.next().text; // Scroll speed // TODO: Checks if (tz.advIfNext(",")) tz.next().toFloat(map.sky1_scroll_speed); } // Sky2 else if (tz.check("sky2")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky2 = tz.next().text; // Scroll speed // TODO: Checks if (tz.advIfNext(",")) tz.next().toFloat(map.sky2_scroll_speed); } // Skybox else if (tz.check("skybox")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; map.sky1 = tz.next().text; } // DoubleSky else if (tz.check("doublesky")) map.sky_double = true; // ForceNoSkyStretch else if (tz.check("forcenoskystretch")) map.sky_force_no_stretch = true; // SkyStretch else if (tz.check("skystretch")) map.sky_stretch = true; // Fade else if (tz.check("fade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; if (!strToCol(tz.next().text, map.fade)) return false; } // OutsideFog else if (tz.check("outsidefog")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; if (!strToCol(tz.next().text, map.fade_outside)) return false; } // EvenLighting else if (tz.check("evenlighting")) { map.lighting_wallshade_h = 0; map.lighting_wallshade_v = 0; } // SmoothLighting else if (tz.check("smoothlighting")) map.lighting_smooth = true; // VertWallShade else if (tz.check("vertwallshade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // TODO: Checks tz.next().toInt(map.lighting_wallshade_v); } // HorzWallShade else if (tz.check("horzwallshade")) { if (!checkEqualsToken(tz, "ZMapInfo")) return false; // TODO: Checks tz.next().toInt(map.lighting_wallshade_h); } // ForceFakeContrast else if (tz.check("forcefakecontrast")) map.force_fake_contrast = true; tz.adv(); }
QByteArray qsgShaderRewriter_insertZAttributes(const char *input, QSurfaceFormat::OpenGLContextProfile profile) { Tokenizer tok; tok.initialize(input); Tokenizer::Token lt = tok.next(); Tokenizer::Token t = tok.next(); // First find "void main() { ... " const char* voidPos = input; while (t != Tokenizer::Token_EOF) { if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) { if (qstrncmp("main", tok.identifier, 4) == 0) break; } voidPos = tok.pos - 4; lt = t; t = tok.next(); } QByteArray result; result.reserve(1024); result += QByteArray::fromRawData(input, voidPos - input); switch (profile) { case QSurfaceFormat::NoProfile: case QSurfaceFormat::CompatibilityProfile: result += QByteArrayLiteral("attribute highp float _qt_order;\n"); result += QByteArrayLiteral("uniform highp float _qt_zRange;\n"); break; case QSurfaceFormat::CoreProfile: result += QByteArrayLiteral("in float _qt_order;\n"); result += QByteArrayLiteral("uniform float _qt_zRange;\n"); break; } // Find first brace '{' while (t != Tokenizer::Token_EOF && t != Tokenizer::Token_OpenBrace) t = tok.next(); int braceDepth = 1; t = tok.next(); // Find matching brace and insert our code there... while (t != Tokenizer::Token_EOF) { switch (t) { case Tokenizer::Token_CloseBrace: braceDepth--; if (braceDepth == 0) { result += QByteArray::fromRawData(voidPos, tok.pos - 1 - voidPos); result += QByteArrayLiteral(" gl_Position.z = (gl_Position.z * _qt_zRange + _qt_order) * gl_Position.w;\n"); result += QByteArray(tok.pos - 1); return result; } break; case Tokenizer::Token_OpenBrace: ++braceDepth; break; default: break; } t = tok.next(); } return QByteArray(); }
// ----------------------------------------------------------------------------- // Parses an assignment operation at [tz]'s current token to [child] // ----------------------------------------------------------------------------- bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const { // Check type of assignment list char list_end = ';'; if (tz.current() == '{' && !tz.current().quoted_string) { list_end = '}'; tz.adv(); } // Parse until ; or } while (true) { auto& token = tz.current(); // Check for list end if (token == list_end && !token.quoted_string) break; // Setup value Property value; // Detect value type if (token.quoted_string) // Quoted string value = token.text; else if (token == "true") // Boolean (true) value = true; else if (token == "false") // Boolean (false) value = false; else if (token.isInteger()) // Integer value = token.asInt(); else if (token.isHex()) // Hex (0xXXXXXX) value = token.asInt(); else if (token.isFloat()) // Floating point value = token.asFloat(); else // Unknown, just treat as string value = token.text; // Add value child->values_.push_back(value); // Check for , if (tz.peek() == ',') tz.adv(); // Skip it else if (tz.peek() != list_end) { logError(tz, fmt::format(R"(Expected "," or "{}", got "{}")", list_end, tz.peek().text)); return false; } tz.adv(); } return true; } // ----------------------------------------------------------------------------- // Parses formatted text data. Current valid formatting is: // (type) child = value; // (type) child = value1, value2, ...; // (type) child = { value1, value2, ... } // (type) child { grandchild = value; etc... } // (type) child : inherited { ... } // // All values are read as strings, but can be retrieved as string, int, bool // or float. // ----------------------------------------------------------------------------- bool ParseTreeNode::parse(Tokenizer& tz) { // Keep parsing until final } is reached (or end of file) string name, type; while (!tz.atEnd() && tz.current() != '}') { // Check for preprocessor stuff if (parser_ && tz.current()[0] == '#') { if (!parsePreprocessor(tz)) return false; tz.advToNextLine(); continue; } // If it's a special character (ie not a valid name), parsing fails if (tz.isSpecialCharacter(tz.current().text[0])) { logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text)); return false; } // So we have either a node or property name name = tz.current().text; type.clear(); if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } // Check for type+name pair if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':') { type = name; name = tz.next().text; if (name.empty()) { logError(tz, "Unexpected empty string"); return false; } } // Log::debug(wxString::Format("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text))); // Assignment if (tz.advIfNext('=', 2)) { if (!parseAssignment(tz, addChildPTN(name, type))) return false; } // Child node else if (tz.advIfNext('{', 2)) { // Parse child node if (!addChildPTN(name, type)->parse(tz)) return false; } // Child node (with no values/children) else if (tz.advIfNext(';', 2)) { // Add child node addChildPTN(name, type); continue; } // Child node + inheritance else if (tz.advIfNext(':', 2)) { // Check for opening brace if (tz.checkNext('{')) { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip { tz.adv(2); // Parse child node if (!child->parse(tz)) return false; } else if (tz.checkNext(';')) // Empty child node { // Add child node auto child = addChildPTN(name, type); child->inherit_ = tz.current().text; // Skip ; tz.adv(2); continue; } else { logError(tz, fmt::format(R"(Expecting "{{" or ";", got "{}")", tz.next().text)); return false; } } // Unexpected token else { logError(tz, fmt::format("Unexpected token \"{}\"", tz.next().text)); return false; } // Continue parsing tz.adv(); } // Success return true; }