virtual parse_result parse( string::const_iterator& i, string::const_iterator e, token& tok ) const { string::const_iterator j = i; while ( j < e && *j != '\n' ) ++j; tok = string(i, j); tok.type( tok_types::comment ); i = j; return done; }
virtual parse_result parse( string::const_iterator& i, string::const_iterator e, token& tok ) const { string::const_iterator j = i; if ( *j != '&' && *j != '+' ) return failed; ++j; while ( j < e && ( *j == '.' || *j == '-' || isalnum(*j) ) ) ++j; tok.assign(i, j); tok.type( tok_types::pn_lit ); i = j; return done; }
bool token::operator==(token compareToken) { if(tokenType == compareToken.type()) { return true; } else { return false; } }
void dotfile_visitor::format_node(abstract_node *node, const char *name, const token &tok) { if (!m_connectTo) { if (tok.type() == token_types::STRING_LITERAL) fprintf(m_file, "\tptr%p [label=\"[%s]\\n[%s]\\n\\\"%.*s\\\"\"];\n", node, name, token_types::names[tok.type()], tok.length() - 2, tok.text() + 1); else if (tok.text()) fprintf(m_file, "\tptr%p [label=\"[%s]\\n[%s]\\n%.*s\"];\n", node, name, token_types::names[tok.type()], tok.length(), tok.text()); else format_node(node, name); } }
virtual void validate( const token& t ) const { using namespace tok_types; switch ( t.type() ) { case name: case num_lit: case open_paren: case close_paren: case comma: case times: case assignment: case new_line: case semicolon: case comment: case string_lit: case transp_lit: case pn_lit: case def_assign: case logic_and: case logic_or: return; case regex_lit: case open_brace: case close_brace: case colon: // These can only work when msiril comments are disabled if ( !args.msiril_comments ) return; break; } throw runtime_error( make_string() << "Unknown token in input: " << t ); }
bool json_tree_builder::insert_to_array(const token &token_) { auto arr = std::static_pointer_cast< json_array >(_current); switch (token_.type()) { case T_COMMA: if (_accept_comma) { _accept_comma = false; _accept_end = false; _accept_value = true; return true; } else { return false; } break; case T_RBRACKET: if (_accept_end) { if (_parents.empty()) { // parsing successfully finished _finished = true; return true; } _current = _parents.top(); _parents.pop(); _accept_colon = false; _accept_comma = true; _accept_end = true; _accept_name = false; _accept_value = _current->jtype() == json_type::J_ARRAY; return true; } else { return false; } break; case T_LBRACE: case T_LBRACKET: if (_accept_value) { auto new_val = to_value(token_); _parents.push(_current); _current = new_val; arr->insert(new_val); _accept_colon = false; _accept_comma = false; _accept_end = true; _accept_name = true; _accept_value = new_val->jtype() == json_type::J_ARRAY; return true; } else { return false; } break; case T_STR: case T_NUM: case T_TRUE: case T_FALSE: case T_NULL: if (_accept_value) { arr->insert(to_value(token_)); _accept_comma = true; _accept_end = true; _accept_value = false; return true; } else { return false; } break; default: return false; // unexpected token } return false; // should not get here }
bool json_tree_builder::insert_to_object(const token &token_) { auto obj = std::static_pointer_cast< json_object >(_current); switch (token_.type()) { case T_STR: if (_accept_name) { if (obj->contains(token_.text())) throw std::logic_error("duplicit key: " + token_.text()); // duplicit key name obj->insert(token_.text()); _accept_colon = true; _accept_comma = false; _accept_name = false; _accept_value = false; _accept_end = false; return true; } else if (_accept_value) { auto str = to_value(token_); obj->insert(str); _accept_colon = false; _accept_comma = true; _accept_name = false; _accept_value = false; _accept_end = true; return true; } else { // unexpected token return false; } break; case T_COLON: if (_accept_colon) { _accept_colon = false; _accept_comma = false; _accept_end = false; _accept_name = false; _accept_value = true; return true; } else { return false; } break; case T_LBRACE: // new object if (_accept_value) { auto new_obj = to_value(token_); _parents.push(_current); _current = new_obj; obj->insert(new_obj); _accept_colon = false; _accept_comma = false; _accept_end = true; _accept_name = true; _accept_value = false; return true; } else { return false; } break; case T_COMMA: if (_accept_comma) { _accept_colon = false; _accept_comma = false; _accept_end = true; // not valid JSON _accept_name = true; _accept_value = false; return true; } else { return false; } break; case T_LBRACKET: // new array if (_accept_value) { auto new_arr = to_value(token_); _parents.push(_current); _current = new_arr; obj->insert(new_arr); _accept_colon = false; _accept_comma = false; _accept_end = true; _accept_name = false; _accept_value = true; return true; } else { return false; } break; case T_RBRACE: if (_accept_end) { if (_parents.empty()) { // parsing successfully finished _finished = true; return true; } _current = _parents.top(); _parents.pop(); _accept_colon = false; _accept_comma = true; _accept_end = true; _accept_name = true; _accept_value = _current->jtype() == json_type::J_ARRAY; return true; } else { return false; } break; case T_TRUE: case T_FALSE: case T_NULL: case T_NUM: if (_accept_value) { obj->insert(to_value(token_)); _accept_colon = false; _accept_comma = true; _accept_end = true; _accept_name = false; _accept_value = false; return true; } else { return false; } break; default: return false; // unexpected token } return false; // should not get here }
//this function finds the next token and outputs it token calcLex() { curToken.clearData(); char currentChar; while(1) //infinite loop, breaks through a return statement { //get the next char! currentChar = filestream.get(); //used to ignore whitespace as a token while( currentChar == ' ' || currentChar == '\t' || currentChar == '\n') { currentChar = filestream.get(); }; //if the current char is the end of a file, it returns the end of file symbol if(currentChar == EOF) { curToken.type(endOfFileSym); return curToken; } //this is the comment logic: if the next chars are /*... if(followingChars("/*", currentChar, curToken)) { //then while it isn't ending the line, or finding the end of comment sentinel... while(currentChar != '\n' && !followingChars("*/", currentChar, curToken)) { //move onto the next character currentChar = filestream.get(); } //move to the next character after the comment or line has ended //hop back up to the top of the while loop continue; } //if the chars ":=" are found, return the assignment symbol if(followingChars(":=", currentChar, curToken)) { curToken.type(assignSym); curToken.data(":="); return curToken; } //if the chars "read" are found, return the read symbol if(followingChars("read", currentChar, curToken)) { curToken.type(readSym); curToken.data("read"); return curToken; } //if the chars "write" are found, return the write symbol if(followingChars("write", currentChar, curToken)) { curToken.type(writeSym); curToken.data("write"); return curToken; } //if this is put above the followingChars, it adds a duplicate letter. curToken.appendChar(currentChar); //checks for identifier strings only starting with letters and underscores if( isalpha(currentChar) || (currentChar == '_') ) { while( isalnum(currentChar = filestream.get()) || (currentChar == '_')) { curToken.appendChar(currentChar); } //once the currentChar falls off the end of the identifier string, it must hop back to read the next one filestream.unget(); //returns the identifier int curToken.type(identifier); return curToken; } //if the current char is a number if(isdigit(currentChar)) { while ( isdigit(currentChar = filestream.get()) || currentChar == '.') { if (currentChar == '.') { curToken.appendChar(currentChar); currentChar = filestream.get(); if (!isdigit(currentChar)) { curToken.type(numConstError); return curToken; } while (isdigit(currentChar)) { curToken.appendChar(currentChar); currentChar = filestream.get(); } filestream.unget(); curToken.type(numConst); return curToken; } curToken.appendChar(currentChar); } filestream.unget(); curToken.type(numConst); return curToken; } if(currentChar == '+') { curToken.type(addOp); curToken.data("+"); return curToken; } if(currentChar == '-') { curToken.type(subOp); curToken.data("-"); return curToken; } if(currentChar == '*') { curToken.type(multOp); curToken.data("*"); return curToken; } if(currentChar == '/') { curToken.type(divOp); curToken.data("/"); return curToken; } if(currentChar == '(') { curToken.type(leftParen); curToken.data("("); return curToken; } if(currentChar == ')') { curToken.type(rightParen); curToken.data(")"); return curToken; } curToken.type(unknownError); return curToken; } curToken.type(unknownError); //this should never get here return curToken; }
std::string to_string(const token &t) { switch(t.type()) { case token::AND: return "&"; case token::AND_EQ: return "&="; case token::ASSIGN: return "="; case token::AUTO: return "auto"; case token::BREAK: return "break"; case token::CASE: return "case"; case token::CHAR: return "char"; case token::CMP: return "~"; case token::COLON: return ":"; case token::COMMA: return ","; case token::CONTINUE: return "continue"; case token::DIV: return "/"; case token::DIV_EQ: return "/="; case token::DO: return "do"; case token::DOT: return "."; case token::DOUBLECOLON: return "::"; case token::ELSE: return "else"; case token::EQ: return "=="; case token::FOR: return "for"; case token::FUNCTION: return "function"; case token::GE: return ">="; case token::GT: return ">"; case token::IF: return "if"; case token::INT: return "int"; case token::LBRACE: return "{"; case token::LBRACKET: return "["; case token::LE: return "<="; case token::LOGICAL_AND: return "&&"; case token::LOGICAL_OR: return "||"; case token::LPAREN: return "("; case token::LSHIFT: return "<<"; case token::LSHIFT_EQ: return "<<="; case token::LT: return "<"; case token::MINUS: return "-"; case token::MINUS_EQ: return "-="; case token::MOD: return "%"; case token::MOD_EQ: return "%="; case token::MUL: return "*"; case token::MUL_EQ: return "*="; case token::NE: return "!="; case token::NOT: return "!"; case token::OR: return "|"; case token::OR_EQ: return "|="; case token::PLUS: return "+"; case token::PLUS_EQ: return "+="; case token::RBRACE: return "}"; case token::RBRACKET: return "]"; case token::RETURN: return "return "; case token::RPAREN: return ")"; case token::RSHIFT: return ">>"; case token::RSHIFT_EQ: return ">>="; case token::SEMICOLON: return ";"; case token::SINGLEQUOTE: return "'"; case token::STRING: return "string"; case token::SWITCH: return "switch"; case token::WHILE: return "while"; case token::XOR: return "^"; case token::XOR_EQ: return "^="; case token::FINISHED: return "<EOF>"; case token::IDENTIFIER: case token::STRING_LITERAL: case token::CHARACTER: case token::INTEGER: return t.string_; default: return ""; } }
bool Foam::functionEntries::ifeqEntry::equalToken ( const token& t1, const token& t2 ) { const bool eqType = (t1.type() == t2.type()); switch (t1.type()) { case token::UNDEFINED: return eqType; case token::PUNCTUATION: return (eqType && t1.pToken() == t2.pToken()); case token::WORD: if (eqType) { return t1.wordToken() == t2.wordToken(); } else if (t2.isString()) { return t1.wordToken() == t2.stringToken(); } else { return false; } case token::STRING: case token::VARIABLE: case token::VERBATIMSTRING: if (eqType) { return t1.stringToken() == t2.stringToken(); } else if (t2.isWord()) { return t1.stringToken() == t2.wordToken(); } else { return false; } case token::LABEL: if (eqType) { return t1.labelToken() == t2.labelToken(); } else if (t2.isScalar()) { return t1.labelToken() == t2.scalarToken(); } else { return false; } case token::FLOAT_SCALAR: if (eqType) { return equal(t1.floatScalarToken(), t2.floatScalarToken()); } else if (t2.isScalar()) { return t1.scalarToken() == t2.scalarToken(); } else { return false; } case token::DOUBLE_SCALAR: if (eqType) { return equal(t1.doubleScalarToken(), t2.doubleScalarToken()); } else if (t2.isScalar()) { return t1.scalarToken() == t2.scalarToken(); } else { return false; } case token::LONG_DOUBLE_SCALAR: if (eqType) { return equal ( t1.longDoubleScalarToken(), t2.longDoubleScalarToken() ); } else if (t2.isScalar()) { return t1.scalarToken() == t2.scalarToken(); } else { return false; } case token::COMPOUND: return false; case token::ERROR: return eqType; } return false; }