/* consume_garbage() * * Advance until the current character is non-whitespace and non-comment. */ void consume_garbage() { consume_whitespace(); if(strategy == JsonParse::COMMENTS) { bool comment_found = false; do { comment_found = consume_comment(); consume_whitespace(); } while(comment_found); } }
/* getword: get next word or character from input */ int getword(char *word, int lim) { int c; char *w = word; while (isspace(c = getch())) ; /*------------------------------------*/ while (c == '\"' || c == '\'' || c == '/') { if (c == '\"') { c = consume_double_quote(); /* either \" or EOF */ } else if (c == '\'') { c = consume_single_quote(); /* either \' or EOF */ } else if (c == '/') { if ((c = getch()) == '*') c = consume_comment(); } if (isspace(c)) c = consume_space(); } /*------------------------------------*/ if (c != EOF) *w++ = c; if (!isbeginvalid(c)) { *w = '\0'; return c; } for (; --lim > 0; w++) { if (!isvalid(*w = getch())) /* isalnum has been relaced by isvalid */ { ungetch(*w); break; } } *w = '\0'; return word[0]; }
void top_level_consume_token(TokenizeContext &context) { if (is_identifier_first_letter(context.next())) { if (context.next() <= 'm') { // a through m if (try_to_consume_keyword(context, tok_And)) return; if (try_to_consume_keyword(context, tok_Break)) return; if (try_to_consume_keyword(context, tok_Case)) return; if (try_to_consume_keyword(context, tok_Continue)) return; if (try_to_consume_keyword(context, tok_Def)) return; if (try_to_consume_keyword(context, tok_Discard)) return; if (try_to_consume_keyword(context, tok_Else)) return; if (try_to_consume_keyword(context, tok_Elif)) return; if (try_to_consume_keyword(context, tok_False)) return; if (try_to_consume_keyword(context, tok_For)) return; if (try_to_consume_keyword(context, tok_If)) return; if (try_to_consume_keyword(context, tok_In)) return; if (try_to_consume_keyword(context, tok_Import)) return; if (try_to_consume_keyword(context, tok_Include)) return; if (try_to_consume_keyword(context, tok_Let)) return; } else { // n through z if (try_to_consume_keyword(context, tok_Namespace)) return; if (try_to_consume_keyword(context, tok_Not)) return; if (try_to_consume_keyword(context, tok_Nil)) return; if (try_to_consume_keyword(context, tok_Or)) return; if (try_to_consume_keyword(context, tok_Return)) return; if (try_to_consume_keyword(context, tok_State)) return; if (try_to_consume_keyword(context, tok_Struct)) return; if (try_to_consume_keyword(context, tok_Switch)) return; if (try_to_consume_keyword(context, tok_True)) return; if (try_to_consume_keyword(context, tok_Require)) return; if (try_to_consume_keyword(context, tok_RequireLocal)) return; if (try_to_consume_keyword(context, tok_Package)) return; if (try_to_consume_keyword(context, tok_Section)) return; if (try_to_consume_keyword(context, tok_While)) return; } consume_identifier(context); return; } if (is_whitespace(context.next())) { consume_whitespace(context); return; } if (context.next() == '0' && context.next(1) == 'x') { consume_hex_number(context); return; } if (match_number(context)) { consume_number(context); return; } // Check for specific characters switch(context.next()) { case '(': context.consume(tok_LParen, 1); return; case ')': context.consume(tok_RParen, 1); return; case '{': if (context.next(1) == '-') { consume_multiline_comment(context); return; } context.consume(tok_LBrace, 1); return; case '}': context.consume(tok_RBrace, 1); return; case '[': context.consume(tok_LSquare, 1); return; case ']': context.consume(tok_RSquare, 1); return; case ',': context.consume(tok_Comma, 1); return; case '@': context.consume(tok_At, 1); return; case '=': if (context.next(1) == '=') { context.consume(tok_DoubleEquals, 2); return; } else if (context.next(1) == '>') { context.consume(tok_FatArrow, 2); return; } context.consume(tok_Equals, 1); return; case '"': case '\'': consume_string_literal(context); return; case '\n': context.consume(tok_Newline, 1); return; case '.': if (context.next(1) == '.') { if (context.next(2) == '.') { context.consume(tok_Ellipsis, 3); } else { context.consume(tok_TwoDots, 2); } } else if (context.next(1) == '@') { context.consume(tok_DotAt, 2); } else { context.consume(tok_Dot, 1); } return; case '?': context.consume(tok_Question, 1); return; case '*': if (context.next(1) == '=') { context.consume(tok_StarEquals, 2); return; } if (context.next(1) == '*') { context.consume(tok_DoubleStar, 2); return; } context.consume(tok_Star, 1); return; case '/': if (context.next(1) == '=') { context.consume(tok_SlashEquals, 2); return; } if (context.next(1) == '/') { context.consume(tok_DoubleSlash, 2); return; } context.consume(tok_Slash, 1); return; case '!': if (context.next(1) == '=') { context.consume(tok_NotEquals, 2); return; } break; case ':': if (context.next(1) == '=') { context.consume(tok_ColonEquals, 2); return; } else if (context.next(1) == ':') { context.consume(tok_DoubleColon, 2); return; } else if (is_acceptable_inside_identifier(context.next(1))) { return consume_symbol(context); } context.consume(tok_Colon, 1); return; case '+': if (context.next(1) == '=') { context.consume(tok_PlusEquals, 2); } else { context.consume(tok_Plus, 1); } return; case '-': if (context.next(1) == '>') { context.consume(tok_RightArrow, 2); return; } if (context.next(1) == '-') return consume_comment(context); if (context.next(1) == '=') { context.consume(tok_MinusEquals, 2); return; } context.consume(tok_Minus, 1); return; case '<': if (context.next(1) == '<' && context.next(2) == '<') { consume_triple_quoted_string_literal(context); return; } if (context.next(1) == '=') { context.consume(tok_LThanEq, 2); return; } if (context.next(1) == '-') { context.consume(tok_LeftArrow, 2); return; } context.consume(tok_LThan, 1); return; case '>': if (context.next(1) == '=') context.consume(tok_GThanEq, 2); else context.consume(tok_GThan, 1); return; case '%': context.consume(tok_Percent, 1); return; case '|': if (context.next(1) == '|') context.consume(tok_DoubleVerticalBar, 2); else context.consume(tok_VerticalBar, 1); return; case '&': if (context.next(1) == '&') context.consume(tok_DoubleAmpersand, 2); else context.consume(tok_Ampersand, 1); return; case ';': context.consume(tok_Semicolon, 1); return; case '#': consume_color_literal(context); return; } // Fall through, consume the next letter as UNRECOGNIZED context.consume(tok_Unrecognized, 1); }
void top_level_consume_token(TokenizeContext &context) { if (is_identifier_first_letter(context.next())) { if (try_to_consume_keyword(context, TK_DEF)) return; if (try_to_consume_keyword(context, TK_TYPE)) return; if (try_to_consume_keyword(context, TK_BEGIN)) return; if (try_to_consume_keyword(context, TK_END)) return; if (try_to_consume_keyword(context, TK_IF)) return; if (try_to_consume_keyword(context, TK_ELSE)) return; if (try_to_consume_keyword(context, TK_ELIF)) return; if (try_to_consume_keyword(context, TK_FOR)) return; if (try_to_consume_keyword(context, TK_STATE)) return; if (try_to_consume_keyword(context, TK_IN)) return; if (try_to_consume_keyword(context, TK_TRUE)) return; if (try_to_consume_keyword(context, TK_FALSE)) return; // check 'do once' before 'do' if (try_to_consume_keyword(context, TK_DO_ONCE)) return; if (try_to_consume_keyword(context, TK_DO)) return; if (try_to_consume_keyword(context, TK_NAMESPACE)) return; if (try_to_consume_keyword(context, TK_INCLUDE)) return; if (try_to_consume_keyword(context, TK_IMPORT)) return; if (try_to_consume_keyword(context, TK_AND)) return; if (try_to_consume_keyword(context, TK_OR)) return; if (try_to_consume_keyword(context, TK_DISCARD)) return; if (try_to_consume_keyword(context, TK_NULL)) return; if (try_to_consume_keyword(context, TK_RETURN)) return; if (try_to_consume_keyword(context, TK_BREAK)) return; if (try_to_consume_keyword(context, TK_CONTINUE)) return; if (try_to_consume_keyword(context, TK_SWITCH)) return; if (try_to_consume_keyword(context, TK_CASE)) return; if (try_to_consume_keyword(context, TK_WHILE)) return; consume_identifier(context); return; } if (is_whitespace(context.next())) { consume_whitespace(context); return; } if (context.next() == '0' && context.next(1) == 'x') { consume_hex_number(context); return; } if (match_number(context)) { consume_number(context); return; } // Check for specific characters switch(context.next()) { case '(': context.consume(TK_LPAREN, 1); return; case ')': context.consume(TK_RPAREN, 1); return; case '{': if (context.next(1) == '-') { consume_multiline_comment(context); return; } context.consume(TK_LBRACE, 1); return; case '}': context.consume(TK_RBRACE, 1); return; case '[': context.consume(TK_LBRACKET, 1); return; case ']': context.consume(TK_RBRACKET, 1); return; case ',': context.consume(TK_COMMA, 1); return; case '@': if (context.next(1) == '.') { context.consume(TK_AT_DOT, 2); } else { context.consume(TK_AT_SIGN, 1); } return; case '=': if (context.next(1) == '=') { context.consume(TK_DOUBLE_EQUALS, 2); return; } context.consume(TK_EQUALS, 1); return; case '"': case '\'': consume_string_literal(context); return; case '\n': context.consume(TK_NEWLINE, 1); return; case '.': if (context.next(1) == '.') { if (context.next(2) == '.') { context.consume(TK_ELLIPSIS, 3); } else { context.consume(TK_TWO_DOTS, 2); } } else { context.consume(TK_DOT, 1); } return; case '?': context.consume(TK_QUESTION, 1); return; case '*': if (context.next(1) == '=') { context.consume(TK_STAR_EQUALS, 2); return; } context.consume(TK_STAR, 1); return; case '/': if (context.next(1) == '=') { context.consume(TK_SLASH_EQUALS, 2); return; } if (context.next(1) == '/') { context.consume(TK_DOUBLE_SLASH, 2); return; } context.consume(TK_SLASH, 1); return; case '!': if (context.next(1) == '=') { context.consume(TK_NOT_EQUALS, 2); return; } break; case ':': if (context.next(1) == '=') { context.consume(TK_COLON_EQUALS, 2); return; } else if (context.next(1) == ':') { context.consume(TK_DOUBLE_COLON, 2); return; } else if (is_identifier_first_letter(context.next(1))) { return consume_name(context); } context.consume(TK_COLON, 1); return; case '+': if (context.next(1) == '=') { context.consume(TK_PLUS_EQUALS, 2); } else { context.consume(TK_PLUS, 1); } return; case '-': if (context.next(1) == '>') { context.consume(TK_RIGHT_ARROW, 2); return; } if (context.next(1) == '-') return consume_comment(context); if (context.next(1) == '=') { context.consume(TK_MINUS_EQUALS, 2); return; } context.consume(TK_MINUS, 1); return; case '<': if (context.next(1) == '<' && context.next(2) == '<') { consume_triple_quoted_string_literal(context); return; } if (context.next(1) == '=') { context.consume(TK_LTHANEQ, 2); return; } if (context.next(1) == '-') { context.consume(TK_LEFT_ARROW, 2); return; } context.consume(TK_LTHAN, 1); return; case '>': if (context.next(1) == '=') { context.consume(TK_GTHANEQ, 2); return; } context.consume(TK_GTHAN, 1); return; case '%': context.consume(TK_PERCENT, 1); return; case '|': if (context.next(1) == '|') { context.consume(TK_DOUBLE_VERTICAL_BAR, 2); return; } break; case '&': if (context.next(1) == '&') { context.consume(TK_DOUBLE_AMPERSAND, 2); return; } context.consume(TK_AMPERSAND, 1); return; case ';': context.consume(TK_SEMICOLON, 1); return; case '#': consume_color_literal(context); return; } // Fall through, consume the next letter as UNRECOGNIZED context.consume(TK_UNRECOGNIZED, 1); }