void read_changes() { int i; scanf("%d\n", &nmergers); for (i = 0; i < nmergers; i++) { read_quoted_string((mergers[i][0])); read_quoted_string((mergers[i][1])); } }
static char * read_string (struct obstack *ob, FILE *infile, int star_if_braced) { char *stringbuf; int saw_paren = 0; int c; c = read_skip_spaces (infile); if (c == '(') { saw_paren = 1; c = read_skip_spaces (infile); } if (c == '"') stringbuf = read_quoted_string (ob, infile); else if (c == '{') { if (star_if_braced) obstack_1grow (ob, '*'); stringbuf = read_braced_string (ob, infile); } else fatal_with_file_and_line (infile, "expected `\"' or `{', found `%c'", c); if (saw_paren) { c = read_skip_spaces (infile); if (c != ')') fatal_expected_char (infile, ')', c); } return stringbuf; }
static char *next_token (const char **buffer) /* {{{ */ { const char *ptr = *buffer; char *ret; while (isspace ((int) (*ptr))) ptr++; if (ptr[0] == 0) return (NULL); else if (ptr[0] == '"') { ret = read_quoted_string (&ptr); if (ret != NULL) { *buffer = ptr; return (ret); } } ret = read_unquoted_word (&ptr); if (ret != NULL) *buffer = ptr; return (ret); } /* }}} char *next_token */
static const char *read_token_or_quoted_string(const char *s, char **token) { while (is_space_char(*s)) s++; if (*s == '"') return read_quoted_string(s, token); else return read_token(s, token); }
static struct scanner_token scan_next(struct scanner_input *I) { char c = 1; int i, num_invalid_chars = 0; struct scanner_token T; T.line = I->line_number; T.value = NULL; while (c != 0) { c = next_char(I); if (isalpha(c)) /* An identifier */ { char name[512]; /* Arbitrary limit to name length */ name[0] = c; i = 1; c = next_char(I); while (isalnum(c) || c == '_') { name[i++] = c; c = next_char(I); } name[i] = 0; put_back(I, c); if ( strcmp(name, "class") == 0 ) { T.type = TOK_CLASS; } else if ( strcmp(name, "end") == 0 ) { T.type = TOK_END; } else if ( strcmp(name, "return") == 0 ) { T.type = TOK_RETURN; } else { T.type = TOK_IDENTIFIER; T.value = GC_strdup(name); } return T; } /* FIXME: Need to check that there is only 1 '.' and 'e' in a number. * Also maybe make 'e' an actual operator instead because that would be cool */ if (isdigit(c)) { char num[512]; int ndots = 0; num[0] = c; i = 1; c = next_char(I); while (isdigit(c) || c == '.') { num[i++] = c; if (c == '.') ndots++; c = next_char(I); } num[i] = 0; put_back(I, c); if (ndots > 1) die("Invalid number: Too many decimal points", 1); T.type = (ndots == 0) ? TOK_INT : TOK_DECIMAL; /* It must be strdup'd to stop it being destroyed as soon as this loop returns */ T.value = GC_strdup(num); return T; } if (c == '=') { T.type = TOK_ASSIGN; /* Check it is =, not == */ c = next_char(I); if (c == '=') /* It must be comparison: == */ T.type = TOK_ISEQUAL; else put_back(I, c); return T; } if (c == '<') { T.type = TOK_LT; c = next_char(I); if (c == '=') T.type = TOK_LTE; else put_back(I, c); return T; } if (c == '>') { T.type = TOK_GT; c = next_char(I); if (c == '=') T.type = TOK_GTE; else put_back(I, c); return T; } if (c == '\"') { char *s = read_quoted_string(I); T.value = s; T.type = TOK_STRING; return T; } if (c == '#') { while (c != '\n') c = next_char(I); } if (isspace(c)) /* Ignore whitespace */ continue; switch (c) { case 0: T.type = TOK_EOF; return T; case '.': T.type = TOK_DOT; return T; case '+': T.type = TOK_ADD; return T; case '-': T.type = TOK_SUB; return T; case '*': T.type = TOK_MUL; return T; case '/': T.type = TOK_DIV; return T; case '(': T.type = TOK_OPEN_BRACKET; return T; case ')': T.type = TOK_CLOSED_BRACKET;return T; //case '{': T.type = TOK_STARTBLOCK; return T; //case '}': T.type = TOK_ENDBLOCK; return T; case ',': T.type = TOK_COMMA; return T; case ';': T.type = TOK_SEMICOLON; return T; default: if (num_invalid_chars++ < 1) fprintf(stderr, "Syntax error, line %lu: Invalid character \'%c\' (%d)\n", I->line_number, c, c); break; } } printf("EOF\n"); T.type = TOK_EOF; return T; }
void AsxTokenizer::move_next () { char c; while ((c = read_char ()) != -1) { if (in_data) { bool entirely_whitespace; current_token->set_type (TOKEN_DATA); current_token->set_value (read_data (c, entirely_whitespace)); current_token->set_entirely_whitespace (entirely_whitespace); return; } if (c == OP_OPEN_ELEMENT) { if (is_comment_open (c)) { read_comment (); continue; } current_token->set_type (TOKEN_OPEN_ELEMENT); current_token->set_value (NULL); in_data = false; return; } if (c == OP_CLOSE_ELEMENT) { current_token->set_type (TOKEN_CLOSE_ELEMENT); current_token->set_value (NULL); in_data = true; return; } if (c == OP_ASSIGNMENT) { current_token->set_type (TOKEN_ASSIGNMENT); current_token->set_value (NULL); return; } if (c == OP_SLASH) { current_token->set_type (TOKEN_SLASH); current_token->set_value (NULL); return; } if (g_ascii_isspace (c)) { current_token->set_type (TOKEN_WHITESPACE); current_token->set_value (NULL); return; } if (is_quote_char (c)) { current_token->set_type (TOKEN_QUOTED_STRING); current_token->set_value (read_quoted_string (c)); return; } if (is_name_char (c)) { current_token->set_type (TOKEN_NAME); current_token->set_value (read_name (c)); return; } } current_token->set_type (TOKEN_EOF); current_token->set_value (NULL); }
Token InputReader::get_next_token() { skip_white_spaces(); Token token, raw_token; std::string qstr; std::string value; get_raw_token(raw_token); switch(raw_token.type_) { case null_token: token.type_ = null_token; // this means no more tokens left break; case digit_token: case negative_token: float_t n_value; input_stream_.unget(); if(!read_number(n_value)) { std::cerr << "fatal error: failed while reading a number" << std::endl; token.type_ = error_token; } else { token.type_ = number_token; token.dvalue_ = n_value; } // if-else break; case object_begin_token: token.type_ = object_begin_token; structure_stack_.push(object_begin_token); break; case object_end_token: token.type_ = object_end_token; if(structure_stack_.top() != object_begin_token) { std::cerr << "fatal error: mismatched object encapsulators" << std::endl; token.type_ = error_token; } else { structure_stack_.pop(); } // if-else break; case array_begin_token: token.type_ = array_begin_token; structure_stack_.push(array_begin_token); break; case array_end_token: token.type_ = array_end_token; if(structure_stack_.top() != array_begin_token) { std::cerr << "fatal error: mismatched array encapsulators" << std::endl; token.type_ = error_token; } else { structure_stack_.pop(); } // if-else break; case string_begin_end_token: // will always be begin since // end will be removed while reading //the whole string earlier if(!read_quoted_string(qstr)) { std::cerr << "fatal error: premature EOF reached while reading string" << std::endl; token.type_ = error_token; } else { token.type_ = string_token; token.svalue_ = qstr; } // if-else break; case character_token: input_stream_.unget(); read_keyword(value); token.type_ = process_keyword_token(value); if(token.type_ == error_token) { std::cerr << "fatal error: unknown keyword '" << value << "'" << std::endl; } // if token.svalue_ = value; break; case assignment_token: token.type_ = assignment_token; break; case separator_token: token.type_ = separator_token; break; case comment_token: skip_comments(); token.type_ = comment_token; break; default: std::cerr << "fatal error: unknown token" << std::endl; token.type_ = error_token; } // switch return token; } // InputReader::get_next_token()