void skip_ahead_to_balanced_parentheses (int parentheses_level) { while (TRUE) { if (current_agent(lexeme).type==EOF_LEXEME) return; if ((current_agent(lexeme).type==R_PAREN_LEXEME) && (parentheses_level==current_agent(current_file)->parentheses_level)) return; get_lexeme(); } }
void skip_ahead_to_balanced_parentheses (agent* thisAgent, int parentheses_level) { while (TRUE) { if (thisAgent->lexeme.type==EOF_LEXEME) return; if ((thisAgent->lexeme.type==R_PAREN_LEXEME) && (parentheses_level==thisAgent->current_file->parentheses_level)) return; get_lexeme(thisAgent); } }
char RuleContext::get_rune_value(Token token) const { #ifdef DEBUG if (token != Token::RUNE) assert(false); #endif std::string lex = get_lexeme(token); std::size_t p = 1; return _parse_rune(lex, p); }
std::string RuleContext::get_string_value(Token token) const { std::string lex = get_lexeme(token); std::string value; if (token == Token::STRING) { for (std::size_t i = 1; i < lex.size() - 1; ++i) value.push_back(_parse_rune(lex, i)); } else if (token == Token::R_STRING) { for (std::size_t i = 1; i < lex.size() - 1; ++i) value.push_back(lex[i]); } #ifdef DEBUG else assert(false); #endif return value; }
void lex_unknown (agent* thisAgent) { if(reading_from_top_level(thisAgent) && thisAgent->current_char == 0) { } else { print (thisAgent, "Error: Unknown character encountered by lexer, code=%d\n", thisAgent->current_char); print (thisAgent, "File %s, line %lu, column %lu.\n", thisAgent->current_file->filename, thisAgent->current_file->current_line, thisAgent->current_file->current_column); if (! reading_from_top_level(thisAgent)) { //respond_to_load_errors (thisAgent); if (thisAgent->load_errors_quit) thisAgent->current_char = EOF; } } get_next_char(thisAgent); get_lexeme(thisAgent); }
void lex_unknown (void) { if(reading_from_top_level() && current_agent(current_char) == 0) { } else { print ("Error: Unknown character encountered by lexer, code=%d\n", current_agent(current_char)); print ("File %s, line %lu, column %lu.\n", current_agent(current_file)->filename, current_agent(current_file)->current_line, current_agent(current_file)->current_column); if (! reading_from_top_level()) { respond_to_load_errors (); if (current_agent(load_errors_quit)) current_agent(current_char) = EOF_AS_CHAR; } } get_next_char(); get_lexeme(); }
long RuleContext::get_int_value(Token token) const { std::string lex = get_lexeme(token); long value = 0; if (token == Token::DEC) { for (char c : lex) value = value * 10 + (c - '0'); } else if (token == Token::OCTAL) { for (char c : lex.substr(1)) value = value * 8 + (c - '0'); } else if (token == Token::HEXADEC) { for (char c : lex.substr(2)) value = value * 16 + (c <= '9' ? c - '0' : c - 'a' + 10); } #ifdef DEBUG else assert(false); #endif return value; }
double RuleContext::get_float_value(Token token) const { #ifdef DEBUG if (token != Token::FLOAT) assert(false); #endif std::string lex = get_lexeme(token); double value = 0; std::size_t i = 0; while (lex[i] != '.' and lex[i] != 'e' and lex[i] != 'E') value = value * 10 + (lex[i++] - '0'); if (lex[i] == '.') { double div = .1; ++i; while (i < lex.size() and lex[i] != 'e' and lex[i] != 'E') { value += (lex[i++] - '0') * div; div *= .1; } } if (i < lex.size() and (lex[i] == 'e' or lex[i] == 'E')) { int exp = 0; bool neg = false; ++i; if (lex[i] == '-') { neg = true; ++i; } else if (lex[i] == '+') { ++i; } while (i < lex.size()) exp = exp * 10 + (lex[i++] - '0'); if (neg) exp = -exp; value = value * std::pow(10, exp); } return value; }
Token * Lexical_Analyser::get_next_token() { static bool update = true; string lexeme, token_name; NFA_State *state = dfa->get_start_state(); int last_accepting = -1; Token * ret_token = NULL; while (true) { //reached end of file if (*forward == '\0') { //TODO handle error return NULL; } //longest matched token if (state ->is_accepting_state()) last_accepting = state->get_token_id(); char input_char = *forward; state = state ->get_dfa_transition(input_char); //check dead state if (state == NULL) { //reset lexeme and token name lexeme = ""; token_name = ""; if (last_accepting == -1) { //Exception //skip the corrupted token //recover if (forward == lexeme_begin) { forward++; lexeme_begin++; forward_counter++; } else { lexeme_begin = forward; } } else { lexeme = get_lexeme(lexeme_begin, forward); token_name = token_names->at(last_accepting); if (token_name.compare("White Space") != 0) { ret_token = new Token(lexeme, token_name); } else { //TODO fix this part assert(last_accepting == 0); } lexeme_begin = forward; } last_accepting = -1; state = dfa->get_start_state(); //lexeme_begin = forward; lexeme_begin_buffer_num = forward_buffer_num; lexeme_begin_counter = forward_counter; update = false; } if (update) { forward++; //update pointer forward_counter++; } else { update = true; } //handle buffers if (forward_counter == BUFFER_SIZE && forward_buffer_num == 1) { //at end buffer one fill_buffer(buffer2); forward = buffer2; forward_buffer_num = 2; forward_counter = 0; } else if (forward_counter == BUFFER_SIZE && forward_buffer_num == 2) { //at end buffer two fill_buffer(buffer1); forward = buffer1; forward_buffer_num = 1; forward_counter = 0; } if (ret_token != NULL) return ret_token; } return ret_token; }