void skip_ahead_to_balanced_parentheses (int parentheses_level) {
while (TRUE) {
if (current_agent(lexeme).type==EOF_LEXEME) return;
if ((current_agent(lexeme).type==R_PAREN_LEXEME) &&
(parentheses_level==current_agent(current_file)->parentheses_level)) return;
get_lexeme();
}
}
void skip_ahead_to_balanced_parentheses (agent* thisAgent,
int parentheses_level) {
while (TRUE) {
if (thisAgent->lexeme.type==EOF_LEXEME) return;
if ((thisAgent->lexeme.type==R_PAREN_LEXEME) &&
(parentheses_level==thisAgent->current_file->parentheses_level)) return;
get_lexeme(thisAgent);
}
}
char RuleContext::get_rune_value(Token token) const {
#ifdef DEBUG
if (token != Token::RUNE)
assert(false);
#endif
std::string lex = get_lexeme(token);
std::size_t p = 1;
return _parse_rune(lex, p);
}
std::string RuleContext::get_string_value(Token token) const {
std::string lex = get_lexeme(token);
std::string value;
if (token == Token::STRING) {
for (std::size_t i = 1; i < lex.size() - 1; ++i)
value.push_back(_parse_rune(lex, i));
} else if (token == Token::R_STRING) {
for (std::size_t i = 1; i < lex.size() - 1; ++i)
value.push_back(lex[i]);
}
#ifdef DEBUG
else
assert(false);
#endif
return value;
}
void lex_unknown (agent* thisAgent) {
if(reading_from_top_level(thisAgent) && thisAgent->current_char == 0) {
}
else {
print (thisAgent, "Error: Unknown character encountered by lexer, code=%d\n",
thisAgent->current_char);
print (thisAgent, "File %s, line %lu, column %lu.\n", thisAgent->current_file->filename,
thisAgent->current_file->current_line,
thisAgent->current_file->current_column);
if (! reading_from_top_level(thisAgent)) {
//respond_to_load_errors (thisAgent);
if (thisAgent->load_errors_quit)
thisAgent->current_char = EOF;
}
}
get_next_char(thisAgent);
get_lexeme(thisAgent);
}
void lex_unknown (void) {
if(reading_from_top_level() && current_agent(current_char) == 0) {
}
else {
print ("Error: Unknown character encountered by lexer, code=%d\n",
current_agent(current_char));
print ("File %s, line %lu, column %lu.\n", current_agent(current_file)->filename,
current_agent(current_file)->current_line,
current_agent(current_file)->current_column);
if (! reading_from_top_level()) {
respond_to_load_errors ();
if (current_agent(load_errors_quit))
current_agent(current_char) = EOF_AS_CHAR;
}
}
get_next_char();
get_lexeme();
}
long RuleContext::get_int_value(Token token) const {
std::string lex = get_lexeme(token);
long value = 0;
if (token == Token::DEC) {
for (char c : lex)
value = value * 10 + (c - '0');
} else if (token == Token::OCTAL) {
for (char c : lex.substr(1))
value = value * 8 + (c - '0');
} else if (token == Token::HEXADEC) {
for (char c : lex.substr(2))
value = value * 16 + (c <= '9' ? c - '0' : c - 'a' + 10);
}
#ifdef DEBUG
else
assert(false);
#endif
return value;
}
double RuleContext::get_float_value(Token token) const {
#ifdef DEBUG
if (token != Token::FLOAT)
assert(false);
#endif
std::string lex = get_lexeme(token);
double value = 0;
std::size_t i = 0;
while (lex[i] != '.' and lex[i] != 'e' and lex[i] != 'E')
value = value * 10 + (lex[i++] - '0');
if (lex[i] == '.') {
double div = .1;
++i;
while (i < lex.size() and lex[i] != 'e' and lex[i] != 'E') {
value += (lex[i++] - '0') * div;
div *= .1;
}
}
if (i < lex.size() and (lex[i] == 'e' or lex[i] == 'E')) {
int exp = 0;
bool neg = false;
++i;
if (lex[i] == '-') {
neg = true;
++i;
} else if (lex[i] == '+') {
++i;
}
while (i < lex.size())
exp = exp * 10 + (lex[i++] - '0');
if (neg)
exp = -exp;
value = value * std::pow(10, exp);
}
return value;
}
Token * Lexical_Analyser::get_next_token() {
static bool update = true;
string lexeme, token_name;
NFA_State *state = dfa->get_start_state();
int last_accepting = -1;
Token * ret_token = NULL;
while (true) {
//reached end of file
if (*forward == '\0') {
//TODO handle error
return NULL;
}
//longest matched token
if (state ->is_accepting_state())
last_accepting = state->get_token_id();
char input_char = *forward;
state = state ->get_dfa_transition(input_char);
//check dead state
if (state == NULL) {
//reset lexeme and token name
lexeme = "";
token_name = "";
if (last_accepting == -1) {
//Exception
//skip the corrupted token
//recover
if (forward == lexeme_begin) {
forward++;
lexeme_begin++;
forward_counter++;
} else {
lexeme_begin = forward;
}
} else {
lexeme = get_lexeme(lexeme_begin, forward);
token_name = token_names->at(last_accepting);
if (token_name.compare("White Space") != 0) {
ret_token = new Token(lexeme, token_name);
} else {
//TODO fix this part
assert(last_accepting == 0);
}
lexeme_begin = forward;
}
last_accepting = -1;
state = dfa->get_start_state();
//lexeme_begin = forward;
lexeme_begin_buffer_num = forward_buffer_num;
lexeme_begin_counter = forward_counter;
update = false;
}
if (update) {
forward++; //update pointer
forward_counter++;
} else {
update = true;
}
//handle buffers
if (forward_counter == BUFFER_SIZE && forward_buffer_num == 1) {
//at end buffer one
fill_buffer(buffer2);
forward = buffer2;
forward_buffer_num = 2;
forward_counter = 0;
} else if (forward_counter == BUFFER_SIZE && forward_buffer_num == 2) {
//at end buffer two
fill_buffer(buffer1);
forward = buffer1;
forward_buffer_num = 1;
forward_counter = 0;
}
if (ret_token != NULL)
return ret_token;
}
return ret_token;
}
