/* get_next_token()
*
* Return the next non-whitespace character. If the end of the input is reached,
* flag an error and return 0.
*/
char get_next_token() {
consume_garbage();
if (i == str.size())
return fail("unexpected end of input", 0);
return str[i++];
}
token lexer::next()
{
consume_garbage();
boost::optional<token> tmp_t;
tmp_t = g.try_match(str, i, line);
if(!tmp_t)
{
std::stringstream s;
s << "Can not match to tokens: \"" << str.substr(i, 10) << '"' << std::endl;
throw std::runtime_error(s.str());
}
token t = tmp_t.get();
for(uint j = 0; j < t.length; j++)
if(str[i+j] == '\n')
line++;
i += t.length;
if(g.fetch_symbol(t.type)->type() == s_ignored)
return next();
return t;
}
