int yylval_token (int symbol) { int offset = scan_offset - yyleng; char *lexeme = inserthash(&stringset, yytext); yylval = new_astree (symbol, filename_stack.last_filenr, scan_linenr, offset, lexeme); addtokens(yylval); add_hashstack( identstack, lexeme); return symbol; }
int stringarray::addtokens (const string& text, const bool preservequotedstrings) { return addtokens (text, " \n\r\t", preservequotedstrings ? "\"" : ""); }