int yylval_token (int symbol) {
int offset = scan_offset - yyleng;
char *lexeme = inserthash(&stringset, yytext);
yylval = new_astree (symbol, filename_stack.last_filenr,
scan_linenr, offset, lexeme);
addtokens(yylval);
add_hashstack( identstack, lexeme);
return symbol;
}
int stringarray::addtokens (const string& text, const bool preservequotedstrings)
{
return addtokens (text, " \n\r\t", preservequotedstrings ? "\"" : "");
}
