// get to the next non-whitespace that equals tok
void match(char tok)
{
skipws();
if (tok != input[offset])
throw syntax_error(string("expected ") + tok, line, linepos);
incoffs();
char s[2];
s[0] = tok; s[1] = 0;
tokout->push_back(s);
}
// read a name
void readid()
{
string tok;
skipws();
if (offset >= input.length() || istok(input[offset])) throw syntax_error("expected identifier", line, linepos);
while (offset < input.length() && !isspace(input[offset]) && !istok(input[offset]))
{
tok += input[offset];
incoffs();
}
tokout->push_back(tok);
}
void Tokenizer::getTokens( const char * prog,tokenList & tokens) throw (Exception *) {
int cursor=0;
int line=1;
Token * obj;
do {
obj=readNextToken(prog,cursor,line,tokens);
if (obj!=NULL) {
Token * t=dynamic_cast<Token*>(obj);
if (t) t->setLine(line);
tokens.push_back(obj);
Token::addReference(obj);
}
} while (!dynamic_cast<TEndScript *>(obj));
}