ANTLR_USE_NAMESPACE(antlr)RefToken LexTokenStream::nextToken()
{
ANTLR_USE_NAMESPACE(antlr)RefToken ret;
if (!reachedEOF)
{
int type;
currentLexer = this;
while ((type = yylex()) == -1) // Token::SKIP
;
if (type != 0)
{
if( type != JavaTokenTypes::STRING_LITERAL )
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,yytext));
else
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,string_buf));
// ANTLR_USE_NAMESPACE(std)cout << "Read token " << type << " [" << ret->getText() << "]" << ANTLR_USE_NAMESPACE(std)endl;
}
else
reachedEOF = true;
}
if (reachedEOF)
{
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(1,"EOF"));
// cout << "EOF reached" << endl;
}
ret->setLine(line);
return ret;
}
// Resolve build-in operations for non-map types, e.g.: 'str'.len()
packToken TypeSpecificFunction(const packToken& p_left, const packToken& p_right, evaluationData* data) {
if (p_left->type == MAP) throw Operation::Reject();
TokenMap& attr_map = calculator::type_attribute_map()[p_left->type];
std::string& key = p_right.asString();
packToken* attr = attr_map.find(key);
if (attr) {
// Note: If attr is a function, it will receive have
// scope["this"] == source, so it can make changes on this object.
// Or just read some information for example: its length.
return RefToken(key, (*attr), p_left);
} else {
throw undefined_operation(data->op, p_left, p_right);
}
}
RefToken TokenStreamRewriteEngine::nextToken( void )
{
RefTokenWithIndex t;
// suck tokens until end of stream or we find a non-discarded token
do {
t = RefTokenWithIndex(stream.nextToken());
if ( t )
{
t->setIndex(index); // what is t's index in list?
if ( t->getType() != Token::EOF_TYPE ) {
tokens.push_back(t); // track all tokens except EOF
}
index++; // move to next position
}
} while ( t && discardMask.member(t->getType()) );
return RefToken(t);
}
RefToken CommonToken::factory()
{
return RefToken(new CommonToken);
}
RefToken CommonHiddenStreamToken::factory()
{
return RefToken(new CommonHiddenStreamToken);
}
namespace antlr {
#endif
// The below initialization ICED AIX Visualage CC
//ANTLR_API RefToken Token::badToken(new Token(Token::INVALID_TYPE, "<no text>"));
// this seemed to work
// Create "static reference counters", i.e. don't do any reference counting
// as these objects are supposed to exist "forever".
ANTLR_API RefToken Token::badToken =
RefToken(new Token(Token::INVALID_TYPE, "<no text>"), true);
ANTLR_API RefToken Token::eofToken =
RefToken(new Token(Token::EOF_TYPE, "EOF"), true);
Token::Token() : type(INVALID_TYPE)
{
}
Token::Token(int t) : type(t)
{
}
Token::Token(int t, const string& txt)
: type(t)
{
type=t;
setText(txt);
}
int Token::getColumn() const
{
return 0;
}
int Token::getLine() const
{
return 0;
}
string Token::getText() const
{
return "<no text>";
}
int Token::getType() const
{
return type;
}
void Token::setColumn(int c)
{}
void Token::setLine(int l)
{}
void Token::setText(const string& t)
{}
void Token::setType(int t)
{
type=t;
}
string Token::toString() const
{
return "[\""+getText()+"\",<"+type+">]";
}
Token::~Token()
{}
ANTLR_API RefToken nullToken;
#ifndef NO_STATIC_CONSTS
const int Token::MIN_USER_TYPE;
const int Token::NULL_TREE_LOOKAHEAD;
const int Token::INVALID_TYPE;
const int Token::EOF_TYPE;
const int Token::SKIP;
#endif
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
