void TokenStreamHiddenTokenFilter::consumeFirst() { consume(); // Handle situation where hidden or discarded tokens // appear first in input stream RefToken p; // while hidden or discarded scarf tokens while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { if ( hideMask.member(LA(1)->getType()) ) { if ( !p ) { p = LA(1); } else { static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link p = LA(1); } lastHiddenToken = p; if (!firstHidden) firstHidden = p; // record hidden token if first } consume(); } }
/** Return the next monitored token. * Test the token following the monitored token. * If following is another monitored token, save it * for the next invocation of nextToken (like a single * lookahead token) and return it then. * If following is unmonitored, nondiscarded (hidden) * channel token, add it to the monitored token. * * Note: EOF must be a monitored Token. */ RefToken TokenStreamHiddenTokenFilter::nextToken() { // handle an initial condition; don't want to get lookahead // token of this splitter until first call to nextToken if ( !LA(1) ) { consumeFirst(); } // we always consume hidden tokens after monitored, thus, // upon entry LA(1) is a monitored token. RefToken monitored = LA(1); // point to hidden tokens found during last invocation static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken); lastHiddenToken = nullToken; // Look for hidden tokens, hook them into list emanating // from the monitored tokens. consume(); RefToken p = monitored; // while hidden or discarded scarf tokens while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { if ( hideMask.member(LA(1)->getType()) ) { // attach the hidden token to the monitored in a chain // link forwards static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); // link backwards if (p != monitored) { //hidden cannot point to monitored tokens static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); } p = lastHiddenToken = LA(1); } consume(); } return monitored; }
RefToken TokenStreamBasicFilter::nextToken() { RefToken tok = input->nextToken(); while ( tok && discardMask.member(tok->getType()) ) { tok = input->nextToken(); } return tok; }
void BinaryCompArithOpAST::initialize( RefToken t ) { BasicAST::initialize(t); switch (t->getType()) { case PTokenTypes::EQ: _opType = ByteCodeNS::EQ; break; case PTokenTypes::NE: _opType = ByteCodeNS::NE; break; case PTokenTypes::OP_LT: _opType = ByteCodeNS::LT; break; case PTokenTypes::LE: _opType = ByteCodeNS::LE; break; case PTokenTypes::GT: _opType = ByteCodeNS::GT; break; case PTokenTypes::GE: _opType = ByteCodeNS::GE; break; case PTokenTypes::PLUS: _opType = ByteCodeNS::PLUS; break; case PTokenTypes::MINUS: _opType = ByteCodeNS::MINUS; break; case PTokenTypes::MUL: _opType = ByteCodeNS::MUL; break; case PTokenTypes::DIV: _opType = ByteCodeNS::DIV; break; case PTokenTypes::MOD: _opType = ByteCodeNS::MOD; break; case PTokenTypes::EXP: _opType = ByteCodeNS::EXP; break; default: assert(false); } }
string BasePortugolParser::getTokenDescription(const RefToken& token) { string str; if(token->getType() == PortugolParserTokenTypes::T_IDENTIFICADOR) { str = "\""; str += token->getText(); str += "\""; } else if(isKeyword(token->getType())) { str = "a palavra-chave "; str += getTokenNames()[token->getType()]; } else if(token->getType() == PortugolParserTokenTypes::EOF_) { str = "fim de arquivo (EOF)"; } else { // str = getTokenNames()[token->getType()]; str = token->getText(); } return str; }
#include "antlr/String.hpp" #ifdef ANTLR_CXX_SUPPORTS_NAMESPACE namespace antlr { #endif ANTLR_USING_NAMESPACE(std) NoViableAltException::NoViableAltException(RefAST t) : RecognitionException("NoViableAlt","<AST>",-1,-1), token(0), node(t) { } NoViableAltException::NoViableAltException( RefToken t, const ANTLR_USE_NAMESPACE(std)string& fileName_ ) : RecognitionException("NoViableAlt",fileName_,t->getLine(),t->getColumn()), token(t), node(nullASTptr) { } ANTLR_USE_NAMESPACE(std)string NoViableAltException::getMessage() const { if (token) { if( token->getType() == Token::EOF_TYPE ) return string("unexpected end of file"); else if( token->getType() == Token::NULL_TREE_LOOKAHEAD ) return string("unexpected end of tree"); else return string("unexpected token: ")+token->getText();
void CommonAST::initialize(RefToken t) { setType(t->getType()); setText(t->getText()); }
CommonAST::CommonAST(RefToken t) : BaseAST(), ttype( t->getType() ), text( t->getText() ) { }
/** Return a ptr to the hidden token appearing immediately before * token t in the input stream. */ RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t) { return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); }
void CommonASTWithHiddenTokens::initialize(RefToken t) { CommonAST::initialize(t); hiddenBefore = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); hiddenAfter = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter(); }