/** Return the next monitored token. * Test the token following the monitored token. * If following is another monitored token, save it * for the next invocation of nextToken (like a single * lookahead token) and return it then. * If following is unmonitored, nondiscarded (hidden) * channel token, add it to the monitored token. * * Note: EOF must be a monitored Token. */ RefToken TokenStreamHiddenTokenFilter::nextToken() { // handle an initial condition; don't want to get lookahead // token of this splitter until first call to nextToken if ( !LA(1) ) { consumeFirst(); } // we always consume hidden tokens after monitored, thus, // upon entry LA(1) is a monitored token. RefToken monitored = LA(1); // point to hidden tokens found during last invocation static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken); lastHiddenToken = nullToken; // Look for hidden tokens, hook them into list emanating // from the monitored tokens. consume(); RefToken p = monitored; // while hidden or discarded scarf tokens while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { if ( hideMask.member(LA(1)->getType()) ) { // attach the hidden token to the monitored in a chain // link forwards static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); // link backwards if (p != monitored) { //hidden cannot point to monitored tokens static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); } p = lastHiddenToken = LA(1); } consume(); } return monitored; }
void TokenStreamHiddenTokenFilter::consumeFirst() { consume(); // Handle situation where hidden or discarded tokens // appear first in input stream RefToken p; // while hidden or discarded scarf tokens while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { if ( hideMask.member(LA(1)->getType()) ) { if ( !p ) { p = LA(1); } else { static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link p = LA(1); } lastHiddenToken = p; if (!firstHidden) firstHidden = p; // record hidden token if first } consume(); } }
/** Return a ptr to the hidden token appearing immediately before * token t in the input stream. */ RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t) { return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); }
void CommonASTWithHiddenTokens::initialize(RefToken t) { CommonAST::initialize(t); hiddenBefore = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); hiddenAfter = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter(); }