/** Get the ith token from the current position 1..n where k=1 is the
* first symbol of lookahead.
*/
static pANTLR3_COMMON_TOKEN
tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
ANTLR3_INT32 i;
ANTLR3_INT32 n;
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
if (k < 0)
{
return LB(cts, -k);
}
if (cts->p == -1)
{
fillBuffer(cts);
}
// Here we used to check for k == 0 and return 0, but this seems
// a superfluous check to me. LT(k=0) is therefore just undefined
// and we won't waste the clock cycles on the check
//
if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index (ts->istream));
teof->setStopIndex (teof, ts->istream->index (ts->istream));
return teof;
}
i = cts->p;
n = 1;
/* Need to find k good tokens, skipping ones that are off channel
*/
while ( n < k)
{
/* Skip off-channel tokens */
i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
n++;
}
if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index(ts->istream));
teof->setStopIndex (teof, ts->istream->index(ts->istream));
return teof;
}
// Here the token must be in the input vector. Rather then incur
// function call penalty, we just return the pointer directly
// from the vector
//
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
//return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
}
/** Get the ith token from the current position 1..n where k=1 is the
* first symbol of lookahead.
*/
static pANTLR3_COMMON_TOKEN
tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
ANTLR3_INT32 i;
ANTLR3_INT32 n;
pANTLR3_COMMON_TOKEN_STREAM cts;
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
if (cts->p == -1)
{
fillBuffer(cts);
}
if (k == 0)
{
return NULL;
}
if (k < 0)
{
return LB(cts, -k);
}
if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index (ts->istream));
teof->setStopIndex (teof, ts->istream->index (ts->istream));
return teof;
}
i = cts->p;
n = 1;
/* Need to find k good tokens, skipping ones that are off channel
*/
while ( n < k)
{
/* Skip off-channel tokens */
i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
n++;
}
if ( (ANTLR3_UINT32) i > ts->istream->cachedSize)
{
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
teof->setStartIndex (teof, ts->istream->index(ts->istream));
teof->setStopIndex (teof, ts->istream->index(ts->istream));
return teof;
}
// Here the token must be in the input vector. Rather then incut
// function call penalty, we jsut return the pointer directly
// from the vector
//
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
//return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
}
/** Move the input pointer to the next incoming token. The stream
* must become active with LT(1) available. consume() simply
* moves the input pointer so that LT(1) points at the next
* input symbol. Consume at least one token.
*
* Walk past any token not on the channel the parser is listening to.
*/
static void
consume (pANTLR3_INT_STREAM is)
{
pANTLR3_COMMON_TOKEN_STREAM cts;
pANTLR3_TOKEN_STREAM ts;
ts = (pANTLR3_TOKEN_STREAM) is->super;
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
if ((ANTLR3_UINT32)cts->p < cts->tokens->count)
{
cts->p++;
cts->p = skipOffTokenChannels(cts, cts->p);
}
}
static void
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
ANTLR3_UINT32 index;
pANTLR3_COMMON_TOKEN tok;
ANTLR3_BOOLEAN discard;
void * channelI;
/* Start at index 0 of course
*/
index = 0;
/* Pick out the next token from the token source
* Remember we just get a pointer (reference if you like) here
* and so if we store it anywhere, we don't set any pointers to auto free it.
*/
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
{
discard = ANTLR3_FALSE; /* Assume we are not discarding */
/* I employ a bit of a trick, or perhaps hack here. Rather than
* store a pointer to a structure in the override map and discard set
* we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
* we can distinguish "not being there" from "being channel or type 0"
*/
if (tokenStream->discardSet != NULL
&& tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
{
discard = ANTLR3_TRUE;
}
else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
&& tok->getChannel(tok) != tokenStream->channel
)
{
discard = ANTLR3_TRUE;
}
else if (tokenStream->channelOverrides != NULL)
{
/* See if this type is in the override map
*/
channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
if (channelI != NULL)
{
/* Override found
*/
tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
}
}
/* If not discarding it, add it to the list at the current index
*/
if (discard == ANTLR3_FALSE)
{
/* Add it, indicating that we will delete it and the table should not
*/
tok->setTokenIndex(tok, index);
tokenStream->p++;
tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
index++;
}
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
}
/* Cache the size so we don't keep doing indirect method calls. We do this as
* early as possible so that anything after this may utilize the cached value.
*/
tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
/* Set the consume pointer to the first token that is on our channel
*/
tokenStream->p = 0;
tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
}