/**
* Helper for the query type determination.
*/
static bool isTokenText(pANTLR3_COMMON_TOKEN token, const std::string &expected)
{
pANTLR3_STRING text = token->getText(token);
if (text == NULL)
return false;
return base::tolower((const char*)text->chars) == expected;
}
void print_token_text(pANTLR3_COMMON_TOKEN tok)
{
if (tok->getType(tok) == T_EOS) {
printf("\n");
}
else {
printf("\%s", tok->getText(tok)->chars);
}
}
static void
dbgSetTokenBoundaries (pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE t, pANTLR3_COMMON_TOKEN startToken, pANTLR3_COMMON_TOKEN stopToken)
{
setTokenBoundaries(adaptor, t, startToken, stopToken);
if (t != NULL && startToken != NULL && stopToken != NULL)
{
adaptor->debugger->setTokenBoundaries(adaptor->debugger, t, startToken->getTokenIndex(startToken), stopToken->getTokenIndex(stopToken));
}
}
static pANTLR3_BASE_TREE
createTypeTokenText (pANTLR3_BASE_TREE_ADAPTOR adaptor, ANTLR3_UINT32 tokenType, pANTLR3_COMMON_TOKEN fromToken, pANTLR3_UINT8 text)
{
/* Create the new token
*/
fromToken = adaptor->createTokenFromToken(adaptor, fromToken);
/* Set the type of the new token to that supplied
*/
fromToken->setType(fromToken, tokenType);
/* Set the text of the token accordingly
*/
fromToken->setText8(fromToken, text);
/* Return a new node based upon this token
*/
return (pANTLR3_BASE_TREE)adaptor->create(adaptor, fromToken);
}
static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
{
pANTLR3_STRING text;
pANTLR3_STRING outtext;
text = token->getText(token);
if (text == NULL)
{
return NULL;
}
if (text->factory == NULL)
{
return text; // This usally means it is the EOF token
}
/* A new empty string to assemble all the stuff in
*/
outtext = text->factory->newRaw(text->factory);
/* Now we use our handy dandy string utility to assemble the
* the reporting string
* return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
*/
outtext->append8(outtext, "[Index: ");
outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
outtext->append8(outtext, " (Start: ");
outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
outtext->append8(outtext, "-Stop: ");
outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
outtext->append8(outtext, ") ='");
outtext->appendS(outtext, text);
outtext->append8(outtext, "', type<");
outtext->addi (outtext, token->type);
outtext->append8(outtext, "> ");
if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
{
outtext->append8(outtext, "(channel = ");
outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
outtext->append8(outtext, ") ");
}
outtext->append8(outtext, "Line: ");
outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
outtext->append8(outtext, " LinePos:");
outtext->addi (outtext, token->getCharPositionInLine(token));
outtext->addc (outtext, ']');
return outtext;
}
void printToken(pANTLR3_COMMON_TOKEN tok)
{
printf("[");
printf("@\%d,", (int)tok->getTokenIndex(tok));
printf("\%d:", (int)tok->getStartIndex(tok));
printf("\%d=", (int)tok->getStopIndex(tok));
printf("'\%s',", tok->getText(tok)->chars);
printf("<\%d>,", (int)tok->getType(tok));
if (tok->getChannel(tok) > ANTLR3_TOKEN_DEFAULT_CHANNEL) {
printf("channel=\%d,", (int)tok->getChannel(tok));
}
static pANTLR3_STRING
toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
{
if (start != NULL && stop != NULL)
{
return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
}
else
{
return NULL;
}
}
/** Tell me how to create a token for use with imaginary token nodes.
* For example, there is probably no input symbol associated with imaginary
* token DECL, but you need to create it as a payload or whatever for
* the DECL node as in ^(DECL type ID).
*
* This is a variant of createToken where the new token is derived from
* an actual real input token. Typically this is for converting '{'
* tokens to BLOCK etc... You'll see
*
* r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
*
* If you care what the token payload objects' type is, you should
* override this method and any other createToken variant.
*
* NB: this being C it is not so easy to extend the types of creaeteToken.
* We will have to see if anyone needs to do this and add any variants to
* this interface.
*/
static pANTLR3_COMMON_TOKEN
createTokenFromToken (pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_COMMON_TOKEN fromToken)
{
pANTLR3_COMMON_TOKEN newToken;
newToken = adaptor->tokenFactory->newToken(adaptor->tokenFactory);
if (newToken != NULL)
{
/* Create the text using our own string factory to avoid complicating
* commontoken.
*/
pANTLR3_STRING text;
newToken->toString = fromToken->toString;
text = fromToken->getText(fromToken);
newToken->text = adaptor->strFactory->newPtr(adaptor->strFactory, text->chars, text->len);
newToken->setLine (newToken, fromToken->getLine(fromToken));
newToken->setTokenIndex (newToken, fromToken->getTokenIndex(fromToken));
newToken->setCharPositionInLine (newToken, fromToken->getCharPositionInLine(fromToken));
newToken->setChannel (newToken, fromToken->getChannel(fromToken));
newToken->setType (newToken, fromToken->getType(fromToken));
}
return newToken;
}
static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
{
if (token->text != NULL)
{
return token->text;
}
if (token->type == ANTLR3_TOKEN_EOF)
{
token->setText8(token, (pANTLR3_UINT8)"<EOF>");
return token->text;
}
if (token->input != NULL)
{
return token->input->substr( token->input,
token->getStartIndex(token),
token->getStopIndex(token));
}
/* Nothing to return
*/
return NULL;
}
/** Track start/stop token for subtree root created for a rule.
* Only works with CommonTree nodes. For rules that match nothing,
* seems like this will yield start=i and stop=i-1 in a nil node.
* Might be useful info so I'll not force to be i..i.
*/
static void
setTokenBoundaries (pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE t, pANTLR3_COMMON_TOKEN startToken, pANTLR3_COMMON_TOKEN stopToken)
{
ANTLR3_MARKER start;
ANTLR3_MARKER stop;
pANTLR3_COMMON_TREE ct;
if (t == NULL)
{
return;
}
if ( startToken != NULL)
{
start = startToken->getTokenIndex(startToken);
}
else
{
start = 0;
}
if ( stopToken != NULL)
{
stop = stopToken->getTokenIndex(stopToken);
}
else
{
stop = 0;
}
ct = (pANTLR3_COMMON_TREE)(t->super);
ct->startIndex = start;
ct->stopIndex = stop;
}
ANTLR3_API void
antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
{
token->getText = getText;
token->setText = setText;
token->setText8 = setText8;
token->getType = getType;
token->setType = setType;
token->getLine = getLine;
token->setLine = setLine;
token->setLine = setLine;
token->getCharPositionInLine = getCharPositionInLine;
token->setCharPositionInLine = setCharPositionInLine;
token->getChannel = getChannel;
token->setChannel = setChannel;
token->getTokenIndex = getTokenIndex;
token->setTokenIndex = setTokenIndex;
token->getStartIndex = getStartIndex;
token->setStartIndex = setStartIndex;
token->getStopIndex = getStopIndex;
token->setStopIndex = setStopIndex;
token->toString = toString;
/* Set defaults
*/
token->setCharPositionInLine(token, -1);
token->custom = NULL;
token->freeCustom = NULL;
token->type = ANTLR3_TOKEN_INVALID;
token->text = NULL;
token->start = 0;
token->stop = 0;
token->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
token->line = 0;
token->index = 0;
token->input = NULL;
return;
}
string mcmt_state::token_text(pANTLR3_COMMON_TOKEN token) {
ANTLR3_MARKER start = token->getStartIndex(token);
size_t size = token->getStopIndex(token) - start + 1;
return string((const char*) start, size);
}
static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
{
switch (token->textState)
{
case ANTLR3_TEXT_STRING:
// Someone already created a string for this token, so we just
// use it.
//
return token->tokText.text;
break;
case ANTLR3_TEXT_CHARP:
// We had a straight text pointer installed, now we
// must convert it to a string. Note we have to do this here
// or otherwise setText8() will just install the same char*
//
if (token->strFactory != NULL)
{
token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
token->textState = ANTLR3_TEXT_STRING;
return token->tokText.text;
}
else
{
// We cannot do anything here
//
return NULL;
}
break;
default:
// EOF is a special case
//
if (token->type == ANTLR3_TOKEN_EOF)
{
token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
token->textState = ANTLR3_TEXT_STRING;
token->tokText.text->factory = token->strFactory;
return token->tokText.text;
}
// We had nothing installed in the token, create a new string
// from the input stream
//
if (token->input != NULL)
{
return token->input->substr( token->input,
token->getStartIndex(token),
token->stop
);
}
// Nothing to return, there is no input stream
//
return NULL;
break;
}
}
ANTLR3_UINT32 GetType(pANTLR3_COMMON_TOKEN token)
{
return token->getType(token);
}