/** Tell me how to create a token for use with imaginary token nodes.
 *  For example, there is probably no input symbol associated with imaginary
 *  token DECL, but you need to create it as a payload or whatever for
 *  the DECL node as in ^(DECL type ID).
 *
 *  This is a variant of createToken where the new token is derived from
 *  an actual real input token.  Typically this is for converting '{'
 *  tokens to BLOCK etc...  You'll see
 *
 *    r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
 *
 *  If you care what the token payload objects' type is, you should
 *  override this method and any other createToken variant.
 *
 * NB: this being C it is not so easy to extend the types of creaeteToken.
 *     We will have to see if anyone needs to do this and add any variants to
 *     this interface.
 */
static	pANTLR3_COMMON_TOKEN
createTokenFromToken	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_COMMON_TOKEN fromToken)
{
    pANTLR3_COMMON_TOKEN    newToken;

    newToken	= adaptor->tokenFactory->newToken(adaptor->tokenFactory);
    
    if	(newToken != NULL)
    {
	/* Create the text using our own string factory to avoid complicating
	 * commontoken.
	 */
	pANTLR3_STRING	text;

	newToken->toString  = fromToken->toString;
	text		    = fromToken->getText(fromToken);
	newToken->text	    = adaptor->strFactory->newPtr(adaptor->strFactory, text->chars, text->len);

	newToken->setLine		(newToken, fromToken->getLine(fromToken));
	newToken->setTokenIndex		(newToken, fromToken->getTokenIndex(fromToken));
	newToken->setCharPositionInLine	(newToken, fromToken->getCharPositionInLine(fromToken));
	newToken->setChannel		(newToken, fromToken->getChannel(fromToken));
	newToken->setType		(newToken, fromToken->getType(fromToken));
    }

    return  newToken;
}
static	void
dbgSetTokenBoundaries	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE t, pANTLR3_COMMON_TOKEN startToken, pANTLR3_COMMON_TOKEN stopToken)
{
	setTokenBoundaries(adaptor, t, startToken, stopToken);

	if	(t != NULL && startToken != NULL && stopToken != NULL)
	{
		adaptor->debugger->setTokenBoundaries(adaptor->debugger, t, startToken->getTokenIndex(startToken), stopToken->getTokenIndex(stopToken));
	}
}
static pANTLR3_STRING	    
toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
{
	if	(start != NULL && stop != NULL)
	{
		return	ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
	}
	else
	{
		return	NULL;
	}
}
Example #4
0
static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
{
    pANTLR3_STRING  text;
    pANTLR3_STRING  outtext;

    text    =	token->getText(token);
    
    if	(text == NULL)
    {
		return NULL;
    }

	if	(text->factory == NULL)
	{
		return text;		// This usally means it is the EOF token
	}

    /* A new empty string to assemble all the stuff in
     */
    outtext = text->factory->newRaw(text->factory);

    /* Now we use our handy dandy string utility to assemble the
     * the reporting string
     * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
     */
    outtext->append8(outtext, "[Index: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
    outtext->append8(outtext, " (Start: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
    outtext->append8(outtext, "-Stop: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
    outtext->append8(outtext, ") ='");
    outtext->appendS(outtext, text);
    outtext->append8(outtext, "', type<");
    outtext->addi   (outtext, token->type);
    outtext->append8(outtext, "> ");

    if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
    {
		outtext->append8(outtext, "(channel = ");
		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
		outtext->append8(outtext, ") ");
    }

    outtext->append8(outtext, "Line: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
    outtext->append8(outtext, " LinePos:");
    outtext->addi   (outtext, token->getCharPositionInLine(token));
    outtext->addc   (outtext, ']');

    return  outtext;
}
/** Track start/stop token for subtree root created for a rule.
 *  Only works with CommonTree nodes.  For rules that match nothing,
 *  seems like this will yield start=i and stop=i-1 in a nil node.
 *  Might be useful info so I'll not force to be i..i.
 */
static	void
setTokenBoundaries	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE t, pANTLR3_COMMON_TOKEN startToken, pANTLR3_COMMON_TOKEN stopToken)
{
	ANTLR3_MARKER   start;
	ANTLR3_MARKER   stop;

	pANTLR3_COMMON_TREE	    ct;

	if	(t == NULL)
	{
		return;
	}

	if	( startToken != NULL)
	{
		start = startToken->getTokenIndex(startToken);
	}
	else
	{
		start = 0;
	}

	if	( stopToken != NULL)
	{
		stop = stopToken->getTokenIndex(stopToken);
	}
	else
	{
		stop = 0;
	}

	ct	= (pANTLR3_COMMON_TREE)(t->super);

	ct->startIndex  = start;
	ct->stopIndex   = stop;

}