/** Tell me how to create a token for use with imaginary token nodes.
 *  For example, there is probably no input symbol associated with imaginary
 *  token DECL, but you need to create it as a payload or whatever for
 *  the DECL node as in ^(DECL type ID).
 *
 *  This is a variant of createToken where the new token is derived from
 *  an actual real input token.  Typically this is for converting '{'
 *  tokens to BLOCK etc...  You'll see
 *
 *    r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
 *
 *  If you care what the token payload objects' type is, you should
 *  override this method and any other createToken variant.
 *
 * NB: this being C it is not so easy to extend the types of creaeteToken.
 *     We will have to see if anyone needs to do this and add any variants to
 *     this interface.
 */
static	pANTLR3_COMMON_TOKEN
createTokenFromToken	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_COMMON_TOKEN fromToken)
{
    pANTLR3_COMMON_TOKEN    newToken;

    newToken	= adaptor->tokenFactory->newToken(adaptor->tokenFactory);
    
    if	(newToken != NULL)
    {
	/* Create the text using our own string factory to avoid complicating
	 * commontoken.
	 */
	pANTLR3_STRING	text;

	newToken->toString  = fromToken->toString;
	text		    = fromToken->getText(fromToken);
	newToken->text	    = adaptor->strFactory->newPtr(adaptor->strFactory, text->chars, text->len);

	newToken->setLine		(newToken, fromToken->getLine(fromToken));
	newToken->setTokenIndex		(newToken, fromToken->getTokenIndex(fromToken));
	newToken->setCharPositionInLine	(newToken, fromToken->getCharPositionInLine(fromToken));
	newToken->setChannel		(newToken, fromToken->getChannel(fromToken));
	newToken->setType		(newToken, fromToken->getType(fromToken));
    }

    return  newToken;
}
/**
*	Helper for the query type determination.
*/
static bool isTokenText(pANTLR3_COMMON_TOKEN token, const std::string &expected)
{
  pANTLR3_STRING text = token->getText(token);
  if (text == NULL)
    return false;

  return base::tolower((const char*)text->chars) == expected;
}
Exemple #3
0
static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
{
    pANTLR3_STRING  text;
    pANTLR3_STRING  outtext;

    text    =	token->getText(token);
    
    if	(text == NULL)
    {
		return NULL;
    }

	if	(text->factory == NULL)
	{
		return text;		// This usally means it is the EOF token
	}

    /* A new empty string to assemble all the stuff in
     */
    outtext = text->factory->newRaw(text->factory);

    /* Now we use our handy dandy string utility to assemble the
     * the reporting string
     * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
     */
    outtext->append8(outtext, "[Index: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
    outtext->append8(outtext, " (Start: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
    outtext->append8(outtext, "-Stop: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
    outtext->append8(outtext, ") ='");
    outtext->appendS(outtext, text);
    outtext->append8(outtext, "', type<");
    outtext->addi   (outtext, token->type);
    outtext->append8(outtext, "> ");

    if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
    {
		outtext->append8(outtext, "(channel = ");
		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
		outtext->append8(outtext, ") ");
    }

    outtext->append8(outtext, "Line: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
    outtext->append8(outtext, " LinePos:");
    outtext->addi   (outtext, token->getCharPositionInLine(token));
    outtext->addc   (outtext, ']');

    return  outtext;
}