Ejemplo n.º 1
0
void initParser(pANTLR3_INPUT_STREAM* input,pJpVocabularyLexer* lxr,
                pANTLR3_COMMON_TOKEN_STREAM* tstream,
                pJpVocabularyParser* psr) {
    *lxr = JpVocabularyLexerNew(*input); // CLexerNew is generated by ANTLR

    if (*lxr == NULL) {
        ANTLR3_FPRINTF(stderr,
                       "Unable to create the lexer due to malloc() failure1\n");
        exit(ANTLR3_ERR_NOMEM);
    }

    *tstream = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT,
               TOKENSOURCE((*lxr)));

    if (*tstream == NULL) {
        ANTLR3_FPRINTF(stderr,
                       "Out of memory trying to allocate token stream\n");
        exit(ANTLR3_ERR_NOMEM);
    }

    *psr = JpVocabularyParserNew(*tstream); // CParserNew is generated by ANTLR3

    if (*psr == NULL) {
        ANTLR3_FPRINTF(stderr, "Out of memory trying to allocate parser\n");
        exit(ANTLR3_ERR_NOMEM);
    }
}
Ejemplo n.º 2
0
/**
 * \brief
 * Prints out the message in all the exceptions in the supplied chain.
 * 
 * \param[in] ex
 * Pointer to the exception structure to print.
 * 
 * \remarks
 * You may wish to override this function by installing a pointer to a new function
 * in the base recognizer context structure.
 * 
 * \see
 * ANTLR3_BASE_RECOGNIZER
 */
static void
antlr3ExceptionPrint(pANTLR3_EXCEPTION ex)
{
    /* Ensure valid pointer
     */
    while   (ex != NULL)
    {
	/* Number if no message, else the message
	 */
	if  (ex->message == NULL)
	{
	    ANTLR3_FPRINTF(stderr, "ANTLR3_EXCEPTION number %d (%08X).\n", ex->type, ex->type);
	}
	else
	{
	    ANTLR3_FPRINTF(stderr, "ANTLR3_EXCEPTION: %s\n", (char *)(ex->message));
	}

	/* Move to next in the chain (if any)
	 */
	ex = ex->nextException;
    }

    return;
}
Ejemplo n.º 3
0
static void mTokens	    (pANTLR3_LEXER lexer)
{
    if	(lexer)	    // Fool compiler, avoid pragmas
    {
		ANTLR3_FPRINTF(stderr, "lexer->mTokens(): Error: No lexer rules were added to the lexer yet!\n");
    }
}
Ejemplo n.º 4
0
short parseJpVoc(const char* str,VocInfo vi) {
    pANTLR3_INPUT_STREAM input;
    pJpVocabularyLexer lxr;
    pANTLR3_COMMON_TOKEN_STREAM tstream;
    pJpVocabularyParser psr;

    ANTLR3_FPRINTF(stdout,"parseJpVoc:%s\n",str);
    input = antlr3StringStreamNew((pANTLR3_UINT8)str, ANTLR3_ENC_UTF8,
                                  strlen(str),(pANTLR3_UINT8)"jpVocabu");

    initParser(&input,&lxr,&tstream,&psr);
    psr->vocabulary(psr,vi.pronun,
                    vi.writing,vi.partOfSpeech,vi.expl);

    ANTLR3_FPRINTF(stdout,"parseJpVoc:pronun:%s\twriting:%s\tpartOfSpeech:%s\texpl:%s\n",
                   vi.pronun,vi.writing,vi.partOfSpeech,vi.expl);
    cleanParser(&input,&lxr,&tstream,&psr);
    return 0;
}
Ejemplo n.º 5
0
ConversionResult ConvertUTF16toUTF32 (
	const UTF16** sourceStart, const UTF16* sourceEnd, 
	UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
    ConversionResult result = conversionOK;
    const UTF16* source = *sourceStart;
    UTF32* target = *targetStart;
    UTF32 ch, ch2;
    while (source < sourceEnd) {
	const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
	ch = *source++;
	/* If we have a surrogate pair, convert to UTF32 first. */
	if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
	    /* If the 16 bits following the high surrogate are in the source buffer... */
	    if (source < sourceEnd) {
		ch2 = *source;
		/* If it's a low surrogate, convert to UTF32. */
		if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
		    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
			+ (ch2 - UNI_SUR_LOW_START) + halfBase;
		    ++source;
		} else if (flags == strictConversion) { /* it's an unpaired high surrogate */
		    --source; /* return to the illegal value itself */
		    result = sourceIllegal;
		    break;
		}
	    } else { /* We don't have the 16 bits following the high surrogate. */
		--source; /* return to the high surrogate */
		result = sourceExhausted;
		break;
	    }
	} else if (flags == strictConversion) {
	    /* UTF-16 surrogate values are illegal in UTF-32 */
	    if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
		--source; /* return to the illegal value itself */
		result = sourceIllegal;
		break;
	    }
	}
	if (target >= targetEnd) {
	    source = oldSource; /* Back up source pointer! */
	    result = targetExhausted; break;
	}
	*target++ = ch;
    }
    *sourceStart = source;
    *targetStart = target;
#ifdef CVTUTF_DEBUG
if (result == sourceIllegal) {
    ANTLR3_FPRINTF(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
    fflush(stderr);
}
#endif
    return result;
}
Ejemplo n.º 6
0
// Main entry point for this example
//
int ANTLR3_CDECL
main(int argc, char *argv[]) {
	pANTLR3_UINT8 fName;
	pANTLR3_INPUT_STREAM input;
	pJpVocabularyLexer lxr;
	pANTLR3_COMMON_TOKEN_STREAM tstream;
	pJpVocabularyParser psr;

	if (argc < 2 || argv[1] == NULL) {
		//fName = (pANTLR3_UINT8) "./input"; // Note in VS2005 debug, working directory must be configured
		VocInfo vi;
		initVocInfo(&vi);
		short ret = parseJpVoc("あいかわらず(相変わらず)[副]照旧,依然", vi);

		printf("pronun:%s\twriting:%s\tpartOfSpeech:%s\texpl:%s\n",
				vi.pronun,vi.writing,
				vi.partOfSpeech,vi.expl);
		/*printf("pronun:%s\twriting:%s\tpartOfSpeech:%s\texpl:%s\n",
				emptyIfNull(vi.pronun),emptyIfNull(vi.writing),
				emptyIfNull(vi.partOfSpeech),emptyIfNull(vi.expl));*/
		clearVocInfo(&vi);
		return 0;
	} else {
		fName = (pANTLR3_UINT8) argv[1];
	}
	ANTLR3_FPRINTF(stdout,"\xE6\x85\x8C\xE3\x81\xA6\xE3\x81\xBE\xE3\x81\x99\n");
	ANTLR3_FPRINTF(stdout,"input file name:%s\n",fName);
	//ANTLR3_ENC_8BIT, ANTLR3_ENC_UTF8, ANTLR3_ENC_UTF16
	input = antlr3FileStreamNew(fName, ANTLR3_ENC_UTF8);

	initParser(&input,&lxr,&tstream,&psr);
	psr->voclist(psr);
	cleanParser(&input,&lxr,&tstream,&psr);

	return 0;
}
Ejemplo n.º 7
0
/** If oldRoot is a nil root, just copy or move the children to newRoot.
 *  If not a nil root, make oldRoot a child of newRoot.
 *
 * \code
 *    old=^(nil a b c), new=r yields ^(r a b c)
 *    old=^(a b c), new=r yields ^(r ^(a b c))
 * \endcode
 *
 *  If newRoot is a nil-rooted single child tree, use the single
 *  child as the new root node.
 *
 * \code
 *    old=^(nil a b c), new=^(nil r) yields ^(r a b c)
 *    old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
 * \endcode
 *
 *  If oldRoot was null, it's ok, just return newRoot (even if isNilNode).
 *
 * \code
 *    old=null, new=r yields r
 *    old=null, new=^(nil r) yields ^(nil r)
 * \endcode
 *
 *  Return newRoot.  Throw an exception if newRoot is not a
 *  simple node or nil root with a single child node--it must be a root
 *  node.  If newRoot is <code>^(nil x)</endcode> return x as newRoot.
 *
 *  Be advised that it's ok for newRoot to point at oldRoot's
 *  children; i.e., you don't have to copy the list.  We are
 *  constructing these nodes so we should have this control for
 *  efficiency.
 */
static	pANTLR3_BASE_TREE	
becomeRoot	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE newRootTree, pANTLR3_BASE_TREE oldRootTree)
{
	/* Protect against tree rewrites if we are in some sort of error
	 * state, but have tried to recover. In C we can end up with a null pointer
	 * for a tree that was not produced.
	 */
	if	(newRootTree == NULL)
	{
		return	oldRootTree;
	}

	/* root is just the new tree as is if there is no
	 * current root tree.
	 */
	if	(oldRootTree == NULL)
	{
		return	newRootTree;
	}

	/* Produce ^(nil real-node)
	 */
	if	(newRootTree->isNilNode(newRootTree))
	{
		if	(newRootTree->getChildCount(newRootTree) > 1)
		{
			/* TODO: Handle tree exceptions 
			 */
			ANTLR3_FPRINTF(stderr, "More than one node as root! TODO: Create tree exception hndling\n");
			return newRootTree;
		}

		/* The new root is the first child
		 */
		newRootTree = newRootTree->getChild(newRootTree, 0);
	}

	/* Add old root into new root. addChild takes care of the case where oldRoot
	 * is a flat list (nill rooted tree). All children of oldroot are added to
	 * new root.
	 */
	newRootTree->addChild(newRootTree, oldRootTree);

	/* Always returns new root structure
	 */
	return	newRootTree;

}
Ejemplo n.º 8
0
int main(int /*argc*/, char** /*argv*/)
{
	pANTLR3_UINT8 fileName = (pANTLR3_UINT8)"test.mat";

	pANTLR3_COMMON_TREE_NODE_STREAM treeNodes;
	pANTLR3_INPUT_STREAM input = antlr3FileStreamNew(fileName, ANTLR3_ENC_UTF8);
	if (input == nullptr)
	{
		ANTLR3_FPRINTF(stderr, "unable to open %s", (char *)fileName);
		return ANTLR3_ERR_NOFILE;
	}

	pTalonMaterialLexer lexer = TalonMaterialLexerNew(input);
	if (lexer == nullptr)
	{
		ANTLR3_FPRINTF(stderr, "unable to create lexer.");
		return ANTLR3_ERR_NOMEM;
	}

	pANTLR3_COMMON_TOKEN_STREAM tokenStream = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT, TOKENSOURCE(lexer));
	if (tokenStream == nullptr)
	{
		ANTLR3_FPRINTF(stderr, "unable to create tokenStream.");
		return ANTLR3_ERR_NOMEM;
	}

	pTalonMaterialParser parser = TalonMaterialParserNew(tokenStream);
	if (parser == nullptr)
	{
		ANTLR3_FPRINTF(stderr, "unable to create parser.");
		return ANTLR3_ERR_NOMEM;
	}

	auto module = parser->module(parser);
	auto errorCount = parser->pParser->rec->getNumberOfSyntaxErrors(parser->pParser->rec);
	if (errorCount > 0)
	{
		ANTLR3_FPRINTF(stderr, "The parser returned %d errors, tree walking aborted.\n", errorCount);
	}
	else
	{
		ANTLR3_FPRINTF(stdout, "Parser found no errors.");
	}

	return 0;
}
Ejemplo n.º 9
0
/** Dummy implementation - will be supplied by super class
 */
static	pANTLR3_STRING	
   getText		(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE t)
{
	ANTLR3_FPRINTF(stderr, "Internal error - implementor of superclass containing ANTLR3_TREE_ADAPTOR did not implement getText()\n");
	return	NULL;
}
Ejemplo n.º 10
0
/** Default lexer error handler (works for 8 bit streams only!!!)
 */
static void			
displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
{
    pANTLR3_LEXER			lexer;
	pANTLR3_EXCEPTION	    ex;
	pANTLR3_STRING			ftext;

    lexer   = (pANTLR3_LEXER)(recognizer->super);
	ex		= lexer->rec->state->exception;

	// See if there is a 'filename' we can use
    //
    if	(ex->name == NULL)
    {
		ANTLR3_FPRINTF(stderr, "-unknown source-(");
    }
    else
    {
		ftext = ex->streamName->to8(ex->streamName);
		ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);
    }

    ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);
    ANTLR3_FPRINTF(stderr, ": lexer error %d :\n\t%s at offset %d, ", 
						ex->type,
						(pANTLR3_UINT8)	   (ex->message),
					    ex->charPositionInLine+1
		    );
	{
		ANTLR3_INT32	width;

		width	= ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));

		if	(width >= 1)
		{			
			if	(isprint(ex->c))
			{
				ANTLR3_FPRINTF(stderr, "near '%c' :\n", ex->c);
			}
			else
			{
				ANTLR3_FPRINTF(stderr, "near char(%#02X) :\n", (ANTLR3_UINT8)(ex->c));
			}
			ANTLR3_FPRINTF(stderr, "\t%.*s\n", width > 20 ? 20 : width ,((pANTLR3_UINT8)ex->index));
		}
		else
		{
			ANTLR3_FPRINTF(stderr, "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n");
			ANTLR3_FPRINTF(stderr, "\t The lexer was matching from line %d, offset %d, which\n\t ", 
								(ANTLR3_UINT32)(lexer->rec->state->tokenStartLine),
								(ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine)
								);
			width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));

			if	(width >= 1)
			{
				ANTLR3_FPRINTF(stderr, "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
			}
			else
			{
				ANTLR3_FPRINTF(stderr, "is also the end of the line, so you must check your lexer rules\n");
			}
		}
	}
}
Ejemplo n.º 11
0
void	    
addChild (pANTLR3_BASE_TREE tree, pANTLR3_BASE_TREE child)
{
	ANTLR3_UINT32   n;
	ANTLR3_UINT32   i;

	if	(child == NULL)
	{
		return;
	}

	if	(child->isNilNode(child) == ANTLR3_TRUE)
	{
		if  (child->children != NULL && child->children == tree->children)
		{
			// TODO: Change to exception rather than ANTLR3_FPRINTF?
			//
			ANTLR3_FPRINTF(stderr, "ANTLR3: An attempt was made to add a child list to itself!\n");
			return;
		}

        // Add all of the children's children to this list
        //
        if (child->children != NULL)
        {
            if (tree->children == NULL)
            {
                // We are build ing the tree structure here, so we need not
                // worry about duplication of pointers as the tree node
                // factory will only clean up each node once. So we just
                // copy in the child's children pointer as the child is
                // a nil node (has not root itself).
                //
                tree->children = child->children;
                child->children = NULL;
                freshenPACIndexesAll(tree);
                
            }
            else
            {
                // Need to copy the children
                //
                n = child->children->size(child->children);

                for (i = 0; i < n; i++)
                {
                    pANTLR3_BASE_TREE entry;
                    entry = child->children->get(child->children, i);

                    // ANTLR3 lists can be sparse, unlike Array Lists
                    //
                    if (entry != NULL)
                    {
                        tree->children->add(tree->children, entry, (void (ANTLR3_CDECL *) (void *))child->free);
                    }
                }
            }
		}
	}
	else
	{
		// Tree we are adding is not a Nil and might have children to copy
		//
		if  (tree->children == NULL)
		{
			// No children in the tree we are adding to, so create a new list on
			// the fly to hold them.
			//
			tree->createChildrenList(tree);
		}

		tree->children->add(tree->children, child, (void (ANTLR3_CDECL *)(void *))child->free);
		
	}
}
/// When constructing trees, sometimes we need to dup a token or AST
/// subtree.  Dup'ing a token means just creating another AST node
/// around it.  For trees, you must call the adaptor.dupTree().
///
static void *	
dupTreeNode	    (pANTLR3_REWRITE_RULE_ELEMENT_STREAM stream, void * element)
{
	ANTLR3_FPRINTF(stderr, "dup() cannot be called on a node rewrite stream!!!");
	return NULL;
}
/// When constructing trees, sometimes we need to dup a token or AST
/// subtree.  Dup'ing a token means just creating another AST node
/// around it.  For trees, you must call the adaptor.dupTree().
///
static void *	
dupTok	    (pANTLR3_REWRITE_RULE_ELEMENT_STREAM stream, void * el)
{
	ANTLR3_FPRINTF(stderr, "dup() cannot be called on a token rewrite stream!!");
	return NULL;
}
Ejemplo n.º 14
0
static	ANTLR3_UINT32	
   getChildCount	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE tree)
{
	ANTLR3_FPRINTF(stderr, "Internal error - implementor of superclass containing ANTLR3_TREE_ADAPTOR did not implement getChildCount()\n");
	return 0;
}
Ejemplo n.º 15
0
static	pANTLR3_BASE_TREE	
   getChild		(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE tree, ANTLR3_UINT32 i)
{
	ANTLR3_FPRINTF(stderr, "Internal error - implementor of superclass containing ANTLR3_TREE_ADAPTOR did not implement getChild()\n");
	return NULL;
}
Ejemplo n.º 16
0
/** Dummy implementation - will be supplied by super class
 */
static	void		
setText8		(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_UINT8 t)
{
	ANTLR3_FPRINTF(stderr, "Internal error - implementor of superclass containing ANTLR3_TREE_ADAPTOR did not implement setText()\n");
}
Ejemplo n.º 17
0
/**
displayRecognitionError()함수를 변경
*/
void produceError4Parser(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
{
	pANTLR3_PARSER			parser;
	pANTLR3_TREE_PARSER	    tparser;
	pANTLR3_INT_STREAM	    is;
	pANTLR3_STRING			ttext;
	pANTLR3_STRING			ftext;
	pANTLR3_EXCEPTION	    ex;
	pANTLR3_COMMON_TOKEN    theToken;
	pANTLR3_BASE_TREE	    theBaseTree;
	pANTLR3_COMMON_TREE	    theCommonTree;

	//char* p;
	FILE *out;
	out = fopen("ParErr.txt", "ab"); // 에러를 저장할 파일
	ANTLR3_FPRINTF(out, "\r\n");

	// Retrieve some info for easy reading.
	//
	ex	    =		recognizer->state->exception;
	ttext   =		NULL;

	// See if there is a 'filename' we can use
	//
	if	(ex->streamName == NULL)
	{
		if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
		{
			ANTLR3_FPRINTF(out, "-end of input-(");
		}
		else
		{
			ANTLR3_FPRINTF(out, "-unknown source-(");
		}
	}
	else
	{   
		/* rwkim 수정*/
		char curPath[200];
		getcwd(curPath, 200);
		ftext = ex->streamName->to8(ex->streamName);
		delstring(ftext->chars, curPath); //앞의 경로 제거
		ANTLR3_FPRINTF(out, "%s(", ftext->chars);
	}

	// Next comes the line number
	//

	ANTLR3_FPRINTF(out, "%d) ", recognizer->state->exception->line);
	//ANTLR3_FPRINTF(out, " : error %d : %s", 
	//									recognizer->state->exception->type,
	//				(pANTLR3_UINT8)	   (recognizer->state->exception->message));
	//다음과 같아 바꾸었음
	ANTLR3_FPRINTF(out, " : %s", (pANTLR3_UINT8)(recognizer->state->exception->message));

	// How we determine the next piece is dependent on which thing raised the
	// error.
	//
	
	switch	(recognizer->type)
	{
	case	ANTLR3_TYPE_PARSER:

		// Prepare the knowledge we know we have
		//
		parser	    = (pANTLR3_PARSER) (recognizer->super);
		tparser	    = NULL;
		is			= parser->tstream->istream;
		theToken    = (pANTLR3_COMMON_TOKEN)(recognizer->state->exception->token);
		ttext	    = theToken->toString(theToken);

		/* rwkim 수정*/
		//p = strchr(ttext->chars, '=');
		//ttext->chars = p;

		ANTLR3_FPRINTF(out, " at offset %d", recognizer->state->exception->charPositionInLine);
		if  (theToken != NULL)
		{
			if (theToken->type == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(out, ", at <EOF>");
			}
			else
			{
				// Guard against null text in a token
				//
				ANTLR3_FPRINTF(out, "\r\n    near %s\r\n    ", ttext == NULL ? (pANTLR3_UINT8)"<no text for the token>" : ttext->chars);
			}
		}
		break;

	case	ANTLR3_TYPE_TREE_PARSER:

		tparser		= (pANTLR3_TREE_PARSER) (recognizer->super);
		parser		= NULL;
		is			= tparser->ctnstream->tnstream->istream;
		theBaseTree	= (pANTLR3_BASE_TREE)(recognizer->state->exception->token);
		ttext		= theBaseTree->toStringTree(theBaseTree);

		if  (theBaseTree != NULL)
		{
			theCommonTree	= (pANTLR3_COMMON_TREE)	    theBaseTree->super;

			if	(theCommonTree != NULL)
			{
				theToken	= (pANTLR3_COMMON_TOKEN)    theBaseTree->getToken(theBaseTree);
			}
			ANTLR3_FPRINTF(out, ", at offset %d", theBaseTree->getCharPositionInLine(theBaseTree));
			ANTLR3_FPRINTF(out, ", near %s", ttext->chars);
		}
		break;

	default:

		ANTLR3_FPRINTF(out, "Base recognizer function displayRecognitionError called by unknown parser type - provide override for this function\r\n");
		return;
		break;
	}

	// Although this function should generally be provided by the implementation, this one
	// should be as helpful as possible for grammar developers and serve as an example
	// of what you can do with each exception type. In general, when you make up your
	// 'real' handler, you should debug the routine with all possible errors you expect
	// which will then let you be as specific as possible about all circumstances.
	//
	// Note that in the general case, errors thrown by tree parsers indicate a problem
	// with the output of the parser or with the tree grammar itself. The job of the parser
	// is to produce a perfect (in traversal terms) syntactically correct tree, so errors
	// at that stage should really be semantic errors that your own code determines and handles
	// in whatever way is appropriate.
	//
	switch  (ex->type)
	{
	case	ANTLR3_UNWANTED_TOKEN_EXCEPTION:

		// Indicates that the recognizer was fed a token which seesm to be
		// spurious input. We can detect this when the token that follows
		// this unwanted token would normally be part of the syntactically
		// correct stream. Then we can see that the token we are looking at
		// is just something that should not be there and throw this exception.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(out, " : Extraneous input...");
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(out, " : Extraneous input - expected <EOF>\r\n");
			}
			else
			{
				ANTLR3_FPRINTF(out, " : Extraneous input - expected %s ...\r\n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_MISSING_TOKEN_EXCEPTION:

		// Indicates that the recognizer detected that the token we just
		// hit would be valid syntactically if preceeded by a particular 
		// token. Perhaps a missing ';' at line end or a missing ',' in an
		// expression list, and such like.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(out, " : Missing token (%d)...\r\n", ex->expecting);
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(out, " : Missing <EOF>\r\n");
			}
			else
			{
				ANTLR3_FPRINTF(out, " : Missing %s \r\n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_RECOGNITION_EXCEPTION:

		// Indicates that the recognizer received a token
		// in the input that was not predicted. This is the basic exception type 
		// from which all others are derived. So we assume it was a syntax error.
		// You may get this if there are not more tokens and more are needed
		// to complete a parse for instance.
		//
		ANTLR3_FPRINTF(out, " : syntax error...\r\n");    
		break;

	case    ANTLR3_MISMATCHED_TOKEN_EXCEPTION:

		// We were expecting to see one thing and got another. This is the
		// most common error if we coudl not detect a missing or unwanted token.
		// Here you can spend your efforts to
		// derive more useful error messages based on the expected
		// token set and the last token and so on. The error following
		// bitmaps do a good job of reducing the set that we were looking
		// for down to something small. Knowing what you are parsing may be
		// able to allow you to be even more specific about an error.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(out, " : syntax error...\r\n");
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(out, " : expected <EOF>\r\n");
			}
			else
			{
				ANTLR3_FPRINTF(out, " : expected %s ...\r\n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_NO_VIABLE_ALT_EXCEPTION:

		// We could not pick any alt decision from the input given
		// so god knows what happened - however when you examine your grammar,
		// you should. It means that at the point where the current token occurred
		// that the DFA indicates nowhere to go from here.
		//
		ANTLR3_FPRINTF(out, " : cannot match to any predicted input...\r\n");

		break;

	case	ANTLR3_MISMATCHED_SET_EXCEPTION:

		{
			ANTLR3_UINT32	  count;
			ANTLR3_UINT32	  bit;
			ANTLR3_UINT32	  size;
			ANTLR3_UINT32	  numbits;
			pANTLR3_BITSET	  errBits;

			// This means we were able to deal with one of a set of
			// possible tokens at this point, but we did not see any
			// member of that set.
			//
			ANTLR3_FPRINTF(out, " : unexpected input...\r\n  expected one of : ");

			// What tokens could we have accepted at this point in the
			// parse?
			//
			count   = 0;
			errBits = antlr3BitsetLoad		(ex->expectingSet);
			numbits = errBits->numBits		(errBits);
			size    = errBits->size			(errBits);

			if  (size > 0)
			{
				// However many tokens we could have dealt with here, it is usually
				// not useful to print ALL of the set here. I arbitrarily chose 8
				// here, but you should do whatever makes sense for you of course.
				// No token number 0, so look for bit 1 and on.
				//
				for	(bit = 1; bit < numbits && count < 8 && count < size; bit++)
				{
					// TODO: This doesn;t look right - should be asking if the bit is set!!
					//
					if  (tokenNames[bit])
					{
						ANTLR3_FPRINTF(out, "%s%s", count > 0 ? ", " : "", tokenNames[bit]); 
						count++;
					}
				}
				ANTLR3_FPRINTF(out, "\r\n");
			}
			else
			{
				ANTLR3_FPRINTF(out, "Actually dude, we didn't seem to be expecting anything here, or at least\r\n");
				ANTLR3_FPRINTF(out, "I could not work out what I was expecting, like so many of us these days!\r\n");
			}
		}
		break;

	case	ANTLR3_EARLY_EXIT_EXCEPTION:

		// We entered a loop requiring a number of token sequences
		// but found a token that ended that sequence earlier than
		// we should have done.
		//
		ANTLR3_FPRINTF(out, " : missing elements...\r\n");
		break;

	default:

		// We don't handle any other exceptions here, but you can
		// if you wish. If we get an exception that hits this point
		// then we are just going to report what we know about the
		// token.
		//
		ANTLR3_FPRINTF(out, " : syntax not recognized...\r\n");
		break;
	}

	// Here you have the token that was in error which if this is
	// the standard implementation will tell you the line and offset
	// and also record the address of the start of the line in the
	// input stream. You could therefore print the source line and so on.
	// Generally though, I would expect that your lexer/parser will keep
	// its own map of lines and source pointers or whatever as there
	// are a lot of specific things you need to know about the input
	// to do something like that.
	// Here is where you do it though :-).
	//

	fclose(out); 
}
Ejemplo n.º 18
0
/** If oldRoot is a nil root, just copy or move the children to newRoot.
 *  If not a nil root, make oldRoot a child of newRoot.
 *
 * \code
 *    old=^(nil a b c), new=r yields ^(r a b c)
 *    old=^(a b c), new=r yields ^(r ^(a b c))
 * \endcode
 *
 *  If newRoot is a nil-rooted single child tree, use the single
 *  child as the new root node.
 *
 * \code
 *    old=^(nil a b c), new=^(nil r) yields ^(r a b c)
 *    old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
 * \endcode
 *
 *  If oldRoot was null, it's ok, just return newRoot (even if isNilNode).
 *
 * \code
 *    old=null, new=r yields r
 *    old=null, new=^(nil r) yields ^(nil r)
 * \endcode
 *
 *  Return newRoot.  Throw an exception if newRoot is not a
 *  simple node or nil root with a single child node--it must be a root
 *  node.  If newRoot is <code>^(nil x)</endcode> return x as newRoot.
 *
 *  Be advised that it's ok for newRoot to point at oldRoot's
 *  children; i.e., you don't have to copy the list.  We are
 *  constructing these nodes so we should have this control for
 *  efficiency.
 */
static	pANTLR3_BASE_TREE	
becomeRoot	(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_TREE newRootTree, pANTLR3_BASE_TREE oldRootTree)
{
    pANTLR3_BASE_TREE saveRoot;

	/* Protect against tree rewrites if we are in some sort of error
	 * state, but have tried to recover. In C we can end up with a null pointer
	 * for a tree that was not produced.
	 */
	if	(newRootTree == NULL)
	{
		return	oldRootTree;
	}

	/* root is just the new tree as is if there is no
	 * current root tree.
	 */
	if	(oldRootTree == NULL)
	{
		return	newRootTree;
	}

	/* Produce ^(nil real-node)
	 */
	if	(newRootTree->isNilNode(newRootTree))
	{
		if	(newRootTree->getChildCount(newRootTree) > 1)
		{
			/* TODO: Handle tree exceptions 
			 */
			ANTLR3_FPRINTF(stderr, "More than one node as root! TODO: Create tree exception handling\n");
			return newRootTree;
		}

		/* The new root is the first child, keep track of the original newRoot
         * because if it was a Nil Node, then we can reuse it now.
		 */
        saveRoot    = newRootTree;
		newRootTree = (pANTLR3_BASE_TREE)newRootTree->getChild(newRootTree, 0);

        // Reclaim the old nilNode()
        //
        saveRoot->reuse(saveRoot);
	}

	/* Add old root into new root. addChild takes care of the case where oldRoot
	 * is a flat list (nill rooted tree). All children of oldroot are added to
	 * new root.
	 */
	newRootTree->addChild(newRootTree, oldRootTree);

    // If the oldroot tree was a nil node, then we know at this point
    // it has become orphaned by the rewrite logic, so we tell it to do
    // whatever it needs to do to be reused.
    //
    if  (oldRootTree->isNilNode(oldRootTree))
    {
        // We have taken an old Root Tree and appended all its children to the new
        // root. In addition though it was a nil node, which means the generated code
        // will not reuse it again, so we will reclaim it here. First we want to zero out
        // any pointers it was carrying around. We are just the baseTree handler so we
        // don't know necessarilly know how to do this for the real node, we just ask the tree itself
        // to do it.
        //
        oldRootTree->reuse(oldRootTree);
    }
	/* Always returns new root structure
	 */
	return	newRootTree;

}
Ejemplo n.º 19
0
/// Delete children from start to stop and replace with t even if t is
/// a list (nil-root tree). Num of children can increase or decrease.
/// For huge child lists, inserting children can force walking rest of
/// children to set their child index; could be slow.
///
static void					
replaceChildren		(pANTLR3_BASE_TREE parent, ANTLR3_INT32 startChildIndex, ANTLR3_INT32 stopChildIndex, pANTLR3_BASE_TREE newTree)
{
	ANTLR3_INT32	replacingHowMany;		// How many nodes will go away
	ANTLR3_INT32	replacingWithHowMany;	// How many nodes will replace them
	ANTLR3_INT32	numNewChildren;			// Tracking variable
	ANTLR3_INT32	delta;					// Difference in new vs existing count

	ANTLR3_INT32	i;
	ANTLR3_INT32	j;

	pANTLR3_VECTOR	newChildren;			// Iterator for whatever we are going to add in
	ANTLR3_BOOLEAN	freeNewChildren;		// Whether we created the iterator locally or reused it

	if	(parent->children == NULL)
	{
		ANTLR3_FPRINTF(stderr, "replaceChildren call: Indexes are invalid; no children in list for %s", parent->getText(parent)->chars);
		return;
	}

	// Either use the existing list of children in the supplied nil node, or build a vector of the
	// tree we were given if it is not a nil node, then we treat both situations exactly the same
	//
	if	(newTree->isNilNode(newTree))
	{
		newChildren = newTree->children;
		freeNewChildren = ANTLR3_FALSE;		// We must NO free this memory
	}
	else
	{
		newChildren = antlr3VectorNew(1);
		if	(newChildren == NULL)
		{
			ANTLR3_FPRINTF(stderr, "replaceChildren: out of memory!!");
			exit(1);
		}
		newChildren->add(newChildren, (void *)newTree, NULL);

		freeNewChildren = ANTLR3_TRUE;		// We must free this memory
	}

	// Initialize
	//
	replacingHowMany		= stopChildIndex - startChildIndex + 1;
	replacingWithHowMany	= newChildren->size(newChildren);
	delta					= replacingHowMany - replacingWithHowMany;
	numNewChildren			= newChildren->size(newChildren);

	// If it is the same number of nodes, then do a direct replacement
	//
	if	(delta == 0)
	{
		pANTLR3_BASE_TREE	child;

		// Same number of nodes
		//
		j	= 0;
		for	(i = startChildIndex; i <= stopChildIndex; i++)
		{
			child = (pANTLR3_BASE_TREE) newChildren->get(newChildren, j);
			parent->children->set(parent->children, i, child, NULL, ANTLR3_FALSE);
			child->setParent(child, parent);
			child->setChildIndex(child, i);
		}
	}
	else if (delta > 0)
	{
		ANTLR3_UINT32	indexToDelete;

		// Less nodes than there were before
		// reuse what we have then delete the rest
		//
		for	(j = 0; j < numNewChildren; j++)
		{
			parent->children->set(parent->children, startChildIndex + j, newChildren->get(newChildren, j), NULL, ANTLR3_FALSE);
		}

		// We just delete the same index position until done
		//
		indexToDelete = startChildIndex + numNewChildren;

		for	(j = indexToDelete; j <= (ANTLR3_INT32)stopChildIndex; j++)
		{
			parent->children->remove(parent->children, indexToDelete);
		}

		parent->freshenPACIndexes(parent, startChildIndex);
	}
	else
	{
		ANTLR3_UINT32 numToInsert;

		// More nodes than there were before
		// Use what we can, then start adding
		//
		for	(j = 0; j < replacingHowMany; j++)
		{
			parent->children->set(parent->children, startChildIndex + j, newChildren->get(newChildren, j), NULL, ANTLR3_FALSE);
		}

		numToInsert = replacingWithHowMany - replacingHowMany;

		for	(j = replacingHowMany; j < replacingWithHowMany; j++)
		{
			parent->children->add(parent->children, newChildren->get(newChildren, j), NULL);
		}

		parent->freshenPACIndexes(parent, startChildIndex);
	}

	if	(freeNewChildren == ANTLR3_TRUE)
	{
		ANTLR3_FREE(newChildren->elements);
		newChildren->elements = NULL;
		newChildren->size = 0;
		ANTLR3_FREE(newChildren);		// Will not free the nodes
	}
}
Ejemplo n.º 20
0
bool parseProto (const char*filename, const char *outputFilename,const char * outputInternalNamespace, const char*outputExternalNamespace, char**package,pANTLR3_HASH_TABLE typeTable, bool cleanUp,ProtoJSParser_protocol_return*retval, pProtoJSLexer*ret_lxr, pProtoJSParser*ret_psr,pANTLR3_COMMON_TOKEN_STREAM*ret_tstream, pANTLR3_INPUT_STREAM* ret_stream) {
    
    pANTLR3_INPUT_STREAM input = antlr3AsciiFileStreamNew((pANTLR3_UINT8)filename);

    if ( input == NULL ) {
        fprintf(stderr, "Failed to open file %s\n", (char *)filename);
        exit(1);
    }
    pProtoJSLexer lxr = ProtoJSLexerNew(input);
    if ( lxr == NULL ) {
        fprintf(stderr, "Unable to create the lexer due to malloc() failure1\n");
        exit(1);
    }

    pANTLR3_COMMON_TOKEN_STREAM tstream = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT, TOKENSOURCE(lxr));
    if (tstream == NULL) {
	fprintf(stderr, "Out of memory trying to allocate token stream\n");
	exit(1);
    }

    pProtoJSParser ctx;
    pProtoJSParser psr = ctx = ProtoJSParserNew(tstream);
    if (psr == NULL) {
        fprintf(stderr, "Out of memory trying to allocate parser\n");
        exit(ANTLR3_ERR_NOMEM);
    }
    SCOPE_TYPE(NameSpace) ns=NameSpacePush(ctx);
    ctx->pProtoJSParser_NameSpaceTop=ns;
    ns->filename=tstream->tstream->tokenSource->strFactory->newRaw(tstream->tstream->tokenSource->strFactory);
    ns->filename->append8(SCOPE_TOP(NameSpace)->filename,(const char*)outputFilename);
    ns->internalNamespace=tstream->tstream->tokenSource->strFactory->newRaw(tstream->tstream->tokenSource->strFactory);
    ns->internalNamespace->append8(ns->internalNamespace,(const char*)outputInternalNamespace);
    ns->externalNamespace=tstream->tstream->tokenSource->strFactory->newRaw(tstream->tstream->tokenSource->strFactory);
    ns->externalNamespace->append8(SCOPE_TOP(NameSpace)->externalNamespace,(const char*)outputExternalNamespace);
    if (strlen(outputExternalNamespace)) {
        ns->externalNamespace->append8(ns->externalNamespace,".");        
    }
    initNameSpace(ctx,ns);
    pANTLR3_HASH_TABLE tempTable=ns->qualifiedTypes;
    if (*package){
        ns->package->set8(ns->package,*package);
        ns->packageDot->set8(ns->packageDot,*package);
        ns->packageDot->append8(ns->packageDot,".");
    }
    if (typeTable) {
        ns->qualifiedTypes=typeTable;
    }
    ProtoJSParser_protocol_return pbjAST=psr->protocol(psr);
    if (!*package) {
        *package=strdup((const char*)ns->package->chars);
    }

    ns->qualifiedTypes=tempTable;

    bool success=true;
    if (psr->pParser->rec->getNumberOfSyntaxErrors(psr->pParser->rec) > 0)
    {
        success=false;
        ANTLR3_FPRINTF(stderr, "The parser returned %d errors, tree walking aborted.\n", psr->pParser->rec->getNumberOfSyntaxErrors(psr->pParser->rec));
    }else {
    }

    if (cleanUp) {
        NameSpacePop(ctx);        
        psr->free(psr);
        psr = NULL;
        
        tstream->free(tstream);
        tstream = NULL;
        
        lxr->free(lxr);
        lxr = NULL;
        input->close(input);
        input = NULL;
    }else {
        *retval=pbjAST;
        *ret_lxr=lxr;
        *ret_psr=psr;
        *ret_tstream=tstream;
        *ret_stream=input;
    }
    return success;
}
Ejemplo n.º 21
0
    static void DisplayRecognitionError(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 *tokenNames)
    {
      pANTLR3_EXCEPTION ex = recognizer->state->exception;
      
      freettcn::translator::CTranslator &translator = freettcn::translator::CTranslator::Instance();
      unsigned line = ex->line;
      unsigned line2;
      unsigned pos = ex->charPositionInLine + 1;
      unsigned pos2;
      std::stringstream msg;
      std::stringstream msg2;
      std::string note;
      
      // How we determine the next piece is dependent on which thing raised the error.
      switch(recognizer->type) {
      case ANTLR3_TYPE_LEXER:
        {
          pANTLR3_LEXER lexer = (pANTLR3_LEXER)recognizer->super;
          
          if(ex->type == ANTLR3_NO_VIABLE_ALT_EXCEPTION)
            msg << "Cannot match to any predicted input";
          else
            msg << (pANTLR3_UINT8)ex->message;

          ANTLR3_INT32 width;
          
          width = ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)lexer->input->data + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));

          if(width >= 1) {
            if(isprint(ex->c))
              msg << " near '" << (char)ex->c << "'";
            else
              msg << " near char(0x" << std::hex << std::setw(2) << (int)ex->c << std::dec << ")";
          }
          else {
            msg << " '<EOF>'";
            
            // prepare second error
            line2 = lexer->rec->state->tokenStartLine;
            pos2 = lexer->rec->state->tokenStartCharPositionInLine;
            //            msg << fileName << ":" << (ANTLR3_UINT32)(lexer->rec->state->tokenStartLine) <<
            //              ":" << (ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine) << ": ";
            width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)lexer->input->data + (lexer->input->size(lexer->input))) - (pANTLR3_UINT8)lexer->rec->state->tokenStartCharIndex);
            if(width >= 1)
              msg2 << "The lexer was matching from: " << std::string((char *)lexer->rec->state->tokenStartCharIndex, 0, 20) << std::endl;
            else
              msg2 << "The lexer was matching from the end of the line" << std::endl;
            
            note = "Above errors indicates a poorly specified lexer RULE or unterminated input element such as: \"STRING[\"]";
          }
        }
        break;

      case ANTLR3_TYPE_PARSER:
        {
          pANTLR3_COMMON_TOKEN token = (pANTLR3_COMMON_TOKEN)ex->token;
          
          if(ex->type == ANTLR3_NO_VIABLE_ALT_EXCEPTION)
            msg << "Cannot match '" << token->getText(token)->chars << "' to any predicted input";
          else {
            msg << (pANTLR3_UINT8)ex->message;
            
            if(token) {
              if (token->type == ANTLR3_TOKEN_EOF)
                msg << " at '<EOF>'";
              else {
                if(ex->type == ANTLR3_MISSING_TOKEN_EXCEPTION) {
                  if(!tokenNames)
                    msg << " [" << ex->expecting << "]";
                  else {
                    if(ex->expecting == ANTLR3_TOKEN_EOF)
                      msg << " '<EOF>'";
                    else
                      msg << " '" << tokenNames[ex->expecting] << "'";
                  }
                }
                else {
                  msg << " near '" << token->getText(token)->chars << "'";
                }
              }
            }
            
            if(ex->type == ANTLR3_UNWANTED_TOKEN_EXCEPTION) {
              if(tokenNames) {
                if(ex->expecting == ANTLR3_TOKEN_EOF)
                  msg << " ('<EOF>' expected)";
                else
                  msg << " ('" << tokenNames[ex->expecting] << "' expected)";
              }
            }
          }
        }
        break;
        
      default:
        std::cerr << "DisplayRecognitionError() called by unknown parser type - provide override for this function" << std::endl;
        return;
      }
      
      switch(ex->type) {
      case ANTLR3_UNWANTED_TOKEN_EXCEPTION:
        // Indicates that the recognizer was fed a token which seesm to be
        // spurious input. We can detect this when the token that follows
        // this unwanted token would normally be part of the syntactically
        // correct stream. Then we can see that the token we are looking at
        // is just something that should not be there and throw this exception.
        //
        if(recognizer->type != ANTLR3_TYPE_PARSER) {
          if(tokenNames == NULL)
            ANTLR3_FPRINTF(stderr, " : Extraneous input...");
          else {
            if(ex->expecting == ANTLR3_TOKEN_EOF)
              ANTLR3_FPRINTF(stderr, " : Extraneous input - expected <EOF>\n");
            else
              ANTLR3_FPRINTF(stderr, " : Extraneous input - expected %s ...\n", tokenNames[ex->expecting]);
          }
        }
        break;
        
      case ANTLR3_MISSING_TOKEN_EXCEPTION:
        // Indicates that the recognizer detected that the token we just
        // hit would be valid syntactically if preceeded by a particular 
        // token. Perhaps a missing ';' at line end or a missing ',' in an
        // expression list, and such like.
        break;
        
      case ANTLR3_RECOGNITION_EXCEPTION:
        // Indicates that the recognizer received a token
        // in the input that was not predicted. This is the basic exception type 
        // from which all others are derived. So we assume it was a syntax error.
        // You may get this if there are not more tokens and more are needed
        // to complete a parse for instance.
        break;
        
      case ANTLR3_MISMATCHED_TOKEN_EXCEPTION:
        // We were expecting to see one thing and got another. This is the
        // most common error if we coudl not detect a missing or unwanted token.
        // Here you can spend your efforts to
        // derive more useful error messages based on the expected
        // token set and the last token and so on. The error following
        // bitmaps do a good job of reducing the set that we were looking
        // for down to something small. Knowing what you are parsing may be
        // able to allow you to be even more specific about an error.
        //
        if(!tokenNames)
          ANTLR3_FPRINTF(stderr, " : syntax error...\n");
        else {
          if(ex->expecting == ANTLR3_TOKEN_EOF)
            ANTLR3_FPRINTF(stderr, " : expected <EOF>\n");
          else
            ANTLR3_FPRINTF(stderr, " : expected %s ...\n", tokenNames[ex->expecting]);
        }
        break;
        
      case ANTLR3_NO_VIABLE_ALT_EXCEPTION:
        // We could not pick any alt decision from the input given
        // so god knows what happened - however when you examine your grammar,
        // you should. It means that at the point where the current token occurred
        // that the DFA indicates nowhere to go from here.
        //
        if(recognizer->type != ANTLR3_TYPE_LEXER && recognizer->type != ANTLR3_TYPE_PARSER)
          ANTLR3_FPRINTF(stderr, " : cannot match to any predicted input...\n");
        break;
        
      case ANTLR3_MISMATCHED_SET_EXCEPTION:
        {
          ANTLR3_UINT32 count;
          ANTLR3_UINT32 bit;
          ANTLR3_UINT32 size;
          ANTLR3_UINT32 numbits;
          pANTLR3_BITSET errBits;
          
          // This means we were able to deal with one of a set of
          // possible tokens at this point, but we did not see any
          // member of that set.
          //
          ANTLR3_FPRINTF(stderr, " : unexpected input...\n  expected one of : ");
          
          // What tokens could we have accepted at this point in the
          // parse?
          //
          count   = 0;
          errBits = antlr3BitsetLoad(ex->expectingSet);
          numbits = errBits->numBits(errBits);
          size    = errBits->size(errBits);
          
          if(size > 0) {
            // However many tokens we could have dealt with here, it is usually
            // not useful to print ALL of the set here. I arbitrarily chose 8
            // here, but you should do whatever makes sense for you of course.
            // No token number 0, so look for bit 1 and on.
            //
            for(bit = 1; bit < numbits && count < 8 && count < size; bit++) {
              // TODO: This doesn;t look right - should be asking if the bit is set!!
              //
              if  (tokenNames[bit]) {
                ANTLR3_FPRINTF(stderr, "%s%s", count > 0 ? ", " : nullptr, tokenNames[bit]); 
                count++;
              }
            }
            ANTLR3_FPRINTF(stderr, "\n");
          }
          else {
            ANTLR3_FPRINTF(stderr, "Actually dude, we didn't seem to be expecting anything here, or at least\n");
            ANTLR3_FPRINTF(stderr, "I could not work out what I was expecting, like so many of us these days!\n");
          }
        }
        break;
        
      case ANTLR3_EARLY_EXIT_EXCEPTION:
        // We entered a loop requiring a number of token sequences
        // but found a token that ended that sequence earlier than
        // we should have done.
        //
        ANTLR3_FPRINTF(stderr, " : missing elements...\n");
        break;
        
      default:
        
        // We don't handle any other exceptions here, but you can
        // if you wish. If we get an exception that hits this point
        // then we are just going to report what we know about the
        // token.
        //
        ANTLR3_FPRINTF(stderr, " : syntax not recognized...\n");
        std::cout << "Unknown exception type: " << ex->type << std::endl;
        break;
      }
      
      translator.Error(CLocation(translator.File(), line, pos), msg.str());
      if(!msg2.str().empty())
        translator.Error(CLocation(translator.File(), line2, pos2), msg2.str());
      if(!note.empty())
        translator.Note(CLocation(translator.File(), line2, pos2), note);
    }