/* Add a new pANTLR3_BASE_TREE to this stream
*/
static void
add (pANTLR3_REWRITE_RULE_ELEMENT_STREAM stream, void * el, void (ANTLR3_CDECL *freePtr)(void *))
{
if (el== NULL)
{
return;
}
if (stream->elements != NULL)
{
/* We have already started with a vector, which means we already have >1
* entries in the stream. So we can just add this new element to the existing
* collection.
*/
stream->elements->add(stream->elements, el, freePtr);
return;
}
if (stream->singleElement == NULL)
{
stream->singleElement = el;
return;
}
/* If we got here then we ahd only the one element so far
* and we must now create a vector to hold a collection of them
*/
stream->elements = antlr3VectorNew(0); /* We will let the vector figure things out as it goes */
stream->elements->add(stream->elements, stream->singleElement, NULL);
stream->elements->add(stream->elements, el, NULL);
stream->singleElement = NULL;
return;
}
void pythonbriefLexer_initLexer( ppythonbriefLexer ctx )
{
ctx->implicitLineJoiningLevel = 0;
ctx->startPos = -1;
ctx->tokens = antlr3VectorNew( 16384 );
ctx->identStack = antlr3StackNew( ANTLR3_LIST_SIZE_HINT );
stackPush( ctx->identStack, (void *)FIRST_CHAR_POSITION, NULL );
// Override nextToken implementation by Python specific
ctx->origNextToken = ctx->pLexer->rec->state->tokSource->nextToken;
ctx->pLexer->rec->state->tokSource->nextToken = pythonbriefLexer_nextTokenImpl;
ctx->origFree = ctx->free;
ctx->free = pythonbriefLexer_FreeImpl;
ctx->onEncoding = NULL;
return;
}
// Reset a token stream so it can be used again and can reuse it's
// resources.
//
static void
reset (pANTLR3_COMMON_TOKEN_STREAM cts)
{
// Free any resources that ar most like specifc to the
// run we just did.
//
if (cts->discardSet != NULL)
{
cts->discardSet->free(cts->discardSet);
cts->discardSet = NULL;
}
if (cts->channelOverrides != NULL)
{
cts->channelOverrides->free(cts->channelOverrides);
cts->channelOverrides = NULL;
}
// Now, if there were any existing tokens in the stream,
// then we just reset the vector count so that it starts
// again. We must traverse the entries unfortunately as
// there may be free pointers for custom token types and
// so on. However that is just a quick NULL check on the
// vector entries.
//
if (cts->tokens != NULL)
{
cts->tokens->clear(cts->tokens);
}
else
{
/* Install the token tracking tables
*/
cts->tokens = antlr3VectorNew(0);
}
// Reset to defaults
//
cts->discardOffChannel = ANTLR3_FALSE;
cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
cts->p = -1;
}
ANTLR3_API pANTLR3_STRING_FACTORY
antlr3StringFactoryNew()
{
pANTLR3_STRING_FACTORY factory;
/* Allocate memory
*/
factory = (pANTLR3_STRING_FACTORY) ANTLR3_MALLOC(sizeof(ANTLR3_STRING_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Now we make a new list to track the strings.
*/
factory->strings = antlr3VectorNew(0);
factory->index = 0;
if (factory->strings == NULL)
{
ANTLR3_FREE(factory);
return NULL;
}
/* Install the API (8 bit assumed)
*/
factory->newRaw = newRaw8;
factory->newSize = newSize8;
factory->newPtr = newPtr8;
factory->newPtr8 = newPtr8;
factory->newStr = newStr8;
factory->newStr8 = newStr8;
factory->destroy = destroy;
factory->printable = printable8;
factory->destroy = destroy;
factory->close = closeFactory;
return factory;
}
/** \brief Reset a re-startable input stream to the start
*
* \param input Input stream context pointer
*/
static void
antlr3InputReset(pANTLR3_INPUT_STREAM input)
{
input->nextChar = input->data; /* Input at first character */
input->line = 1; /* starts at line 1 */
input->charPositionInLine = -1;
input->currentLine = input->data;
input->markDepth = 0; /* Reset markers */
/* Free up the markers table if it is there
*/
if (input->markers != NULL)
{
input->markers->free(input->markers);
}
/* Install a new markers table
*/
input->markers = antlr3VectorNew(0);
}
ANTLR3_API pANTLR3_COMMON_TREE_NODE_STREAM
antlr3CommonTreeNodeStreamNew(pANTLR3_STRING_FACTORY strFactory, ANTLR3_UINT32 hint)
{
pANTLR3_COMMON_TREE_NODE_STREAM stream;
pANTLR3_COMMON_TOKEN token;
// Memory for the interface structure
//
stream = (pANTLR3_COMMON_TREE_NODE_STREAM) ANTLR3_CALLOC(1, sizeof(ANTLR3_COMMON_TREE_NODE_STREAM));
if (stream == NULL)
{
return NULL;
}
// String factory for tree walker
//
stream->stringFactory = strFactory;
// Create an adaptor for the common tree node stream
//
stream->adaptor = ANTLR3_TREE_ADAPTORNew(strFactory);
if (stream->adaptor == NULL)
{
stream->free(stream);
return NULL;
}
// Create space for the tree node stream interface
//
stream->tnstream = antlr3TreeNodeStreamNew();
if (stream->tnstream == NULL)
{
stream->adaptor->free (stream->adaptor);
stream->free (stream);
return NULL;
}
// Create space for the INT_STREAM interface
//
stream->tnstream->istream = antlr3IntStreamNew();
if (stream->tnstream->istream == NULL)
{
stream->adaptor->free (stream->adaptor);
stream->tnstream->free (stream->tnstream);
stream->free (stream);
return NULL;
}
// Install the common tree node stream API
//
stream->addNavigationNode = addNavigationNode;
stream->hasUniqueNavigationNodes = hasUniqueNavigationNodes;
stream->newDownNode = newDownNode;
stream->newUpNode = newUpNode;
stream->reset = reset;
stream->push = push;
stream->pop = pop;
stream->free = antlr3CommonTreeNodeStreamFree;
// Install the tree node stream API
//
stream->tnstream->getTreeAdaptor = getTreeAdaptor;
stream->tnstream->getTreeSource = getTreeSource;
stream->tnstream->_LT = _LT;
stream->tnstream->setUniqueNavigationNodes = setUniqueNavigationNodes;
stream->tnstream->toString = toString;
stream->tnstream->toStringSS = toStringSS;
stream->tnstream->toStringWork = toStringWork;
stream->tnstream->get = get;
// Install INT_STREAM interface
//
stream->tnstream->istream->consume = consume;
stream->tnstream->istream->index = tindex;
stream->tnstream->istream->_LA = _LA;
stream->tnstream->istream->mark = mark;
stream->tnstream->istream->release = release;
stream->tnstream->istream->rewind = rewindMark;
stream->tnstream->istream->rewindLast = rewindLast;
stream->tnstream->istream->seek = seek;
stream->tnstream->istream->size = size;
// Initialize data elements of INT stream
//
stream->tnstream->istream->type = ANTLR3_COMMONTREENODE;
stream->tnstream->istream->super = (stream->tnstream);
// Initialize data elements of TREE stream
//
stream->tnstream->ctns = stream;
// Initialize data elements of the COMMON TREE NODE stream
//
stream->super = NULL;
stream->uniqueNavigationNodes = ANTLR3_FALSE;
stream->markers = NULL;
stream->nodeStack = antlr3StackNew(INITIAL_CALL_STACK_SIZE);
// Create the node list map
//
if (hint == 0)
{
hint = DEFAULT_INITIAL_BUFFER_SIZE;
}
stream->nodes = antlr3VectorNew(hint);
stream->p = -1;
// Install the navigation nodes
//
antlr3SetCTAPI(&(stream->UP));
antlr3SetCTAPI(&(stream->DOWN));
antlr3SetCTAPI(&(stream->EOF_NODE));
antlr3SetCTAPI(&(stream->INVALID_NODE));
token = antlr3CommonTokenNew(ANTLR3_TOKEN_UP);
token->strFactory = strFactory;
token->textState = ANTLR3_TEXT_CHARP;
token->tokText.chars = (pANTLR3_UCHAR)"UP";
stream->UP.token = token;
token = antlr3CommonTokenNew(ANTLR3_TOKEN_DOWN);
token->strFactory = strFactory;
token->textState = ANTLR3_TEXT_CHARP;
token->tokText.chars = (pANTLR3_UCHAR)"DOWN";
stream->DOWN.token = token;
token = antlr3CommonTokenNew(ANTLR3_TOKEN_EOF);
token->strFactory = strFactory;
token->textState = ANTLR3_TEXT_CHARP;
token->tokText.chars = (pANTLR3_UCHAR)"EOF";
stream->EOF_NODE.token = token;
token = antlr3CommonTokenNew(ANTLR3_TOKEN_INVALID);
token->strFactory = strFactory;
token->textState = ANTLR3_TEXT_CHARP;
token->tokText.chars = (pANTLR3_UCHAR)"INVALID";
stream->INVALID_NODE.token = token;
return stream;
}
ANTLR3_API pANTLR3_COMMON_TREE_NODE_STREAM
antlr3CommonTreeNodeStreamNewStream(pANTLR3_COMMON_TREE_NODE_STREAM inStream)
{
pANTLR3_COMMON_TREE_NODE_STREAM stream;
// Memory for the interface structure
//
stream = (pANTLR3_COMMON_TREE_NODE_STREAM) ANTLR3_CALLOC(1, sizeof(ANTLR3_COMMON_TREE_NODE_STREAM));
if (stream == NULL)
{
return NULL;
}
// Copy in all the reusable parts of the originating stream and create new
// pieces where necessary.
//
// String factory for tree walker
//
stream->stringFactory = inStream->stringFactory;
// Create an adaptor for the common tree node stream
//
stream->adaptor = inStream->adaptor;
// Create space for the tree node stream interface
//
stream->tnstream = antlr3TreeNodeStreamNew();
if (stream->tnstream == NULL)
{
stream->free (stream);
return NULL;
}
// Create space for the INT_STREAM interface
//
stream->tnstream->istream = antlr3IntStreamNew();
if (stream->tnstream->istream == NULL)
{
stream->tnstream->free (stream->tnstream);
stream->free (stream);
return NULL;
}
// Install the common tree node stream API
//
stream->addNavigationNode = addNavigationNode;
stream->hasUniqueNavigationNodes = hasUniqueNavigationNodes;
stream->newDownNode = newDownNode;
stream->newUpNode = newUpNode;
stream->reset = reset;
stream->push = push;
stream->pop = pop;
stream->getLookaheadSize = getLookaheadSize;
stream->free = antlr3CommonTreeNodeStreamFree;
// Install the tree node stream API
//
stream->tnstream->getTreeAdaptor = getTreeAdaptor;
stream->tnstream->getTreeSource = getTreeSource;
stream->tnstream->_LT = _LT;
stream->tnstream->setUniqueNavigationNodes = setUniqueNavigationNodes;
stream->tnstream->toString = toString;
stream->tnstream->toStringSS = toStringSS;
stream->tnstream->toStringWork = toStringWork;
stream->tnstream->get = get;
// Install INT_STREAM interface
//
stream->tnstream->istream->consume = consume;
stream->tnstream->istream->index = tindex;
stream->tnstream->istream->_LA = _LA;
stream->tnstream->istream->mark = mark;
stream->tnstream->istream->release = release;
stream->tnstream->istream->rewind = rewindMark;
stream->tnstream->istream->rewindLast = rewindLast;
stream->tnstream->istream->seek = seek;
stream->tnstream->istream->size = size;
// Initialize data elements of INT stream
//
stream->tnstream->istream->type = ANTLR3_COMMONTREENODE;
stream->tnstream->istream->super = (stream->tnstream);
// Initialize data elements of TREE stream
//
stream->tnstream->ctns = stream;
// Initialize data elements of the COMMON TREE NODE stream
//
stream->super = NULL;
stream->uniqueNavigationNodes = ANTLR3_FALSE;
stream->markers = NULL;
stream->nodeStack = inStream->nodeStack;
// Create the node list map
//
stream->nodes = antlr3VectorNew(DEFAULT_INITIAL_BUFFER_SIZE);
stream->p = -1;
// Install the navigation nodes
//
// Install the navigation nodes
//
antlr3SetCTAPI(&(stream->UP));
antlr3SetCTAPI(&(stream->DOWN));
antlr3SetCTAPI(&(stream->EOF_NODE));
antlr3SetCTAPI(&(stream->INVALID_NODE));
stream->UP.token = inStream->UP.token;
inStream->UP.token->strFactory = stream->stringFactory;
stream->DOWN.token = inStream->DOWN.token;
inStream->DOWN.token->strFactory = stream->stringFactory;
stream->EOF_NODE.token = inStream->EOF_NODE.token;
inStream->EOF_NODE.token->strFactory = stream->stringFactory;
stream->INVALID_NODE.token = inStream->INVALID_NODE.token;
inStream->INVALID_NODE.token->strFactory= stream->stringFactory;
// Reuse the root tree of the originating stream
//
stream->root = inStream->root;
// Signal that this is a rewriting stream so we don't
// free the originating tree. Anything that we rewrite or
// duplicate here will be done through the adaptor or
// the original tree factory.
//
stream->isRewriter = ANTLR3_TRUE;
return stream;
}
ANTLR3_API pANTLR3_STRING_FACTORY
antlr3StringFactoryNew(ANTLR3_UINT32 encoding)
{
pANTLR3_STRING_FACTORY factory;
/* Allocate memory
*/
factory = (pANTLR3_STRING_FACTORY) ANTLR3_MALLOC(sizeof(ANTLR3_STRING_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Now we make a new list to track the strings.
*/
factory->strings = antlr3VectorNew(0);
factory->index = 0;
if (factory->strings == NULL)
{
ANTLR3_FREE(factory);
return NULL;
}
// Install the API
//
switch(encoding)
{
case ANTLR3_ENC_UTF16:
factory->newRaw = newRawUTF16;
factory->newSize = newSizeUTF16;
factory->newPtr = newPtrUTF16_UTF16;
factory->newPtr8 = newPtrUTF16_8;
factory->newStr = newStrUTF16_UTF16;
factory->newStr8 = newStrUTF16_8;
factory->printable = printableUTF16;
factory->destroy = destroy;
factory->close = closeFactory;
break;
// TODO: These encodings need equivalent functions to
// UTF16 and 8Bit if I am going to support those encodings in the STRING stuff.
// The STRING stuff was intended as a quick and dirty hack for people that did not
// want to worry about memory and performance very much, but nobody ever reads the
// notes or comments or uses the email list search. I want to discourage using these
// interfaces as it is much more efficient to use the pointers within the tokens
// directly, so I am not implementing the string stuff for the newer encodings.
//
case ANTLR3_ENC_UTF8:
break;
case ANTLR3_ENC_UTF32:
break;
case ANTLR3_ENC_UTF16BE:
break;
case ANTLR3_ENC_UTF16LE:
break;
case ANTLR3_ENC_UTF32BE:
break;
case ANTLR3_ENC_UTF32LE:
break;
case ANTLR3_ENC_EBCDIC:
case ANTLR3_ENC_8BIT:
default:
factory->newRaw = newRaw8;
factory->newSize = newSize8;
factory->newPtr = newPtr8;
factory->newPtr8 = newPtr8;
factory->newStr = newStr8;
factory->newStr8 = newStr8;
factory->printable = printable8;
factory->destroy = destroy;
factory->close = closeFactory;
break;
}
return factory;
}
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
{
pANTLR3_COMMON_TOKEN_STREAM stream;
/* Memory for the interface structure
*/
stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
if (stream == NULL)
{
return NULL;
}
/* Create space for the token stream interface
*/
stream->tstream = antlr3TokenStreamNew();
stream->tstream->super = stream;
/* Create space for the INT_STREAM interfacce
*/
stream->tstream->istream = antlr3IntStreamNew();
stream->tstream->istream->super = (stream->tstream);
stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
/* Install the token tracking tables
*/
stream->tokens = antlr3VectorNew(0);
/* Defaults
*/
stream->p = -1;
/* Install the common token stream API
*/
stream->setTokenTypeChannel = setTokenTypeChannel;
stream->discardTokenType = discardTokenType;
stream->discardOffChannelToks = discardOffChannel;
stream->getTokens = getTokens;
stream->getTokenRange = getTokenRange;
stream->getTokensSet = getTokensSet;
stream->getTokensList = getTokensList;
stream->getTokensType = getTokensType;
stream->reset = reset;
/* Install the token stream API
*/
stream->tstream->_LT = tokLT;
stream->tstream->get = get;
stream->tstream->getTokenSource = getTokenSource;
stream->tstream->setTokenSource = setTokenSource;
stream->tstream->toString = toString;
stream->tstream->toStringSS = toStringSS;
stream->tstream->toStringTT = toStringTT;
stream->tstream->setDebugListener = setDebugListener;
/* Install INT_STREAM interface
*/
stream->tstream->istream->_LA = _LA;
stream->tstream->istream->mark = mark;
stream->tstream->istream->release = release;
stream->tstream->istream->size = size;
stream->tstream->istream->index = tindex;
stream->tstream->istream->rewind = rewindStream;
stream->tstream->istream->rewindLast= rewindLast;
stream->tstream->istream->seek = seek;
stream->tstream->istream->consume = consume;
stream->tstream->istream->getSourceName = getSourceName;
return stream;
}
/// Delete children from start to stop and replace with t even if t is
/// a list (nil-root tree). Num of children can increase or decrease.
/// For huge child lists, inserting children can force walking rest of
/// children to set their child index; could be slow.
///
static void
replaceChildren (pANTLR3_BASE_TREE parent, ANTLR3_INT32 startChildIndex, ANTLR3_INT32 stopChildIndex, pANTLR3_BASE_TREE newTree)
{
ANTLR3_INT32 replacingHowMany; // How many nodes will go away
ANTLR3_INT32 replacingWithHowMany; // How many nodes will replace them
ANTLR3_INT32 numNewChildren; // Tracking variable
ANTLR3_INT32 delta; // Difference in new vs existing count
ANTLR3_INT32 i;
ANTLR3_INT32 j;
pANTLR3_VECTOR newChildren; // Iterator for whatever we are going to add in
ANTLR3_BOOLEAN freeNewChildren; // Whether we created the iterator locally or reused it
if (parent->children == NULL)
{
ANTLR3_FPRINTF(stderr, "replaceChildren call: Indexes are invalid; no children in list for %s", parent->getText(parent)->chars);
return;
}
// Either use the existing list of children in the supplied nil node, or build a vector of the
// tree we were given if it is not a nil node, then we treat both situations exactly the same
//
if (newTree->isNilNode(newTree))
{
newChildren = newTree->children;
freeNewChildren = ANTLR3_FALSE; // We must NO free this memory
}
else
{
newChildren = antlr3VectorNew(1);
if (newChildren == NULL)
{
ANTLR3_FPRINTF(stderr, "replaceChildren: out of memory!!");
exit(1);
}
newChildren->add(newChildren, (void *)newTree, NULL);
freeNewChildren = ANTLR3_TRUE; // We must free this memory
}
// Initialize
//
replacingHowMany = stopChildIndex - startChildIndex + 1;
replacingWithHowMany = newChildren->size(newChildren);
delta = replacingHowMany - replacingWithHowMany;
numNewChildren = newChildren->size(newChildren);
// If it is the same number of nodes, then do a direct replacement
//
if (delta == 0)
{
pANTLR3_BASE_TREE child;
// Same number of nodes
//
j = 0;
for (i = startChildIndex; i <= stopChildIndex; i++)
{
child = (pANTLR3_BASE_TREE) newChildren->get(newChildren, j);
parent->children->set(parent->children, i, child, NULL, ANTLR3_FALSE);
child->setParent(child, parent);
child->setChildIndex(child, i);
}
}
else if (delta > 0)
{
ANTLR3_UINT32 indexToDelete;
// Less nodes than there were before
// reuse what we have then delete the rest
//
for (j = 0; j < numNewChildren; j++)
{
parent->children->set(parent->children, startChildIndex + j, newChildren->get(newChildren, j), NULL, ANTLR3_FALSE);
}
// We just delete the same index position until done
//
indexToDelete = startChildIndex + numNewChildren;
for (j = indexToDelete; j <= (ANTLR3_INT32)stopChildIndex; j++)
{
parent->children->remove(parent->children, indexToDelete);
}
parent->freshenPACIndexes(parent, startChildIndex);
}
else
{
ANTLR3_UINT32 numToInsert;
// More nodes than there were before
// Use what we can, then start adding
//
for (j = 0; j < replacingHowMany; j++)
{
parent->children->set(parent->children, startChildIndex + j, newChildren->get(newChildren, j), NULL, ANTLR3_FALSE);
}
numToInsert = replacingWithHowMany - replacingHowMany;
for (j = replacingHowMany; j < replacingWithHowMany; j++)
{
parent->children->add(parent->children, newChildren->get(newChildren, j), NULL);
}
parent->freshenPACIndexes(parent, startChildIndex);
}
if (freeNewChildren == ANTLR3_TRUE)
{
ANTLR3_FREE(newChildren->elements);
newChildren->elements = NULL;
newChildren->size = 0;
ANTLR3_FREE(newChildren); // Will not free the nodes
}
}
