pANTLR3_TOKEN_FACTORY
BoundedTokenFactoryNew(pANTLR3_INPUT_STREAM input,ANTLR3_UINT32 size)
{
pANTLR3_TOKEN_FACTORY factory;
pANTLR3_COMMON_TOKEN tok;
ANTLR3_UINT32 i;
/* allocate memory
*/
factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Install factory API
*/
factory->newToken = newPoolToken;
factory->close = factoryClose;
factory->setInputStream = setInputStream;
/* Allocate the initial pool
*/
factory->thisPool = size;
factory->nextToken = 0;
factory->pools = (pANTLR3_COMMON_TOKEN*) ANTLR3_MALLOC(sizeof(pANTLR3_COMMON_TOKEN));
factory->pools[0] =
(pANTLR3_COMMON_TOKEN)
ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * size));
/* Set up the tokens once and for all */
for( i=0; i < size; i++ ) {
tok = factory->pools[0] + i;
antlr3SetTokenAPI(tok);
/* It is factory made, and we need to copy the string factory pointer
*/
tok->factoryMade = ANTLR3_TRUE;
tok->strFactory = input == NULL ? NULL : input->strFactory;
tok->input = input;
}
/* Factory space is good, we now want to initialize our cheating token
* which one it is initialized is the model for all tokens we manufacture
*/
antlr3SetTokenAPI(&factory->unTruc);
/* Set some initial variables for future copying
*/
factory->unTruc.factoryMade = ANTLR3_TRUE;
// Input stream
//
setInputStream(factory, input);
return factory;
}
static void
newPool(pANTLR3_TOKEN_FACTORY factory)
{
/* Increment factory count
*/
factory->thisPool++;
/* Ensure we have enough pointers allocated
*/
factory->pools = (pANTLR3_COMMON_TOKEN *)
ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */
(ANTLR3_UINT64)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
);
/* Allocate a new pool for the factory
*/
factory->pools[factory->thisPool] =
(pANTLR3_COMMON_TOKEN)
ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
/* Reset the counters
*/
factory->nextToken = 0;
/* Done
*/
return;
}
/// \brief Create an in-place ASCII string stream as input to ANTLR 3.
///
/// An in-place string steam is the preferred method of supplying strings to ANTLR as input
/// for lexing and compiling. This is because we make no copies of the input string but
/// read from it right where it is.
///
/// \param[in] inString Pointer to the string to be used as the input stream
/// \param[in] size Size (in 8 bit ASCII characters) of the input string
/// \param[in] name NAme to attach the input stream (can be NULL pointer)
///
/// \return
/// - Pointer to new input stream context upon success
/// - One of the ANTLR3_ERR_ defines on error.
///
/// \remark
/// - ANTLR does not alter the input string in any way.
/// - String is slightly incorrect in that the passed in pointer can be to any
/// memory in C version of ANTLR3 of course.
////
ANTLR3_API pANTLR3_INPUT_STREAM
antlr3NewAsciiStringInPlaceStream (pANTLR3_UINT8 inString, ANTLR3_UINT32 size, pANTLR3_UINT8 name)
{
// Pointer to the input stream we are going to create
//
pANTLR3_INPUT_STREAM input;
// Allocate memory for the input stream structure
//
input = (pANTLR3_INPUT_STREAM)
ANTLR3_MALLOC(sizeof(ANTLR3_INPUT_STREAM));
if (input == NULL)
{
return NULL;
}
// Structure was allocated correctly, now we can install the pointer.
//
input->isAllocated = ANTLR3_FALSE;
input->data = inString;
input->sizeBuf = size;
// Call the common 8 bit ASCII input stream handler initializer.
//
antlr3AsciiSetupStream(input, ANTLR3_CHARSTREAM);
// Now we can set up the file name
//
input->istream->streamName = input->strFactory->newStr(input->strFactory, name == NULL ? (pANTLR3_UINT8)"-memory-" : name);
input->fileName = input->istream->streamName;
return input;
}
/**
* \brief
* Creates a new ANTLR3 exception structure
*
* \param[in] exception
* One of the ANTLR3_xxx_EXCEPTION indicators such as #ANTLR3_RECOGNITION_EXCEPTION
*
* \param[in] message
* Pointer to message string
*
* \param[in] freeMessage
* Set to ANTLR3_TRUE if the message parameter should be freed by a call to
* ANTLR3_FREE() when the exception is destroyed.
*
* \returns
* Pointer to newly initialized exception structure, or an ANTLR3_ERR_xx defined value
* upon failure.
*
* An exception is 'thrown' by a recognizer when input is seen that is not predicted by
* the grammar productions or when some other error condition occurs. In C we do not have
* the luxury of try and catch blocks, so exceptions are added in the order they occur to
* a list in the baserecognizer structure. The last one to be thrown is inserted at the head of
* the list and the one currently installed is pointed to by the newly installed exception.
*
* \remarks
* After an exception is created, you may add a pointer to your own structure and a pointer
* to a function to free this structure when the exception is destroyed.
*
* \see
* ANTLR3_EXCEPTION
*/
pANTLR3_EXCEPTION
antlr3ExceptionNew(ANTLR3_UINT32 exception, void * name, void * message, ANTLR3_BOOLEAN freeMessage)
{
pANTLR3_EXCEPTION ex;
/* Allocate memory for the structure
*/
ex = (pANTLR3_EXCEPTION) ANTLR3_MALLOC(sizeof(ANTLR3_EXCEPTION));
/* Check for memory allocation
*/
if (ex == NULL)
{
return (pANTLR3_EXCEPTION)ANTLR3_FUNC_PTR(ANTLR3_ERR_NOMEM);
}
ex->name = name; /* Install exception name */
ex->type = exception; /* Install the exception number */
ex->message = message; /* Install message string */
/* Indicate whether the string should be freed if exception is destroyed
*/
ex->freeMessage = freeMessage;
/* Install the API
*/
ex->print = antlr3ExceptionPrint;
ex->freeEx = antlr3ExceptionFree;
return ex;
}
/* Functions for creating streams
*/
static pANTLR3_REWRITE_RULE_ELEMENT_STREAM
antlr3RewriteRuleElementStreamNewAE(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_UINT8 description)
{
pANTLR3_REWRITE_RULE_ELEMENT_STREAM stream;
/* First job is to create the memory we need.
*/
stream = (pANTLR3_REWRITE_RULE_ELEMENT_STREAM) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_REWRITE_RULE_ELEMENT_STREAM)));
if (stream == NULL)
{
return (pANTLR3_REWRITE_RULE_ELEMENT_STREAM)ANTLR3_FUNC_PTR(ANTLR3_ERR_NOMEM);
}
/* Populate the generic interface */
stream->reset = reset;
stream->add = add;
stream->next = next;
stream->_next = _next;
stream->hasNext = hasNext;
stream->size = size;
stream->getDescription = getDescription;
stream->free = freeRS;
/* Install the description
*/
stream->elementDescription = adaptor->strFactory->newStr8(adaptor->strFactory, description);
/* Install the adaptor
*/
stream->adaptor = adaptor;
return stream;
}
static void
newPool(pANTLR3_ARBORETUM factory)
{
// Increment factory count
//
factory->thisPool++;
// Ensure we have enough pointers allocated
//
factory->pools = (pANTLR3_COMMON_TREE *)
ANTLR3_REALLOC( (void *)factory->pools, // Current pools pointer (starts at NULL)
(ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TREE *)) // Memory for new pool pointers
);
// Allocate a new pool for the factory
//
factory->pools[factory->thisPool] =
(pANTLR3_COMMON_TREE)
ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TREE) * ANTLR3_FACTORY_POOL_SIZE));
// Reset the counters
//
factory->nextTree = 0;
// Done
//
return;
}
ANTLR3_API pANTLR3_ARBORETUM
antlr3ArboretumNew(pANTLR3_STRING_FACTORY strFactory)
{
pANTLR3_ARBORETUM factory;
// Allocate memory
//
factory = (pANTLR3_ARBORETUM) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_ARBORETUM));
if (factory == NULL)
{
return NULL;
}
// Install a vector factory to create, track and free() any child
// node lists.
//
factory->vFactory = antlr3VectorFactoryNew(0);
if (factory->vFactory == NULL)
{
free(factory);
return NULL;
}
// We also keep a reclaim stack, so that any Nil nodes that are
// orphaned are not just left in the pool but are reused, other wise
// we create 6 times as many nilNodes as ordinary nodes and use loads of
// memory. Perhaps at some point, the analysis phase will generate better
// code and we won't need to do this here.
//
factory->nilStack = antlr3StackNew(0);
// Install factory API
//
factory->newTree = newPoolTree;
factory->newFromTree = newFromTree;
factory->newFromToken = newFromToken;
factory->close = factoryClose;
// Allocate the initial pool
//
factory->thisPool = -1;
factory->pools = NULL;
newPool(factory);
// Factory space is good, we now want to initialize our cheating token
// which one it is initialized is the model for all tokens we manufacture
//
antlr3SetCTAPI(&factory->unTruc);
// Set some initial variables for future copying, including a string factory
// that we can use later for converting trees to strings.
//
factory->unTruc.factory = factory;
factory->unTruc.baseTree.strFactory = strFactory;
return factory;
}
/// \brief Create an ASCII string stream as input to ANTLR 3, copying the input string.
///
/// This string stream first makes a copy of the string at the supplied pointer
///
/// \param[in] inString Pointer to the string to be copied as the input stream
/// \param[in] size Size (in 8 bit ASCII characters) of the input string
/// \param[in] name NAme to attach the input stream (can be NULL pointer)
///
/// \return
/// - Pointer to new input stream context upon success
/// - One of the ANTLR3_ERR_ defines on error.
///
/// \remark
/// - ANTLR does not alter the input string in any way.
/// - String is slightly incorrect in that the passed in pointer can be to any
/// memory in C version of ANTLR3 of course.
////
pANTLR3_INPUT_STREAM antlr3NewAsciiStringCopyStream (pANTLR3_UINT8 inString, ANTLR3_UINT32 size, pANTLR3_UINT8 name)
{
// Pointer to the input stream we are going to create
//
pANTLR3_INPUT_STREAM input;
// Allocate memory for the input stream structure
//
input = (pANTLR3_INPUT_STREAM)
ANTLR3_MALLOC(sizeof(ANTLR3_INPUT_STREAM));
if (input == NULL)
{
return NULL;
}
// Indicate that we allocated this input and allocate it
//
input->isAllocated = ANTLR3_TRUE;
input->data = ANTLR3_MALLOC((size_t)size);
if (input->data == NULL)
{
return NULL;
}
// Structure was allocated correctly, now we can install the pointer and set the size.
//
ANTLR3_MEMMOVE(input->data, (const void *)inString, size);
input->sizeBuf = size;
// Call the common 8 bit ASCII input stream handler
// initializer type thingy doobry function.
//
antlr3AsciiSetupStream(input, ANTLR3_CHARSTREAM);
input->istream->streamName = input->strFactory->newStr(input->strFactory, name == NULL ? (pANTLR3_UINT8)"-memory-" : name);
input->fileName = input->istream->streamName;
return input;
}
// Convert a UTF16 string into a UTF8 representation using the Unicode.org
// supplied C algorithms, which are now contained within the ANTLR3 C runtime
// as permitted by the Unicode license (within the source code antlr3convertutf.c/.h
// UCS2 has the same encoding as UTF16 so we can use UTF16 converter.
//
static pANTLR3_STRING
toUTF8_UTF16 (pANTLR3_STRING string)
{
UTF8 * outputEnd;
UTF16 * inputEnd;
pANTLR3_STRING utf8String;
ConversionResult cResult;
// Allocate the output buffer, which needs to accommodate potentially
// 3X (in bytes) the input size (in chars).
//
utf8String = string->factory->newStr8(string->factory, (pANTLR3_UINT8)"");
if (utf8String != NULL)
{
// Free existing allocation
//
ANTLR3_FREE(utf8String->chars);
// Reallocate according to maximum expected size
//
utf8String->size = string->len *3;
utf8String->chars = (pANTLR3_UINT8)ANTLR3_MALLOC(utf8String->size +1);
if (utf8String->chars != NULL)
{
inputEnd = (UTF16 *) (string->chars);
outputEnd = (UTF8 *) (utf8String->chars);
// Call the Unicode converter
//
cResult = ConvertUTF16toUTF8
(
(const UTF16**)&inputEnd,
((const UTF16 *)(string->chars)) + string->len,
&outputEnd,
outputEnd + utf8String->size - 1,
lenientConversion
);
// We don't really care if things failed or not here, we just converted
// everything that was vaguely possible and stopped when it wasn't. It is
// up to the grammar programmer to verify that the input is sensible.
//
utf8String->len = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)outputEnd) - utf8String->chars);
*(outputEnd+1) = '\0'; // Always null terminate
}
}
return utf8String;
}
ANTLR3_API pANTLR3_TREE_PARSER
antlr3TreeParserNewStream(ANTLR3_UINT32 sizeHint, pANTLR3_COMMON_TREE_NODE_STREAM ctnstream, pANTLR3_RECOGNIZER_SHARED_STATE state)
{
pANTLR3_TREE_PARSER parser;
/** Allocate tree parser memory
*/
parser =(pANTLR3_TREE_PARSER) ANTLR3_MALLOC(sizeof(ANTLR3_TREE_PARSER));
if (parser == NULL)
{
return NULL;
}
/* Create and install a base recognizer which does most of the work for us
*/
parser->rec = antlr3BaseRecognizerNew(ANTLR3_TYPE_PARSER, sizeHint, state);
if (parser->rec == NULL)
{
parser->free(parser);
return NULL;
}
/* Ensure we can track back to the tree parser super structure
* from the base recognizer structure
*/
parser->rec->super = parser;
parser->rec->type = ANTLR3_TYPE_TREE_PARSER;
/* Install our base recognizer overrides
*/
parser->rec->mismatch = mismatch;
parser->rec->exConstruct = antlr3MTNExceptionNew;
parser->rec->getCurrentInputSymbol = getCurrentInputSymbol;
parser->rec->getMissingSymbol = getMissingSymbol;
/* Install tree parser API
*/
parser->getTreeNodeStream = getTreeNodeStream;
parser->setTreeNodeStream = setTreeNodeStream;
parser->free = freeParser;
/* Install the tree node stream
*/
parser->setTreeNodeStream(parser, ctnstream);
return parser;
}
ANTLR3_API pANTLR3_COMMON_TREE
antlr3CommonTreeNew()
{
pANTLR3_COMMON_TREE tree;
tree = (pANTLR3_COMMON_TREE) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TREE));
if (tree == NULL)
{
return NULL;
}
antlr3SetCTAPI(tree);
return tree;
}
/** \brief Mark the current input point in an Ascii 8 bit stream
* such as a file stream, where all the input is available in the
* buffer.
*
* \param[in] is Input stream context pointer
*/
static ANTLR3_MARKER
antlr3AsciiMark (pANTLR3_INT_STREAM is)
{
pANTLR3_LEX_STATE state;
pANTLR3_INPUT_STREAM input;
input = ((pANTLR3_INPUT_STREAM) (is->super));
/* New mark point
*/
input->markDepth++;
/* See if we are revisiting a mark as we can just reuse the vector
* entry if we are, otherwise, we need a new one
*/
if (input->markDepth > input->markers->count)
{
state = ANTLR3_MALLOC(sizeof(ANTLR3_LEX_STATE));
/* Add it to the table
*/
input->markers->add(input->markers, state, ANTLR3_FREE_FUNC); /* No special structure, just free() on delete */
}
else
{
state = (pANTLR3_LEX_STATE)input->markers->get(input->markers, input->markDepth - 1);
/* Assume no errors for speed, it will just blow up if the table failed
* for some reasons, hence lots of unit tests on the tables ;-)
*/
}
/* We have created or retrieved the state, so update it with the current
* elements of the lexer state.
*/
state->charPositionInLine = input->charPositionInLine;
state->currentLine = input->currentLine;
state->line = input->line;
state->nextChar = input->nextChar;
is->lastMarker = input->markDepth;
/* And that's it
*/
return input->markDepth;
}
ANTLR3_API pANTLR3_TOKEN_FACTORY
antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
{
pANTLR3_TOKEN_FACTORY factory;
/* allocate memory
*/
factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Install factory API
*/
factory->newToken = newPoolToken;
factory->close = factoryClose;
factory->setInputStream = setInputStream;
factory->reset = factoryReset;
/* Allocate the initial pool
*/
factory->thisPool = -1;
factory->pools = NULL;
factory->maxPool = -1;
newPool(factory);
/* Factory space is good, we now want to initialize our cheating token
* which one it is initialized is the model for all tokens we manufacture
*/
antlr3SetTokenAPI(&factory->unTruc);
/* Set some initial variables for future copying
*/
factory->unTruc.factoryMade = ANTLR3_TRUE;
// Input stream
//
setInputStream(factory, input);
return factory;
}
static ANTLR3_BOOLEAN
newPool(pANTLR3_ARBORETUM factory)
{
pANTLR3_COMMON_TREE *newPools;
// Increment factory count
//
++factory->thisPool;
// Ensure we have enough pointers allocated
//
newPools = (pANTLR3_COMMON_TREE *)
ANTLR3_REALLOC( (void *)factory->pools, // Current pools pointer (starts at NULL)
(ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TREE *)) // Memory for new pool pointers
);
if (newPools == NULL)
{
// realloc failed, but we still have the old allocation
--factory->thisPool;
return ANTLR3_FALSE;
}
factory->pools = newPools;
// Allocate a new pool for the factory
//
factory->pools[factory->thisPool] =
(pANTLR3_COMMON_TREE)
ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TREE) * ANTLR3_FACTORY_POOL_SIZE));
if (factory->pools[factory->thisPool] == NULL)
{
// malloc failed
--factory->thisPool;
return ANTLR3_FALSE;
}
// Reset the counters
//
factory->nextTree = 0;
// Done
//
return ANTLR3_TRUE;
}
ANTLR3_API ANTLR3_UINT32
antlr3readAscii(pANTLR3_INPUT_STREAM input, pANTLR3_UINT8 fileName)
{
ANTLR3_FDSC infile;
ANTLR3_UINT32 fSize;
/* Open the OS file in read binary mode
*/
infile = antlr3Fopen(fileName, "rb");
/* Check that it was there
*/
if (infile == NULL)
{
return (ANTLR3_UINT32)ANTLR3_ERR_NOFILE;
}
/* It was there, so we can read the bytes now
*/
fSize = antlr3Fsize(fileName); /* Size of input file */
/* Allocate buffer for this input set
*/
input->data = ANTLR3_MALLOC((size_t)fSize);
input->sizeBuf = fSize;
if (input->data == NULL)
{
return (ANTLR3_UINT32)ANTLR3_ERR_NOMEM;
}
input->isAllocated = ANTLR3_TRUE;
/* Now we read the file. Characters are not converted to
* the internal ANTLR encoding until they are read from the buffer
*/
antlr3Fread(infile, fSize, input->data);
/* And close the file handle
*/
antlr3Fclose(infile);
return ANTLR3_SUCCESS;
}
/**
* Creates a new string with enough capacity for size UTF16 characters plus a terminator.
*
* \param[in] factory - Pointer to the string factory that owns strings
* \param[in] size - In characters (count double for surrogate pairs!!!)
* \return pointer to the new string.
*/
static pANTLR3_STRING
newSizeUTF16 (pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size)
{
pANTLR3_STRING string;
string = factory->newRaw(factory);
if (string == NULL)
{
return string;
}
/* Always add one more byte for a terminator ;-)
*/
string->chars = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_UINT16) * (size+1)));
*(string->chars) = '\0';
string->size = size+1; /* Size is always in characters, as is len */
return string;
}
/** \brief Use the contents of an operating system file as the input
* for an input stream.
*
* \param fileName Name of operating system file to read.
* \return
* - Pointer to new input stream context upon success
* - One of the ANTLR3_ERR_ defines on error.
*/
ANTLR3_API pANTLR3_INPUT_STREAM
antlr3AsciiFileStreamNew(pANTLR3_UINT8 fileName)
{
/* Pointer to the input stream we are going to create
*/
pANTLR3_INPUT_STREAM input;
ANTLR3_UINT64 status;
/* Allocate memory for the input stream structure
*/
input = (pANTLR3_INPUT_STREAM)
ANTLR3_MALLOC(sizeof(ANTLR3_INPUT_STREAM));
if (input == NULL)
{
return (pANTLR3_INPUT_STREAM) ANTLR3_ERR_NOMEM;
}
input->fileName = ANTLR3_STRDUP(fileName);
/* Structure was allocated correctly, now we can read the file.
*/
status = antlr3readAscii(input);
if (status != ANTLR3_SUCCESS)
{
ANTLR3_FREE(input->fileName);
ANTLR3_FREE(input);
return ANTLR3_FUNC_PTR(status);
}
/* Call the common 8 bit ASCII input stream handler
* intializer type thingy doobry function.
*/
antlr3AsciiSetupStream(input, ANTLR3_CHARSTREAM);
return input;
}
ANTLR3_API pANTLR3_STRING_FACTORY
antlr3StringFactoryNew()
{
pANTLR3_STRING_FACTORY factory;
/* Allocate memory
*/
factory = (pANTLR3_STRING_FACTORY) ANTLR3_MALLOC(sizeof(ANTLR3_STRING_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Now we make a new list to track the strings.
*/
factory->strings = antlr3VectorNew(0);
factory->index = 0;
if (factory->strings == NULL)
{
ANTLR3_FREE(factory);
return NULL;
}
/* Install the API (8 bit assumed)
*/
factory->newRaw = newRaw8;
factory->newSize = newSize8;
factory->newPtr = newPtr8;
factory->newPtr8 = newPtr8;
factory->newStr = newStr8;
factory->newStr8 = newStr8;
factory->destroy = destroy;
factory->printable = printable8;
factory->destroy = destroy;
factory->close = closeFactory;
return factory;
}
ANTLR3_API pANTLR3_TOKEN_STREAM
antlr3TokenStreamNew()
{
pANTLR3_TOKEN_STREAM stream;
// Memory for the interface structure
//
stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
if (stream == NULL)
{
return NULL;
}
// Install basic API
//
stream->free = antlr3TokenStreamFree;
return stream;
}
static pANTLR3_COMMON_TOKEN
newToken(void)
{
pANTLR3_COMMON_TOKEN token;
/* Allocate memory for this
*/
token = (pANTLR3_COMMON_TOKEN) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
if (token == NULL)
{
return (pANTLR3_COMMON_TOKEN)ANTLR3_FUNC_PTR(ANTLR3_ERR_NOMEM);
}
/* Install the API
*/
antlr3SetTokenAPI(token);
token->factoryMade = ANTLR3_FALSE;
return token;
}
static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
{
if (token->text == NULL)
{
/* Do we have a string factory to build a new string with?
*/
if (token->input == NULL || token->input->strFactory == NULL)
{
/* There was no input stream for this token, or
* it did not pay the rent on a string factory.
*/
/* There was no string factory, therefore, if this is not a factory made
* token, we assume no resizing etc will go on and just set the text as it is given.
*/
if (token->factoryMade == ANTLR3_FALSE)
{
token->text = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING));
token->text->len = (ANTLR3_UINT32)strlen((const char *)text);
token->text->size = token->text->len ;
token->text->chars = text;
}
return;
}
/* We can make a new string from the supplied text then
*/
token->text = token->input->strFactory->newStr8(token->input->strFactory, text);
}
else
{
token->text->set8(token->text, (const char *)text);
}
/* We are done
*/
return;
}
/* Function that returns an 8 bit version of the string,
* which in this case is returning all the UTF16 characters
* narrowed back into 8 bits, with characters that are too large
* replaced with '_'
*/
static pANTLR3_STRING to8_UTF16 (pANTLR3_STRING string)
{
pANTLR3_STRING newStr;
ANTLR3_UINT32 i;
/* Create a new 8 bit string
*/
newStr = newRaw8(string->factory);
if (newStr == NULL)
{
return NULL;
}
/* Always add one more byte for a terminator
*/
newStr->chars = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(string->len + 1));
newStr->size = string->len + 1;
newStr->len = string->len;
/* Now copy each UTF16 charActer , making it an 8 bit character of
* some sort.
*/
for (i=0; i<string->len; i++)
{
ANTLR3_UCHAR c;
c = *(((pANTLR3_UINT16)(string->chars)) + i);
*(newStr->chars + i) = (ANTLR3_UINT8)(c > 255 ? '_' : c);
}
/* Terminate
*/
*(newStr->chars + newStr->len) = '\0';
return newStr;
}
/// \brief Create an in-place UCS2 string stream as input to ANTLR 3.
///
/// An in-place string steam is the preferred method of supplying strings to ANTLR as input
/// for lexing and compiling. This is because we make no copies of the input string but
/// read from it right where it is.
///
/// \param[in] inString Pointer to the string to be used as the input stream
/// \param[in] size Size (in 16 bit ASCII characters) of the input string
/// \param[in] name Name to attach the input stream (can be NULL pointer)
///
/// \return
/// - Pointer to new input stream context upon success
/// - One of the ANTLR3_ERR_ defines on error.
///
/// \remark
/// - ANTLR does not alter the input string in any way.
/// - String is slightly incorrect in that the passed in pointer can be to any
/// memory in C version of ANTLR3 of course.
////
ANTLR3_API pANTLR3_INPUT_STREAM
antlr3NewUCS2StringInPlaceStream (pANTLR3_UINT16 inString, ANTLR3_UINT32 size, pANTLR3_UINT16 name)
{
// Pointer to the input stream we are going to create
//
pANTLR3_INPUT_STREAM input;
// Layout default file name string in correct encoding
//
ANTLR3_UINT16 defaultName[] = { '-', 'm', 'e', 'm', 'o', 'r', 'y', '-', '\0' };
// Allocate memory for the input stream structure
//
input = (pANTLR3_INPUT_STREAM)
ANTLR3_MALLOC(sizeof(ANTLR3_INPUT_STREAM));
if (input == NULL)
{
return NULL;
}
// Structure was allocated correctly, now we can install the pointer.
//
input->isAllocated = ANTLR3_FALSE;
input->data = inString;
input->sizeBuf = size;
// Call the common 16 bit input stream handler initializer.
//
antlr3UCS2SetupStream (input, ANTLR3_CHARSTREAM);
input->istream->streamName = input->strFactory->newStr(input->strFactory, name == NULL ? (pANTLR3_UINT8)defaultName : (pANTLR3_UINT8)name);
input->fileName = input->istream->streamName;
return input;
}
/**
*
* \param factory
* \return
*/
static pANTLR3_STRING
newRawUTF16 (pANTLR3_STRING_FACTORY factory)
{
pANTLR3_STRING string;
string = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING));
if (string == NULL)
{
return NULL;
}
/* Structure is allocated, now fill in the API etc.
*/
stringInitUTF16(string);
string->factory = factory;
/* Add the string into the allocated list
*/
factory->strings->set(factory->strings, factory->index, (void *) string, (void (ANTLR3_CDECL *)(void *))(stringFree), ANTLR3_TRUE);
string->index = factory->index++;
return string;
}
/** Create a new tree adaptor. Note that despite the fact that this is
* creating a new COMMON_TREE adaptor, we return the address of the
* BASE_TREE interface, as should any other adaptor that wishes to be
* used as the tree element of a tree parse/build. It needs to be given the
* address of a valid string factory as we do not know what the originating
* input stream encoding type was. This way we can rely on just using
* the original input stream's string factory or one of the correct type
* which the user supplies us.
*/
ANTLR3_API pANTLR3_BASE_TREE_ADAPTOR
ANTLR3_TREE_ADAPTORNew(pANTLR3_STRING_FACTORY strFactory)
{
pANTLR3_COMMON_TREE_ADAPTOR cta;
// First job is to create the memory we need for the tree adaptor interface.
//
cta = (pANTLR3_COMMON_TREE_ADAPTOR) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_COMMON_TREE_ADAPTOR)));
if (cta == NULL)
{
return NULL;
}
// Memory is initialized, so initialize the base tree adaptor
//
antlr3BaseTreeAdaptorInit(&(cta->baseAdaptor), NULL);
// Install our interface overrides.
//
cta->baseAdaptor.dupNode = (void * (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
dupNode;
cta->baseAdaptor.create = (void * (*) (pANTLR3_BASE_TREE_ADAPTOR, pANTLR3_COMMON_TOKEN))
create;
cta->baseAdaptor.createToken =
createToken;
cta->baseAdaptor.createTokenFromToken =
createTokenFromToken;
cta->baseAdaptor.setTokenBoundaries = (void (*) (pANTLR3_BASE_TREE_ADAPTOR, void *, pANTLR3_COMMON_TOKEN, pANTLR3_COMMON_TOKEN))
setTokenBoundaries;
cta->baseAdaptor.getTokenStartIndex = (ANTLR3_MARKER (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
getTokenStartIndex;
cta->baseAdaptor.getTokenStopIndex = (ANTLR3_MARKER (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
getTokenStopIndex;
cta->baseAdaptor.getText = (pANTLR3_STRING (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
getText;
cta->baseAdaptor.getType = (ANTLR3_UINT32 (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
getType;
cta->baseAdaptor.getChild = (void * (*) (pANTLR3_BASE_TREE_ADAPTOR, void *, ANTLR3_UINT32))
getChild;
cta->baseAdaptor.setChild = (void (*) (pANTLR3_BASE_TREE_ADAPTOR, void *, ANTLR3_UINT32, void *))
setChild;
cta->baseAdaptor.deleteChild = (void (*) (pANTLR3_BASE_TREE_ADAPTOR, void *, ANTLR3_UINT32))
deleteChild;
cta->baseAdaptor.getChildCount = (ANTLR3_UINT32 (*) (pANTLR3_BASE_TREE_ADAPTOR, void *))
getChildCount;
cta->baseAdaptor.free = (void (*) (pANTLR3_BASE_TREE_ADAPTOR))
ctaFree;
cta->baseAdaptor.setDebugEventListener =
setDebugEventListener;
cta->baseAdaptor.replaceChildren = (void (*) (pANTLR3_BASE_TREE_ADAPTOR, void *, ANTLR3_INT32, ANTLR3_INT32, void *))
replaceChildren;
cta->baseAdaptor.errorNode = (void * (*) (pANTLR3_BASE_TREE_ADAPTOR, pANTLR3_TOKEN_STREAM, pANTLR3_COMMON_TOKEN, pANTLR3_COMMON_TOKEN, pANTLR3_EXCEPTION))
errorNode;
// Install the super class pointer
//
cta->baseAdaptor.super = cta;
// Install a tree factory for creating new tree nodes
//
cta->arboretum = antlr3ArboretumNew(strFactory);
// Install a token factory for imaginary tokens, these imaginary
// tokens do not require access to the input stream so we can
// dummy the creation of it.
//
cta->baseAdaptor.tokenFactory = antlr3TokenFactoryNew(NULL);
// Allow the base tree adaptor to share the tree factory's string factory.
//
cta->baseAdaptor.strFactory = strFactory;
// Return the address of the base adaptor interface.
//
return &(cta->baseAdaptor);
}
ANTLR3_API pANTLR3_LEXER
antlr3LexerNew(ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
{
pANTLR3_LEXER lexer;
pANTLR3_COMMON_TOKEN specialT;
/* Allocate memory
*/
lexer = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER));
if (lexer == NULL)
{
return NULL;
}
/* Now we need to create the base recognizer
*/
lexer->rec = antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state);
if (lexer->rec == NULL)
{
lexer->free(lexer);
return NULL;
}
lexer->rec->super = lexer;
lexer->rec->displayRecognitionError = displayRecognitionError;
lexer->rec->reportError = reportError;
lexer->rec->reset = reset;
lexer->rec->getCurrentInputSymbol = getCurrentInputSymbol;
lexer->rec->getMissingSymbol = getMissingSymbol;
/* Now install the token source interface
*/
if (lexer->rec->state->tokSource == NULL)
{
lexer->rec->state->tokSource = (pANTLR3_TOKEN_SOURCE)ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_SOURCE));
if (lexer->rec->state->tokSource == NULL)
{
lexer->rec->free(lexer->rec);
lexer->free(lexer);
return NULL;
}
lexer->rec->state->tokSource->super = lexer;
/* Install the default nextToken() method, which may be overridden
* by generated code, or by anything else in fact.
*/
lexer->rec->state->tokSource->nextToken = nextToken;
lexer->rec->state->tokSource->strFactory = NULL;
lexer->rec->state->tokFactory = NULL;
}
/* Install the lexer API
*/
lexer->setCharStream = setCharStream;
lexer->mTokens = (void (*)(void *))(mTokens);
lexer->setCharStream = setCharStream;
lexer->pushCharStream = pushCharStream;
lexer->popCharStream = popCharStream;
lexer->emit = emit;
lexer->emitNew = emitNew;
lexer->matchs = matchs;
lexer->matchc = matchc;
lexer->matchRange = matchRange;
lexer->matchAny = matchAny;
lexer->recover = recover;
lexer->getLine = getLine;
lexer->getCharIndex = getCharIndex;
lexer->getCharPositionInLine = getCharPositionInLine;
lexer->getText = getText;
lexer->free = freeLexer;
/* Initialise the eof token
*/
specialT = &(lexer->rec->state->tokSource->eofToken);
antlr3SetTokenAPI (specialT);
specialT->setType (specialT, ANTLR3_TOKEN_EOF);
specialT->factoryMade = ANTLR3_TRUE; // Prevent things trying to free() it
specialT->strFactory = NULL;
specialT->textState = ANTLR3_TEXT_NONE;
specialT->custom = NULL;
specialT->user1 = 0;
specialT->user2 = 0;
specialT->user3 = 0;
// Initialize the skip token.
//
specialT = &(lexer->rec->state->tokSource->skipToken);
antlr3SetTokenAPI (specialT);
specialT->setType (specialT, ANTLR3_TOKEN_INVALID);
specialT->factoryMade = ANTLR3_TRUE; // Prevent things trying to free() it
specialT->strFactory = NULL;
specialT->custom = NULL;
specialT->user1 = 0;
specialT->user2 = 0;
specialT->user3 = 0;
return lexer;
}
pBOUNDED_TOKEN_BUFFER
BoundedTokenBufferSourceNew(ANTLR3_UINT32 k, pANTLR3_TOKEN_SOURCE source)
{
pBOUNDED_TOKEN_BUFFER buffer;
pANTLR3_COMMON_TOKEN_STREAM stream;
assert( k > 0 );
/* Memory for the interface structure
*/
buffer = (pBOUNDED_TOKEN_BUFFER) ANTLR3_MALLOC(sizeof(BOUNDED_TOKEN_BUFFER_struct));
if (buffer == NULL)
{
return NULL;
}
buffer->tokenBuffer = (pANTLR3_COMMON_TOKEN*) ANTLR3_MALLOC(2*k*sizeof(pANTLR3_COMMON_TOKEN));
buffer->currentIndex = 0;
buffer->maxIndex = 0;
buffer->k = k;
buffer->bufferSize = 2*k;
buffer->empty = ANTLR3_TRUE;
buffer->done = ANTLR3_FALSE;
stream = antlr3CommonTokenStreamSourceNew(k,source);
if (stream == NULL)
{
return NULL;
}
stream->super = buffer;
buffer->commonTstream = stream;
/* Defaults
*/
stream->p = -1;
/* Install the token stream API
*/
stream->tstream->_LT = tokLT;
stream->tstream->get = get;
stream->tstream->getTokenSource = getTokenSource;
stream->tstream->setTokenSource = setTokenSource;
stream->tstream->toString = toString;
stream->tstream->toStringSS = toStringSS;
stream->tstream->toStringTT = toStringTT;
stream->tstream->setDebugListener = setDebugListener;
/* Install INT_STREAM interface
*/
stream->tstream->istream->_LA = _LA;
stream->tstream->istream->mark = mark;
stream->tstream->istream->release = release;
stream->tstream->istream->size = size;
stream->tstream->istream->index = tindex;
stream->tstream->istream->rewind = rewindStream;
stream->tstream->istream->rewindLast= rewindLast;
stream->tstream->istream->seek = seek;
stream->tstream->istream->consume = consume;
stream->tstream->istream->getSourceName = getSourceName;
return buffer;
}
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
{
pANTLR3_COMMON_TOKEN_STREAM stream;
/* Memory for the interface structure
*/
stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
if (stream == NULL)
{
return NULL;
}
/* Create space for the token stream interface
*/
stream->tstream = antlr3TokenStreamNew();
stream->tstream->super = stream;
/* Create space for the INT_STREAM interfacce
*/
stream->tstream->istream = antlr3IntStreamNew();
stream->tstream->istream->super = (stream->tstream);
stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
/* Install the token tracking tables
*/
stream->tokens = antlr3VectorNew(0);
/* Defaults
*/
stream->p = -1;
/* Install the common token stream API
*/
stream->setTokenTypeChannel = setTokenTypeChannel;
stream->discardTokenType = discardTokenType;
stream->discardOffChannelToks = discardOffChannel;
stream->getTokens = getTokens;
stream->getTokenRange = getTokenRange;
stream->getTokensSet = getTokensSet;
stream->getTokensList = getTokensList;
stream->getTokensType = getTokensType;
stream->reset = reset;
/* Install the token stream API
*/
stream->tstream->_LT = tokLT;
stream->tstream->get = get;
stream->tstream->getTokenSource = getTokenSource;
stream->tstream->setTokenSource = setTokenSource;
stream->tstream->toString = toString;
stream->tstream->toStringSS = toStringSS;
stream->tstream->toStringTT = toStringTT;
stream->tstream->setDebugListener = setDebugListener;
/* Install INT_STREAM interface
*/
stream->tstream->istream->_LA = _LA;
stream->tstream->istream->mark = mark;
stream->tstream->istream->release = release;
stream->tstream->istream->size = size;
stream->tstream->istream->index = tindex;
stream->tstream->istream->rewind = rewindStream;
stream->tstream->istream->rewindLast= rewindLast;
stream->tstream->istream->seek = seek;
stream->tstream->istream->consume = consume;
stream->tstream->istream->getSourceName = getSourceName;
return stream;
}
ANTLR3_API pANTLR3_STRING_FACTORY
antlr3StringFactoryNew(ANTLR3_UINT32 encoding)
{
pANTLR3_STRING_FACTORY factory;
/* Allocate memory
*/
factory = (pANTLR3_STRING_FACTORY) ANTLR3_MALLOC(sizeof(ANTLR3_STRING_FACTORY));
if (factory == NULL)
{
return NULL;
}
/* Now we make a new list to track the strings.
*/
factory->strings = antlr3VectorNew(0);
factory->index = 0;
if (factory->strings == NULL)
{
ANTLR3_FREE(factory);
return NULL;
}
// Install the API
//
switch(encoding)
{
case ANTLR3_ENC_UTF16:
factory->newRaw = newRawUTF16;
factory->newSize = newSizeUTF16;
factory->newPtr = newPtrUTF16_UTF16;
factory->newPtr8 = newPtrUTF16_8;
factory->newStr = newStrUTF16_UTF16;
factory->newStr8 = newStrUTF16_8;
factory->printable = printableUTF16;
factory->destroy = destroy;
factory->close = closeFactory;
break;
// TODO: These encodings need equivalent functions to
// UTF16 and 8Bit if I am going to support those encodings in the STRING stuff.
// The STRING stuff was intended as a quick and dirty hack for people that did not
// want to worry about memory and performance very much, but nobody ever reads the
// notes or comments or uses the email list search. I want to discourage using these
// interfaces as it is much more efficient to use the pointers within the tokens
// directly, so I am not implementing the string stuff for the newer encodings.
//
case ANTLR3_ENC_UTF8:
break;
case ANTLR3_ENC_UTF32:
break;
case ANTLR3_ENC_UTF16BE:
break;
case ANTLR3_ENC_UTF16LE:
break;
case ANTLR3_ENC_UTF32BE:
break;
case ANTLR3_ENC_UTF32LE:
break;
case ANTLR3_ENC_EBCDIC:
case ANTLR3_ENC_8BIT:
default:
factory->newRaw = newRaw8;
factory->newSize = newSize8;
factory->newPtr = newPtr8;
factory->newPtr8 = newPtr8;
factory->newStr = newStr8;
factory->newStr8 = newStr8;
factory->printable = printable8;
factory->destroy = destroy;
factory->close = closeFactory;
break;
}
return factory;
}
// Functions for creating streams
//
static pANTLR3_REWRITE_RULE_ELEMENT_STREAM
antlr3RewriteRuleElementStreamNewAE(pANTLR3_BASE_TREE_ADAPTOR adaptor, pANTLR3_BASE_RECOGNIZER rec, pANTLR3_UINT8 description)
{
pANTLR3_REWRITE_RULE_ELEMENT_STREAM stream;
// First - do we already have a rewrite stream that was returned
// to the pool? If we do, then we will just reuse it by resetting
// the generic interface.
//
if (rec->state->rStreams->count > 0)
{
// Remove the entry from the vector. We do not
// cause it to be freed by using remove.
//
stream = rec->state->rStreams->remove(rec->state->rStreams, rec->state->rStreams->count - 1);
// We found a stream we can reuse.
// If the stream had a vector, then it will have been cleared
// when the freeRS was called that put it in this stack
//
}
else
{
// Ok, we need to allocate a new one as there were none on the stack.
// First job is to create the memory we need.
//
stream = (pANTLR3_REWRITE_RULE_ELEMENT_STREAM) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_REWRITE_RULE_ELEMENT_STREAM)));
if (stream == NULL)
{
return NULL;
}
stream->elements = NULL;
stream->freeElements = ANTLR3_FALSE;
}
// Populate the generic interface
//
stream->rec = rec;
stream->reset = reset;
stream->add = add;
stream->next = next;
stream->nextTree = nextTree;
stream->nextNode = nextNode;
stream->nextToken = nextToken;
stream->_next = _next;
stream->hasNext = hasNext;
stream->size = size;
stream->getDescription = getDescription;
stream->toTree = toTree;
stream->free = freeRS;
stream->singleElement = NULL;
// Reset the stream to empty.
//
stream->cursor = 0;
stream->dirty = ANTLR3_FALSE;
// Install the description
//
stream->elementDescription = description;
// Install the adaptor
//
stream->adaptor = adaptor;
return stream;
}
