/**
* @brief This function dequeues data from a queue.
*
* @param queue the queue to dequeue data from
*/
const void *queue_dequeue(Queue *queue) {
const void *result;
if(queue->length < 1)
return NULL;
if(queue->size > QUEUE_DEFAULT_SIZE && (queue->length - 1) < QUEUE_SIZE_FACTOR * queue->size)
queue_realloc(queue, 0.5 * queue->size);
result = *(queue->data + queue->start);
queue->length--;
queue->start = (queue->start + 1) % queue->size;
return result;
}
/**
* @brief This function enqueues data into a queue.
*
* @param queue the queue to queue data in
* @param data the data to queue
*/
void queue_enqueue(Queue *queue, const void *data) {
if(queue->length == queue->size)
queue_realloc(queue, queue->size * 2);
*(queue->data + ((queue->start + queue->length) % queue->size)) = data;
queue->length++;
}
/*
* Initialize an IterParser object
*
* The Python arguments are:
*
* *fd*: A Python file object or a callable object
* *buffersize*: The size of the read buffer
*/
static int
IterParser_init(IterParser *self, PyObject *args, PyObject *kwds)
{
PyObject* fd = NULL;
PyObject* read = NULL;
ssize_t buffersize = 1 << 14;
static char *kwlist[] = {"fd", "buffersize", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n:IterParser.__init__", kwlist,
&fd, &buffersize)) {
return -1;
}
/* Keep the buffersize within a reasonable range */
self->buffersize = CLAMP(buffersize, (ssize_t)(1 << 10), (ssize_t)(1 << 24));
#ifdef __clang__
/* Clang can't handle the file descriptors Python gives us,
so in that case, we just call the object's read method. */
read = PyObject_GetAttrString(fd, "read");
if (read != NULL) {
fd = read;
}
#else
self->file = PyObject_AsFileDescriptor(fd);
if (self->file != -1) {
/* This is a real C file handle or descriptor. We therefore
need to allocate our own read buffer, and get the real C
object. */
self->buffer = malloc((size_t)self->buffersize);
if (self->buffer == NULL) {
PyErr_SetString(PyExc_MemoryError, "Out of memory");
goto fail;
}
self->fd = fd; Py_INCREF(self->fd);
lseek(self->file, 0, SEEK_SET);
} else
#endif
if (PyCallable_Check(fd)) {
/* fd is a Python callable */
self->fd = fd; Py_INCREF(self->fd);
self->read = fd; Py_INCREF(self->read);
} else {
PyErr_SetString(
PyExc_TypeError,
"Arg 1 to iterparser must be a file object or callable object");
goto fail;
}
PyErr_Clear();
self->queue_read_idx = 0;
self->queue_write_idx = 0;
self->done = 0;
self->text = malloc((size_t)buffersize * sizeof(XML_Char));
self->text_alloc = buffersize;
if (self->text == NULL) {
PyErr_SetString(PyExc_MemoryError, "Out of memory");
goto fail;
}
text_clear(self);
self->read_args = Py_BuildValue("(n)", buffersize);
if (self->read_args == NULL) {
goto fail;
}
self->dict_singleton = PyDict_New();
if (self->dict_singleton == NULL) {
goto fail;
}
self->td_singleton = PyUnicode_FromString("TD");
if (self->td_singleton == NULL) {
goto fail;
}
if (queue_realloc(self, buffersize)) {
goto fail;
}
/* Set up an expat parser with our callbacks */
self->parser = XML_ParserCreate(NULL);
if (self->parser == NULL) {
PyErr_SetString(PyExc_MemoryError, "Out of memory");
goto fail;
}
XML_SetUserData(self->parser, self);
XML_SetElementHandler(
self->parser,
(XML_StartElementHandler)startElement,
(XML_EndElementHandler)endElement);
XML_SetCharacterDataHandler(
self->parser,
(XML_CharacterDataHandler)characterData);
XML_SetXmlDeclHandler(
self->parser,
(XML_XmlDeclHandler)xmlDecl);
Py_XDECREF(read);
return 0;
fail:
Py_XDECREF(read);
Py_XDECREF(self->fd);
Py_XDECREF(self->read);
free(self->text);
Py_XDECREF(self->dict_singleton);
Py_XDECREF(self->td_singleton);
Py_XDECREF(self->read_args);
free(self->queue);
return -1;
}
/*
* Get the next element from the iterator.
*
* The expat event handlers above (startElement, endElement, characterData) add
* elements to the queue, which are then dequeued by this method.
*
* Care must be taken to store and later raise exceptions. Any
* exceptions raised in the expat callbacks must be stored and then
* later thrown once the queue is emptied, otherwise the exception is
* raised "too early" in queue order.
*/
static PyObject *
IterParser_next(IterParser* self)
{
PyObject* data = NULL;
XML_Char* buf;
Py_ssize_t buflen;
/* Is there anything in the queue to return? */
if (self->queue_read_idx < self->queue_write_idx) {
return self->queue[self->queue_read_idx++];
}
/* Now that the queue is empty, is there an error we need to raise? */
if (self->error_type) {
PyErr_Restore(self->error_type, self->error_value, self->error_traceback);
self->error_type = NULL;
self->error_value = NULL;
self->error_traceback = NULL;
return NULL;
}
/* The queue is empty -- have we already fed the entire file to
expat? If so, we are done and indicate the end of the iterator
by simply returning NULL. */
if (self->done) {
return NULL;
}
self->queue_read_idx = 0;
self->queue_write_idx = 0;
do {
/* Handle a generic Python read method */
if (self->read) {
data = PyObject_CallObject(self->read, self->read_args);
if (data == NULL) {
goto fail;
}
if (PyBytes_AsStringAndSize(data, &buf, &buflen) == -1) {
Py_DECREF(data);
goto fail;
}
if (buflen < self->buffersize) {
/* EOF detection method only works for local regular files */
self->done = 1;
}
/* Handle a real C file descriptor or handle -- this is faster
if we've got one. */
} else {
buflen = (Py_ssize_t)read(
self->file, self->buffer, (size_t)self->buffersize);
if (buflen == -1) {
PyErr_SetFromErrno(PyExc_OSError);
goto fail;
} else if (buflen < self->buffersize) {
/* EOF detection method only works for local regular files */
self->done = 1;
}
buf = self->buffer;
}
if(queue_realloc(self, buflen)) {
Py_XDECREF(data);
goto fail;
}
/* Feed the read buffer to expat, which will call the event handlers */
if (XML_Parse(self->parser, buf, (int)buflen, self->done) == XML_STATUS_ERROR) {
/* One of the event handlers raised a Python error, make
note of it -- it won't be thrown until the queue is
emptied. */
if (PyErr_Occurred() != NULL) {
goto fail;
}
/* expat raised an error, make note of it -- it won't be thrown
until the queue is emptied. */
Py_XDECREF(data);
PyErr_Format(
PyExc_ValueError, "%lu:%lu: %s",
XML_GetCurrentLineNumber(self->parser),
XML_GetCurrentColumnNumber(self->parser),
XML_ErrorString(XML_GetErrorCode(self->parser)));
goto fail;
}
Py_XDECREF(data);
if (PyErr_Occurred() != NULL) {
goto fail;
}
} while (self->queue_write_idx == 0 && self->done == 0);
if (self->queue_write_idx == 0) {
return NULL;
}
if (self->queue_write_idx >= self->queue_size) {
PyErr_SetString(
PyExc_RuntimeError,
"XML queue overflow. This most likely indicates an internal bug.");
return NULL;
}
return self->queue[self->queue_read_idx++];
fail:
/* We got an exception somewhere along the way. Store the exception in
the IterParser object, but clear the exception in the Python interpreter,
so we can empty the event queue and raise the exception later. */
PyErr_Fetch(&self->error_type, &self->error_value, &self->error_traceback);
PyErr_Clear();
if (self->queue_read_idx < self->queue_write_idx) {
return self->queue[self->queue_read_idx++];
}
PyErr_Restore(self->error_type, self->error_value, self->error_traceback);
self->error_type = NULL;
self->error_value = NULL;
self->error_traceback = NULL;
return NULL;
}