void reader_init_zlibinflate(reader_t*r, reader_t*input)
{
#ifdef HAVE_ZLIB
zlibinflate_t*z = (zlibinflate_t*)malloc(sizeof(zlibinflate_t));
memset(z, 0, sizeof(zlibinflate_t));
int ret;
memset(r, 0, sizeof(reader_t));
r->internal = z;
r->read = reader_zlibinflate;
r->seek = reader_zlibseek;
r->dealloc = reader_zlibinflate_dealloc;
r->type = READER_TYPE_ZLIB;
r->pos = 0;
z->input = input;
memset(&z->zs,0,sizeof(z_stream));
z->zs.zalloc = Z_NULL;
z->zs.zfree = Z_NULL;
z->zs.opaque = Z_NULL;
ret = inflateInit(&z->zs);
if (ret != Z_OK) zlib_error(ret, "bitio:inflate_init", &z->zs);
reader_resetbits(r);
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
void writer_init_zlibdeflate(writer_t*w, writer_t*output)
{
#ifdef HAVE_ZLIB
zlibdeflate_t*z;
int ret;
memset(w, 0, sizeof(writer_t));
z = (zlibdeflate_t*)malloc(sizeof(zlibdeflate_t));
memset(z, 0, sizeof(zlibdeflate_t));
w->internal = z;
w->write = writer_zlibdeflate_write;
w->flush = writer_zlibdeflate_flush;
w->finish = writer_zlibdeflate_finish;
w->type = WRITER_TYPE_ZLIB;
w->pos = 0;
z->output = output;
memset(&z->zs,0,sizeof(z_stream));
z->zs.zalloc = Z_NULL;
z->zs.zfree = Z_NULL;
z->zs.opaque = Z_NULL;
ret = deflateInit(&z->zs, 9);
if (ret != Z_OK) zlib_error(ret, "bitio:deflate_init", &z->zs);
w->bitpos = 0;
w->mybyte = 0;
z->zs.next_out = z->writebuffer;
z->zs.avail_out = ZLIB_BUFFER_SIZE;
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
void writer_zlibdeflate_flush(writer_t*writer)
{
#ifdef HAVE_ZLIB
zlibdeflate_t*z = (zlibdeflate_t*)writer->internal;
int ret;
if(writer->type != WRITER_TYPE_ZLIB) {
fprintf(stderr, "Wrong writer ID (writer not initialized?)\n");
return;
}
if(!z) {
fprintf(stderr, "zlib not initialized!\n");
return;
}
z->zs.next_in = 0;
z->zs.avail_in = 0;
while(1) {
ret = deflate(&z->zs, Z_SYNC_FLUSH);
if (ret != Z_OK) zlib_error(ret, "bitio:deflate_flush", &z->zs);
if(z->zs.next_out != z->writebuffer) {
writer->pos += z->zs.next_out - (Bytef*)z->writebuffer;
z->output->write(z->output, z->writebuffer, z->zs.next_out - (Bytef*)z->writebuffer);
z->zs.next_out = z->writebuffer;
z->zs.avail_out = ZLIB_BUFFER_SIZE;
}
/* TODO: how will zlib let us know it needs more buffer space? */
break;
}
return;
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
static PyObject *
PyZlib_decompressobj(PyObject *selfptr, PyObject *args)
{
int wbits=DEF_WBITS, err;
compobject *self;
if (!PyArg_ParseTuple(args, "|i:decompressobj", &wbits))
return NULL;
self = newcompobject(&Decomptype);
if (self == NULL)
return(NULL);
self->zst.zalloc = (alloc_func)NULL;
self->zst.zfree = (free_func)Z_NULL;
self->zst.next_in = NULL;
self->zst.avail_in = 0;
err = inflateInit2(&self->zst, wbits);
switch(err) {
case (Z_OK):
self->is_initialised = 1;
return (PyObject*)self;
case(Z_STREAM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_ValueError, "Invalid initialization option");
return NULL;
case (Z_MEM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory for decompression object");
return NULL;
default:
zlib_error(self->zst, err, "while creating decompression object");
Py_DECREF(self);
return NULL;
}
}
static int reader_zlibinflate(reader_t*reader, void* data, int len)
{
#ifdef HAVE_ZLIB
zlibinflate_t*z = (zlibinflate_t*)reader->internal;
int ret;
if(!z) {
return 0;
}
if(!len)
return 0;
z->zs.next_out = (Bytef *)data;
z->zs.avail_out = len;
while(1) {
if(!z->zs.avail_in) {
z->zs.avail_in = z->input->read(z->input, z->readbuffer, ZLIB_BUFFER_SIZE);
z->zs.next_in = z->readbuffer;
}
if(z->zs.avail_in)
ret = inflate(&z->zs, Z_NO_FLUSH);
else
ret = inflate(&z->zs, Z_FINISH);
if (ret != Z_OK &&
ret != Z_STREAM_END) zlib_error(ret, "bitio:inflate_inflate", &z->zs);
if (ret == Z_STREAM_END) {
int pos = z->zs.next_out - (Bytef*)data;
ret = inflateEnd(&z->zs);
if (ret != Z_OK) zlib_error(ret, "bitio:inflate_end", &z->zs);
free(reader->internal);
reader->internal = 0;
reader->pos += pos;
return pos;
}
if(!z->zs.avail_out) {
break;
}
}
reader->pos += len;
return len;
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
static PyObject *
PyZlib_unflush(compobject *self, PyObject *args)
{
int err, length = DEFAULTALLOC;
PyObject * retval = NULL;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "|i:flush", &length))
return NULL;
if (!(retval = PyString_FromStringAndSize(NULL, length)))
return NULL;
ENTER_ZLIB
start_total_out = self->zst.total_out;
self->zst.avail_out = length;
self->zst.next_out = (Byte *)PyString_AS_STRING(retval);
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH);
Py_END_ALLOW_THREADS
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) {
if (_PyString_Resize(&retval, length << 1) < 0)
goto error;
self->zst.next_out = (Byte *)PyString_AS_STRING(retval) + length;
self->zst.avail_out = length;
length = length << 1;
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH);
Py_END_ALLOW_THREADS
}
/* If flushmode is Z_FINISH, we also have to call deflateEnd() to free
various data structures. Note we should only get Z_STREAM_END when
flushmode is Z_FINISH */
if (err == Z_STREAM_END) {
err = inflateEnd(&(self->zst));
self->is_initialised = 0;
if (err != Z_OK) {
zlib_error(self->zst, err, "from inflateEnd()");
Py_DECREF(retval);
retval = NULL;
goto error;
}
}
_PyString_Resize(&retval, self->zst.total_out - start_total_out);
error:
LEAVE_ZLIB
return retval;
}
static PyObject *
PyZlib_objcompress(compobject *self, PyObject *args)
{
int err, inplen;
Py_ssize_t length = DEFAULTALLOC;
PyObject *RetVal;
Byte *input;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "s#:compress", &input, &inplen))
return NULL;
if (!(RetVal = PyString_FromStringAndSize(NULL, length)))
return NULL;
ENTER_ZLIB
start_total_out = self->zst.total_out;
self->zst.avail_in = inplen;
self->zst.next_in = input;
self->zst.avail_out = length;
self->zst.next_out = (unsigned char *)PyString_AS_STRING(RetVal);
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH);
Py_END_ALLOW_THREADS
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) {
if (_PyString_Resize(&RetVal, length << 1) < 0)
goto error;
self->zst.next_out = (unsigned char *)PyString_AS_STRING(RetVal) \
+ length;
self->zst.avail_out = length;
length = length << 1;
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH);
Py_END_ALLOW_THREADS
}
/* We will only get Z_BUF_ERROR if the output buffer was full but
there wasn't more output when we tried again, so it is not an error
condition.
*/
if (err != Z_OK && err != Z_BUF_ERROR) {
zlib_error(self->zst, err, "while compressing");
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
_PyString_Resize(&RetVal, self->zst.total_out - start_total_out);
error:
LEAVE_ZLIB
return RetVal;
}
static void writer_zlibdeflate_finish(writer_t*writer)
{
#ifdef HAVE_ZLIB
zlibdeflate_t*z = (zlibdeflate_t*)writer->internal;
writer_t*output;
int ret;
if(writer->type != WRITER_TYPE_ZLIB) {
fprintf(stderr, "Wrong writer ID (writer not initialized?)\n");
return;
}
if(!z)
return;
output= z->output;
while(1) {
ret = deflate(&z->zs, Z_FINISH);
if (ret != Z_OK &&
ret != Z_STREAM_END) zlib_error(ret, "bitio:deflate_finish", &z->zs);
if(z->zs.next_out != z->writebuffer) {
writer->pos += z->zs.next_out - (Bytef*)z->writebuffer;
z->output->write(z->output, z->writebuffer, z->zs.next_out - (Bytef*)z->writebuffer);
z->zs.next_out = z->writebuffer;
z->zs.avail_out = ZLIB_BUFFER_SIZE;
}
if (ret == Z_STREAM_END) {
break;
}
}
ret = deflateEnd(&z->zs);
if (ret != Z_OK) zlib_error(ret, "bitio:deflate_end", &z->zs);
free(writer->internal);
memset(writer, 0, sizeof(writer_t));
//output->finish(output);
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
/*
* zlib_compress --
* WiredTiger zlib compression.
*/
static int
zlib_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_len,
size_t *result_lenp, int *compression_failed)
{
ZLIB_COMPRESSOR *zlib_compressor;
ZLIB_OPAQUE opaque;
z_stream zs;
int ret;
zlib_compressor = (ZLIB_COMPRESSOR *)compressor;
memset(&zs, 0, sizeof(zs));
zs.zalloc = zalloc;
zs.zfree = zfree;
opaque.compressor = compressor;
opaque.session = session;
zs.opaque = &opaque;
if ((ret = deflateInit(&zs, zlib_compressor->zlib_level)) != Z_OK)
return (zlib_error(compressor, session, "deflateInit", ret));
zs.next_in = src;
zs.avail_in = (uint32_t)src_len;
zs.next_out = dst;
zs.avail_out = (uint32_t)dst_len;
if (deflate(&zs, Z_FINISH) == Z_STREAM_END) {
*compression_failed = 0;
*result_lenp = (size_t)zs.total_out;
} else
*compression_failed = 1;
if ((ret = deflateEnd(&zs)) != Z_OK && ret != Z_DATA_ERROR)
return (zlib_error(compressor, session, "deflateEnd", ret));
return (0);
}
void zlib_error::check BOOST_PREVENT_MACRO_SUBSTITUTION(int error)
{
switch (error) {
case Z_OK:
case Z_STREAM_END:
//case Z_BUF_ERROR:
return;
case Z_MEM_ERROR:
boost::throw_exception(std::bad_alloc());
default:
boost::throw_exception(zlib_error(error));
;
}
}
void zlib_error::check(int error)
{
switch (error) {
case Z_OK:
case Z_STREAM_END:
//case Z_BUF_ERROR:
return;
case Z_MEM_ERROR:
throw std::bad_alloc();
default:
throw zlib_error(error);
;
}
}
/*
* zlib_decompress --
* WiredTiger zlib decompression.
*/
static int
zlib_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_len,
size_t *result_lenp)
{
ZLIB_OPAQUE opaque;
z_stream zs;
int ret, tret;
memset(&zs, 0, sizeof(zs));
zs.zalloc = zalloc;
zs.zfree = zfree;
opaque.compressor = compressor;
opaque.session = session;
zs.opaque = &opaque;
if ((ret = inflateInit(&zs)) != Z_OK)
return (zlib_error(compressor, session, "inflateInit", ret));
zs.next_in = src;
zs.avail_in = (uint32_t)src_len;
zs.next_out = dst;
zs.avail_out = (uint32_t)dst_len;
while ((ret = inflate(&zs, Z_FINISH)) == Z_OK)
;
if (ret == Z_STREAM_END) {
*result_lenp = (size_t)zs.total_out;
ret = Z_OK;
}
if ((tret = inflateEnd(&zs)) != Z_OK && ret == Z_OK)
ret = tret;
return (ret == Z_OK ?
0 : zlib_error(compressor, session, "inflate", ret));
}
static PyObject *
PyZlib_uncopy(compobject *self)
{
compobject *retval = NULL;
int err;
retval = newcompobject(&Decomptype);
if (!retval) return NULL;
/* Copy the zstream state
* We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe
*/
ENTER_ZLIB(self);
err = inflateCopy(&retval->zst, &self->zst);
switch(err) {
case(Z_OK):
break;
case(Z_STREAM_ERROR):
PyErr_SetString(PyExc_ValueError, "Inconsistent stream state");
goto error;
case(Z_MEM_ERROR):
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory for decompression object");
goto error;
default:
zlib_error(self->zst, err, "while copying decompression object");
goto error;
}
Py_INCREF(self->unused_data);
Py_INCREF(self->unconsumed_tail);
Py_XDECREF(retval->unused_data);
Py_XDECREF(retval->unconsumed_tail);
retval->unused_data = self->unused_data;
retval->unconsumed_tail = self->unconsumed_tail;
retval->eof = self->eof;
/* Mark it as being initialized */
retval->is_initialised = 1;
LEAVE_ZLIB(self);
return (PyObject *)retval;
error:
LEAVE_ZLIB(self);
Py_XDECREF(retval);
return NULL;
}
/**
deflate_init : level:int -> 'dstream
<doc>Open a compression stream with the given level of compression</doc>
**/
static value deflate_init( value level ) {
z_stream *z;
value s;
int err;
val_check(level,int);
z = (z_stream*)malloc(sizeof(z_stream) + sizeof(int));
memset(z,0,sizeof(z_stream));
val_flush(z) = Z_NO_FLUSH;
if( (err = deflateInit(z,val_int(level))) != Z_OK ) {
free(z);
zlib_error(NULL,err);
}
s = alloc_abstract(k_stream_def,z);
//val_gc(s,free_stream_def);
return s;
}
HL_PRIM fmt_zip *HL_NAME(deflate_init)( int level ) {
z_stream *z;
int err;
fmt_zip *s;
z = (z_stream*)malloc(sizeof(z_stream));
memset(z,0,sizeof(z_stream));
if( (err = deflateInit(z,level)) != Z_OK ) {
free(z);
zlib_error(NULL,err);
}
s = (fmt_zip*)hl_gc_alloc_finalizer(sizeof(fmt_zip));
s->finalize = free_stream_inf;
s->flush = Z_NO_FLUSH;
s->z = z;
s->inflate = false;
return s;
}
HL_PRIM bool HL_NAME(deflate_buffer)( fmt_zip *zip, vbyte *src, int srcpos, int srclen, vbyte *dst, int dstpos, int dstlen, int *read, int *write ) {
int slen, dlen, err;
z_stream *z = zip->z;
slen = srclen - srcpos;
dlen = dstlen - dstpos;
if( srcpos < 0 || dstpos < 0 || slen < 0 || dlen < 0 )
hl_error("Out of range");
z->next_in = (Bytef*)(src + srcpos);
z->next_out = (Bytef*)(dst + dstpos);
z->avail_in = slen;
z->avail_out = dlen;
if( (err = deflate(z,zip->flush)) < 0 )
zlib_error(z,err);
z->next_in = NULL;
z->next_out = NULL;
*read = slen - z->avail_in;
*write = dlen - z->avail_out;
return err == Z_STREAM_END;
}
HL_PRIM fmt_zip *HL_NAME(inflate_init)( int wbits ) {
z_stream *z;
int err;
fmt_zip *s;
if( wbits == 0 )
wbits = MAX_WBITS;
z = (z_stream*)malloc(sizeof(z_stream));
memset(z,0,sizeof(z_stream));
if( (err = inflateInit2(z,wbits)) != Z_OK ) {
free(z);
zlib_error(NULL,err);
}
s = (fmt_zip*)hl_gc_alloc_finalizer(sizeof(fmt_zip));
s->finalize = free_stream_inf;
s->flush = Z_NO_FLUSH;
s->z = z;
s->inflate = true;
return s;
}
Array<uint8_t> compress(RawArray<const uint8_t> data, int level, event_t event) {
thread_time_t time(compress_kind,event);
if (level<20) { // zlib
size_t dest_size = compressBound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
int z = compress2(compressed.data(),&dest_size,(uint8_t*)data.data(),data.size(),level);
if (z!=Z_OK)
THROW(IOError,"zlib failure in compress_and_write: %s",zlib_error(z));
return compressed.slice_own(0,CHECK_CAST_INT(dest_size));
} else { // lzma
size_t dest_size = lzma_stream_buffer_bound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
size_t pos = 0;
lzma_ret r = lzma_easy_buffer_encode(level-20,LZMA_CHECK_CRC64,0,data.data(),data.size(),compressed.data(),&pos,dest_size);
if (r!=LZMA_OK)
THROW(RuntimeError,"lzma compression error: %s (%d)",lzma_error(r),r);
return compressed.slice_own(0,CHECK_CAST_INT(pos));
}
}
static int writer_zlibdeflate_write(writer_t*writer, void* data, int len)
{
#ifdef HAVE_ZLIB
zlibdeflate_t*z = (zlibdeflate_t*)writer->internal;
int ret;
if(writer->type != WRITER_TYPE_ZLIB) {
fprintf(stderr, "Wrong writer ID (writer not initialized?)\n");
return 0;
}
if(!z) {
fprintf(stderr, "zlib not initialized!\n");
return 0;
}
if(!len)
return 0;
z->zs.next_in = (Bytef *)data;
z->zs.avail_in = len;
while(1) {
ret = deflate(&z->zs, Z_NO_FLUSH);
if (ret != Z_OK) zlib_error(ret, "bitio:deflate_deflate", &z->zs);
if(z->zs.next_out != z->writebuffer) {
writer->pos += z->zs.next_out - (Bytef*)z->writebuffer;
z->output->write(z->output, z->writebuffer, z->zs.next_out - (Bytef*)z->writebuffer);
z->zs.next_out = z->writebuffer;
z->zs.avail_out = ZLIB_BUFFER_SIZE;
}
if(!z->zs.avail_in) {
break;
}
}
return len;
#else
fprintf(stderr, "Error: swftools was compiled without zlib support");
exit(1);
#endif
}
Array<uint8_t> decompress(RawArray<const uint8_t> compressed, const size_t uncompressed_size, event_t event) {
GEODE_ASSERT(uncompressed_size<(uint64_t)1<<31);
thread_time_t time(decompress_kind,event);
size_t dest_size = uncompressed_size;
Array<uint8_t> uncompressed = aligned_buffer<uint8_t>(CHECK_CAST_INT(dest_size));
if (!is_lzma(compressed)) { // zlib
int z = uncompress((uint8_t*)uncompressed.data(),&dest_size,compressed.data(),compressed.size());
if (z!=Z_OK)
THROW(IOError,"zlib failure in read_and_uncompress: %s",zlib_error(z));
} else { // lzma
const uint32_t flags = LZMA_TELL_NO_CHECK | LZMA_TELL_UNSUPPORTED_CHECK;
uint64_t memlimit = UINT64_MAX;
size_t in_pos = 0, out_pos = 0;
lzma_ret r = lzma_stream_buffer_decode(&memlimit,flags,0,compressed.data(),&in_pos,compressed.size(),uncompressed.data(),&out_pos,dest_size);
if (r!=LZMA_OK)
THROW(IOError,"lzma failure in read_and_uncompress: %s (%d)",lzma_error(r),r);
}
if (dest_size != uncompressed_size)
THROW(IOError,"read_and_compress: expected uncompressed size %zu, got %zu",uncompressed_size,dest_size);
return uncompressed;
}
/**
inflate_buffer : 'istream -> src:string -> srcpos:int -> dst:string -> dstpos:int -> { done => bool, read => int, write => int }
**/
static value inflate_buffer( value s, value src, value srcpos, value dst, value dstpos ) {
z_stream *z;
int err;
value o;
val_check_kind(s,k_stream_inf);
val_check(srcpos,int);
buffer src_buf = val_to_buffer(src);
if (!src_buf)
hx_failure("invalid source buffer");
buffer dst_buf = val_to_buffer(dst);
if (!dst_buf)
hx_failure("invalid destination buffer");
int slen = buffer_size(src_buf);
int dlen = buffer_size(dst_buf);
val_check(dstpos,int);
z = val_stream(s);
if( val_int(srcpos) < 0 || val_int(dstpos) < 0 )
return alloc_null();
slen -= val_int(srcpos);
dlen -= val_int(dstpos);
if( slen < 0 || dlen < 0 )
return alloc_null();
z->next_in = (Bytef*)buffer_data(src_buf) + val_int(srcpos);
z->next_out = (Bytef*)buffer_data(dst_buf) + val_int(dstpos);
z->avail_in = slen;
z->avail_out = dlen;
if( (err = inflate(z,val_flush(z))) < 0 )
zlib_error(z,err);
z->next_in = NULL;
z->next_out = NULL;
o = alloc_empty_object();
alloc_field(o,id_done,alloc_bool(err == Z_STREAM_END));
alloc_field(o,id_read,alloc_int((int)(slen - z->avail_in)));
alloc_field(o,id_write,alloc_int((int)(dlen - z->avail_out)));
return o;
}
/**
inflate_init : window_size:int? -> 'istream
<doc>Open a decompression stream</doc>
**/
static value inflate_init( value wsize ) {
z_stream *z;
value s;
int err;
int wbits;
if( val_is_null(wsize) )
wbits = MAX_WBITS;
else {
val_check(wsize,int);
wbits = val_int(wsize);
}
z = (z_stream*)malloc(sizeof(z_stream) + sizeof(int));
memset(z,0,sizeof(z_stream));
val_flush(z) = Z_NO_FLUSH;
if( (err = inflateInit2(z,wbits)) != Z_OK ) {
free(z);
zlib_error(NULL,err);
}
s = alloc_abstract(k_stream_inf,z);
//val_gc(s,free_stream_inf);
return s;
}
static PyObject *
PyZlib_compressobj(PyObject *selfptr, PyObject *args)
{
compobject *self;
int level=Z_DEFAULT_COMPRESSION, method=DEFLATED;
int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err;
if (!PyArg_ParseTuple(args, "|iiiii:compressobj", &level, &method, &wbits,
&memLevel, &strategy))
return NULL;
self = newcompobject(&Comptype);
if (self==NULL)
return(NULL);
self->zst.zalloc = (alloc_func)NULL;
self->zst.zfree = (free_func)Z_NULL;
self->zst.next_in = NULL;
self->zst.avail_in = 0;
err = deflateInit2(&self->zst, level, method, wbits, memLevel, strategy);
switch(err) {
case (Z_OK):
self->is_initialised = 1;
return (PyObject*)self;
case (Z_MEM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory for compression object");
return NULL;
case(Z_STREAM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_ValueError, "Invalid initialization option");
return NULL;
default:
zlib_error(self->zst, err, "while creating compression object");
Py_DECREF(self);
return NULL;
}
}
static PyObject *
PyZlib_compress(PyObject *self, PyObject *args)
{
PyObject *ReturnVal = NULL;
Py_buffer pinput;
Byte *input, *output = NULL;
unsigned int length;
int level=Z_DEFAULT_COMPRESSION, err;
z_stream zst;
/* require Python string object, optional 'level' arg */
if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level))
return NULL;
if (pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error;
}
input = pinput.buf;
length = pinput.len;
zst.avail_out = length + length/1000 + 12 + 1;
output = (Byte*)malloc(zst.avail_out);
if (output == NULL) {
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory to compress data");
goto error;
}
/* Past the point of no return. From here on out, we need to make sure
we clean up mallocs & INCREFs. */
zst.zalloc = (alloc_func)NULL;
zst.zfree = (free_func)Z_NULL;
zst.next_out = (Byte *)output;
zst.next_in = (Byte *)input;
zst.avail_in = length;
err = deflateInit(&zst, level);
switch(err) {
case(Z_OK):
break;
case(Z_MEM_ERROR):
PyErr_SetString(PyExc_MemoryError,
"Out of memory while compressing data");
goto error;
case(Z_STREAM_ERROR):
PyErr_SetString(ZlibError,
"Bad compression level");
goto error;
default:
deflateEnd(&zst);
zlib_error(zst, err, "while compressing data");
goto error;
}
Py_BEGIN_ALLOW_THREADS;
err = deflate(&zst, Z_FINISH);
Py_END_ALLOW_THREADS;
if (err != Z_STREAM_END) {
zlib_error(zst, err, "while compressing data");
deflateEnd(&zst);
goto error;
}
err=deflateEnd(&zst);
if (err == Z_OK)
ReturnVal = PyBytes_FromStringAndSize((char *)output,
zst.total_out);
else
zlib_error(zst, err, "while finishing compression");
error:
PyBuffer_Release(&pinput);
free(output);
return ReturnVal;
}
static PyObject *
PyZlib_decompress(PyObject *self, PyObject *args)
{
PyObject *result_str = NULL;
Py_buffer pinput;
Byte *input;
unsigned int length;
int err;
int wsize=DEF_WBITS;
Py_ssize_t r_strlen=DEFAULTALLOC;
z_stream zst;
if (!PyArg_ParseTuple(args, "y*|in:decompress",
&pinput, &wsize, &r_strlen))
return NULL;
if (pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error;
}
input = pinput.buf;
length = pinput.len;
if (r_strlen <= 0)
r_strlen = 1;
zst.avail_in = length;
zst.avail_out = r_strlen;
if (!(result_str = PyBytes_FromStringAndSize(NULL, r_strlen)))
goto error;
zst.zalloc = (alloc_func)NULL;
zst.zfree = (free_func)Z_NULL;
zst.next_out = (Byte *)PyBytes_AS_STRING(result_str);
zst.next_in = (Byte *)input;
err = inflateInit2(&zst, wsize);
switch(err) {
case(Z_OK):
break;
case(Z_MEM_ERROR):
PyErr_SetString(PyExc_MemoryError,
"Out of memory while decompressing data");
goto error;
default:
inflateEnd(&zst);
zlib_error(zst, err, "while preparing to decompress data");
goto error;
}
do {
Py_BEGIN_ALLOW_THREADS
err=inflate(&zst, Z_FINISH);
Py_END_ALLOW_THREADS
switch(err) {
case(Z_STREAM_END):
break;
case(Z_BUF_ERROR):
/*
* If there is at least 1 byte of room according to zst.avail_out
* and we get this error, assume that it means zlib cannot
* process the inflate call() due to an error in the data.
*/
if (zst.avail_out > 0) {
zlib_error(zst, err, "while decompressing data");
inflateEnd(&zst);
goto error;
}
/* fall through */
case(Z_OK):
/* need more memory */
if (_PyBytes_Resize(&result_str, r_strlen << 1) < 0) {
inflateEnd(&zst);
goto error;
}
zst.next_out =
(unsigned char *)PyBytes_AS_STRING(result_str) + r_strlen;
zst.avail_out = r_strlen;
r_strlen = r_strlen << 1;
break;
default:
inflateEnd(&zst);
zlib_error(zst, err, "while decompressing data");
goto error;
}
} while (err != Z_STREAM_END);
err = inflateEnd(&zst);
if (err != Z_OK) {
zlib_error(zst, err, "while finishing decompression");
goto error;
}
if (_PyBytes_Resize(&result_str, zst.total_out) < 0)
goto error;
PyBuffer_Release(&pinput);
return result_str;
error:
PyBuffer_Release(&pinput);
Py_XDECREF(result_str);
return NULL;
}
static PyObject *
PyZlib_objcompress(compobject *self, PyObject *args)
{
int err;
unsigned int inplen;
Py_ssize_t length = DEFAULTALLOC;
PyObject *RetVal = NULL;
Py_buffer pinput;
Byte *input;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "y*:compress", &pinput))
return NULL;
if (pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error_outer;
}
input = pinput.buf;
inplen = pinput.len;
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
goto error_outer;
ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = inplen;
self->zst.next_in = input;
self->zst.avail_out = length;
self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal);
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH);
Py_END_ALLOW_THREADS
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length;
length = length << 1;
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH);
Py_END_ALLOW_THREADS
}
/* We will only get Z_BUF_ERROR if the output buffer was full but
there wasn't more output when we tried again, so it is not an error
condition.
*/
if (err != Z_OK && err != Z_BUF_ERROR) {
zlib_error(self->zst, err, "while compressing data");
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
}
error:
LEAVE_ZLIB(self);
error_outer:
PyBuffer_Release(&pinput);
return RetVal;
}
static PyObject *
PyZlib_objdecompress(compobject *self, PyObject *args)
{
int err, max_length = 0;
unsigned int inplen;
Py_ssize_t old_length, length = DEFAULTALLOC;
PyObject *RetVal = NULL;
Py_buffer pinput;
Byte *input;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "y*|i:decompress", &pinput,
&max_length))
return NULL;
if (pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error_outer;
}
input = pinput.buf;
inplen = pinput.len;
if (max_length < 0) {
PyErr_SetString(PyExc_ValueError,
"max_length must be greater than zero");
goto error_outer;
}
/* limit amount of data allocated to max_length */
if (max_length && length > max_length)
length = max_length;
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
goto error_outer;
ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = inplen;
self->zst.next_in = input;
self->zst.avail_out = length;
self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal);
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_SYNC_FLUSH);
Py_END_ALLOW_THREADS
/* While Z_OK and the output buffer is full, there might be more output.
So extend the output buffer and try again.
*/
while (err == Z_OK && self->zst.avail_out == 0) {
/* If max_length set, don't continue decompressing if we've already
reached the limit.
*/
if (max_length && length >= max_length)
break;
/* otherwise, ... */
old_length = length;
length = length << 1;
if (max_length && length > max_length)
length = max_length;
if (_PyBytes_Resize(&RetVal, length) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + old_length;
self->zst.avail_out = length - old_length;
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_SYNC_FLUSH);
Py_END_ALLOW_THREADS
}
if(max_length) {
/* Not all of the compressed data could be accommodated in a buffer of
the specified size. Return the unconsumed tail in an attribute. */
Py_DECREF(self->unconsumed_tail);
self->unconsumed_tail = PyBytes_FromStringAndSize((char *)self->zst.next_in,
self->zst.avail_in);
}
else if (PyBytes_GET_SIZE(self->unconsumed_tail) > 0) {
/* All of the compressed data was consumed. Clear unconsumed_tail. */
Py_DECREF(self->unconsumed_tail);
self->unconsumed_tail = PyBytes_FromStringAndSize("", 0);
}
if (self->unconsumed_tail == NULL) {
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
/* The end of the compressed data has been reached, so set the
unused_data attribute to a string containing the remainder of the
data in the string. Note that this is also a logical place to call
inflateEnd, but the old behaviour of only calling it on flush() is
preserved.
*/
if (err == Z_STREAM_END) {
Py_XDECREF(self->unused_data); /* Free original empty string */
self->unused_data = PyBytes_FromStringAndSize(
(char *)self->zst.next_in, self->zst.avail_in);
if (self->unused_data == NULL) {
Py_DECREF(RetVal);
goto error;
}
self->eof = 1;
/* We will only get Z_BUF_ERROR if the output buffer was full
but there wasn't more output when we tried again, so it is
not an error condition.
*/
} else if (err != Z_OK && err != Z_BUF_ERROR) {
zlib_error(self->zst, err, "while decompressing data");
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
}
error:
LEAVE_ZLIB(self);
error_outer:
PyBuffer_Release(&pinput);
return RetVal;
}
static PyObject *
PyZlib_flush(compobject *self, PyObject *args)
{
int err, length = DEFAULTALLOC;
PyObject *RetVal;
int flushmode = Z_FINISH;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "|i:flush", &flushmode))
return NULL;
/* Flushing with Z_NO_FLUSH is a no-op, so there's no point in
doing any work at all; just return an empty string. */
if (flushmode == Z_NO_FLUSH) {
return PyBytes_FromStringAndSize(NULL, 0);
}
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
return NULL;
ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = 0;
self->zst.avail_out = length;
self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal);
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), flushmode);
Py_END_ALLOW_THREADS
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length;
length = length << 1;
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), flushmode);
Py_END_ALLOW_THREADS
}
/* If flushmode is Z_FINISH, we also have to call deflateEnd() to free
various data structures. Note we should only get Z_STREAM_END when
flushmode is Z_FINISH, but checking both for safety*/
if (err == Z_STREAM_END && flushmode == Z_FINISH) {
err = deflateEnd(&(self->zst));
if (err != Z_OK) {
zlib_error(self->zst, err, "while finishing compression");
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
else
self->is_initialised = 0;
/* We will only get Z_BUF_ERROR if the output buffer was full
but there wasn't more output when we tried again, so it is
not an error condition.
*/
} else if (err!=Z_OK && err!=Z_BUF_ERROR) {
zlib_error(self->zst, err, "while flushing");
Py_DECREF(RetVal);
RetVal = NULL;
goto error;
}
if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
Py_DECREF(RetVal);
RetVal = NULL;
}
error:
LEAVE_ZLIB(self);
return RetVal;
}
static PyObject *
PyZlib_unflush(compobject *self, PyObject *args)
{
int err, length = DEFAULTALLOC;
PyObject * retval = NULL;
unsigned long start_total_out;
if (!PyArg_ParseTuple(args, "|i:flush", &length))
return NULL;
if (length <= 0) {
PyErr_SetString(PyExc_ValueError, "length must be greater than zero");
return NULL;
}
if (!(retval = PyBytes_FromStringAndSize(NULL, length)))
return NULL;
ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_out = length;
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval);
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH);
Py_END_ALLOW_THREADS
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&retval, length << 1) < 0) {
Py_DECREF(retval);
retval = NULL;
goto error;
}
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval) + length;
self->zst.avail_out = length;
length = length << 1;
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH);
Py_END_ALLOW_THREADS
}
/* If at end of stream, clean up any memory allocated by zlib. */
if (err == Z_STREAM_END) {
self->eof = 1;
self->is_initialised = 0;
err = inflateEnd(&(self->zst));
if (err != Z_OK) {
zlib_error(self->zst, err, "while finishing decompression");
Py_DECREF(retval);
retval = NULL;
goto error;
}
}
if (_PyBytes_Resize(&retval, self->zst.total_out - start_total_out) < 0) {
Py_DECREF(retval);
retval = NULL;
}
error:
LEAVE_ZLIB(self);
return retval;
}
BOOST_NORETURN inline void throw_zlib_error(int code, const char * msg) { throw zlib_error(code, msg); }
