/**
* \brief Create a cipher \c codec_t object
*
* Create a gzip \c codec_t object at \p *piz suitable for compression or
* decompression (depending on \p op).
*
* \param op one of \c GZIP_COMPRESS or \c GZIP_UNCOMPRESS
* \param piz the created codec as a value-result arguement
*
* \return \c 0 on success, \c ~0 otherwise
*/
int codec_gzip_create(int op, codec_t **piz)
{
codec_gzip_t *iz = NULL;
dbg_return_if (piz == NULL, ~0);
iz = u_zalloc(sizeof(codec_gzip_t));
dbg_err_if(iz == NULL);
iz->codec.transform = gzip_transform;
iz->codec.flush = gzip_flush;
iz->codec.free = gzip_free;
iz->action = op;
switch(op)
{
case GZIP_COMPRESS:
iz->op = deflate;
iz->opEnd = deflateEnd;
dbg_err_if(deflateInit2(&iz->zstr, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
-MAX_WBITS, 8, Z_DEFAULT_STRATEGY));
break;
case GZIP_UNCOMPRESS:
iz->op = inflate;
iz->opEnd = inflateEnd;
dbg_err_if(inflateInit2(&iz->zstr, -MAX_WBITS) != Z_OK);
break;
default:
dbg_err_if("bad gzip op");
}
*piz = (codec_t*)iz;
return 0;
err:
U_FREE(iz);
return ~0;
}
int shooter_compress(const char* src, int srcLen, char* dst, int dstLen)
{
z_stream strm;
// zalloc, zfree, opaque, use default
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = srcLen;
strm.avail_out = dstLen;
strm.next_in = (Bytef*)src;
strm.next_out = (Bytef*)dst;
int err = -1;
// init z_stream
// deflateInit:zlib, deflateInit2:gzip
// err = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
err = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (err == Z_OK) {
err = deflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
deflateEnd(&strm);
return dstLen - strm.avail_out;
}
else {
deflateEnd(&strm);
m_ErrMsg = "compression failed, deflate return: ";
return -1;
}
}
else {
(void)deflateEnd(&strm);
m_ErrMsg = "compression initialization failed, quit!";
return 0;
}
}
static void
gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
{
int l, i;
z_stream vz;
memset(&vz, 0, sizeof vz);
l = strlen(txt);
*body = calloc(l + OVERHEAD, 1);
AN(*body);
vz.next_in = TRUST_ME(txt);
vz.avail_in = l;
vz.next_out = TRUST_ME(*body);
vz.avail_out = l + OVERHEAD;
assert(Z_OK == deflateInit2(&vz,
hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
i = vz.stop_bit & 7;
if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
vtc_log(hp->vl, hp->fatal,
"Wrong gzip residual got %d wanted %d",
i, hp->gzipresidual);
*bodylen = vz.total_out;
vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
(uintmax_t)vz.start_bit,
(uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
(uintmax_t)vz.last_bit,
(uintmax_t)vz.last_bit >> 3, (uintmax_t)vz.last_bit & 7);
vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
(uintmax_t)vz.stop_bit,
(uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
assert(Z_OK == deflateEnd(&vz));
}
void
xxx(void)
{
z_stream vz;
int n;
char ibuf[200];
char obuf[200];
int fl[8];
int i, j;
for (n = 0; n < 8; n++)
fl[n] = 9999;
memset(&vz, 0, sizeof vz);
for(n = 0; n < 999999999; n++) {
*ibuf = 0;
for (j = 0; j < 7; j++) {
sprintf(strchr(ibuf, 0), "%x",
(unsigned)random() & 0xffff);
vz.next_in = TRUST_ME(ibuf);
vz.avail_in = strlen(ibuf);
vz.next_out = TRUST_ME(obuf);
vz.avail_out = sizeof obuf;
assert(Z_OK == deflateInit2(&vz,
9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
i = vz.stop_bit & 7;
if (fl[i] > strlen(ibuf)) {
printf("%d %jd <%s>\n", i, vz.stop_bit, ibuf);
fl[i] = strlen(ibuf);
}
assert(Z_OK == deflateEnd(&vz));
}
}
printf("FOO\n");
}
int gzipProcessorInit(TProcessingModule *ProcMod, const char *Args)
{
int result=FALSE;
#ifdef HAVE_LIBZ
zlibData *ZData;
int CompressionLevel=5;
char *ptr, *Name=NULL, *Value=NULL;
ptr=GetNameValuePair(Args,"\\S","=",&Name,&Value);
while (ptr)
{
if (strcasecmp(Name,"CompressionLevel")==0) CompressionLevel=atoi(Value);
if (strcasecmp(Name,"Level")==0) CompressionLevel=atoi(Value);
ptr=GetNameValuePair(ptr,"\\S","=",&Name,&Value);
}
ProcMod->ReadMax=4096;
ProcMod->WriteMax=4096;
ZData=(zlibData *) calloc(1,sizeof(zlibData));
ZData->z_in.avail_in=0;
ZData->z_in.avail_out=0;
result=inflateInit2(&ZData->z_in,47);
ZData->z_out.avail_in=0;
ZData->z_out.avail_out=0;
deflateInit2(&ZData->z_out,5,Z_DEFLATED,30,8,Z_DEFAULT_STRATEGY);
ProcMod->Data=(void *) ZData;
result=TRUE;
DestroyString(Name);
DestroyString(Value);
#endif
return(result);
}
static PyObject *
PyZlib_compressobj(PyObject *selfptr, PyObject *args)
{
compobject *self;
int level=Z_DEFAULT_COMPRESSION, method=DEFLATED;
int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err;
if (!PyArg_ParseTuple(args, "|iiiii:compressobj", &level, &method, &wbits,
&memLevel, &strategy))
return NULL;
self = newcompobject(&Comptype);
if (self==NULL)
return(NULL);
self->zst.zalloc = (alloc_func)NULL;
self->zst.zfree = (free_func)Z_NULL;
self->zst.next_in = NULL;
self->zst.avail_in = 0;
err = deflateInit2(&self->zst, level, method, wbits, memLevel, strategy);
switch(err) {
case (Z_OK):
self->is_initialised = 1;
return (PyObject*)self;
case (Z_MEM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory for compression object");
return NULL;
case(Z_STREAM_ERROR):
Py_DECREF(self);
PyErr_SetString(PyExc_ValueError, "Invalid initialization option");
return NULL;
default:
zlib_error(self->zst, err, "while creating compression object");
Py_DECREF(self);
return NULL;
}
}
/* Initialize state for writing a gzip file. Mark initialization by setting
state->size to non-zero. Return -1 on failure or 0 on success. */
local int gz_init(gz_statep state)
{
int ret;
z_streamp strm = &(state->strm);
/* allocate input and output buffers */
state->in = (unsigned char *)malloc(state->want);
state->out = (unsigned char *)malloc(state->want);
if (state->in == NULL || state->out == NULL) {
if (state->out != NULL)
free(state->out);
if (state->in != NULL)
free(state->in);
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
/* allocate deflate memory, set up for gzip compression */
strm->zalloc = Z_NULL;
strm->zfree = Z_NULL;
strm->opaque = Z_NULL;
ret = deflateInit2(strm, state->level, Z_DEFLATED,
15 + 16, 8, state->strategy);
if (ret != Z_OK) {
free(state->in);
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
/* mark state as initialized */
state->size = state->want;
/* initialize write buffer */
strm->avail_out = state->size;
strm->next_out = state->out;
state->next = strm->next_out;
return 0;
}
// ------ Extern function implementation -------
bool
Zlib_compress (
Zlib *self,
void *data,
size_t dataSize
) {
z_stream stream;
stream.next_in = data;
stream.avail_in = dataSize;
stream.avail_out = sizeof (self->buffer);
stream.next_out = self->buffer;
stream.total_in = 0;
stream.total_out = 0;
stream.zalloc = 0;
stream.zfree = 0;
int result;
if ((result = deflateInit2 (&stream, 1, Z_DEFLATED, -15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) != Z_OK) {
warning ("Can't init compression : error code = %x.", result);
return false;
}
if ((result = deflate (&stream, Z_FINISH) != Z_STREAM_END)) {
warning ("Can't compress : error code = %x.", result);
return false;
}
if ((result = deflateEnd (&stream) != Z_OK)) {
warning ("Can't end compression : error code = %x", result);
return false;
}
self->header.magic = ZLIB_MAGIC_HEADER;
self->header.size = stream.total_out;
return true;
}
ContentReaderState(RESTContentReaderCallback callback_write,
RESTRequestFree callback_free,
void* callback_data,
bool gzip = false)
: callback_write_(callback_write),
callback_free_(callback_free),
callback_data_(callback_data),
gzip_(gzip),
callback_finished_(false),
z_finished_(false)
{
if (gzip_)
{
zstrm_.zalloc = Z_NULL;
zstrm_.zfree = Z_NULL;
zstrm_.opaque = Z_NULL;
/* NOTE: adding 16 to 'windowBits' parameter enables gzip compression */
int rc = deflateInit2(&zstrm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + 15, 8, Z_DEFAULT_STRATEGY);
if (rc != Z_OK)
throw std::runtime_error("Illegal state: failed to initialize zlib");
}
}
extern "C" JNIEXPORT jlong JNICALL
Java_java_util_zip_Deflater_make(JNIEnv* e,
jclass,
jboolean nowrap,
jint level)
{
z_stream* s = static_cast<z_stream*>(malloc(sizeof(z_stream)));
if (s == 0) {
throwNew(e, "java/lang/OutOfMemoryError", 0);
return 0;
}
memset(s, 0, sizeof(z_stream));
int r = deflateInit2(s, level, (nowrap ? -15 : 15));
if (r != Z_OK) {
free(s);
throwNew(e, "java/lang/RuntimeException", zError(r));
return 0;
}
return reinterpret_cast<jlong>(s);
}
int zip(u_char *in, int size, u_char *out, int maxsz) {
z_stream z;
int ret;
z.zalloc = (alloc_func)0;
z.zfree = (free_func)0;
z.opaque = (voidpf)0;
if(deflateInit2(&z, Z_BEST_COMPRESSION, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY)) {
fputs("\nError: zlib initialization error\n", stdout);
exit(1);
}
z.next_in = in;
z.avail_in = size;
z.next_out = out;
z.avail_out = maxsz;
deflate(&z, Z_FINISH);
ret = z.total_out;
deflateEnd(&z);
return(ret);
}
static int lz_deflate_new(lua_State *L) {
int level = luaL_optint(L, 1, Z_DEFAULT_COMPRESSION);
int window_size = luaL_optint(L, 2, MAX_WBITS);
/* Allocate the stream: */
z_stream* stream = (z_stream*)lua_newuserdata(L, sizeof(z_stream));
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
int result = deflateInit2(stream, level, Z_DEFLATED, window_size,
DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
lz_assert(L, result, stream, __FILE__, __LINE__);
/* Don't allow destructor to execute unless deflateInit2 was successful: */
luaL_getmetatable(L, "lz.deflate.meta");
lua_setmetatable(L, -2);
lua_pushnil(L);
lua_pushcclosure(L, lz_deflate, 2);
return 1;
}
DeflatingStreamBuf::DeflatingStreamBuf(std::ostream& ostr, StreamType type, int level):
BufferedStreamBuf(STREAM_BUFFER_SIZE, std::ios::out),
_pIstr(0),
_pOstr(&ostr),
_eof(false)
{
_zstr.zalloc = Z_NULL;
_zstr.zfree = Z_NULL;
_zstr.opaque = Z_NULL;
_zstr.next_in = 0;
_zstr.avail_in = 0;
_zstr.next_out = 0;
_zstr.avail_out = 0;
_buffer = new char[DEFLATE_BUFFER_SIZE];
int rc = deflateInit2(&_zstr, level, Z_DEFLATED, 15 + (type == STREAM_GZIP ? 16 : 0), 8, Z_DEFAULT_STRATEGY);
if (rc != Z_OK)
{
delete [] _buffer;
throw IOException(zError(rc));
}
}
GZipWriteStream(Common::WriteStream *w) : _wrapped(w) {
assert(w != 0);
_stream.zalloc = Z_NULL;
_stream.zfree = Z_NULL;
_stream.opaque = Z_NULL;
// Adding 16 to windowBits indicates to zlib that it is supposed to
// write gzip headers. This feature was added in zlib 1.2.0.4,
// released 10 August 2003.
// Note: This is *crucial* for savegame compatibility, do *not* remove!
_zlibErr = deflateInit2(&_stream,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
MAX_WBITS + 16,
8,
Z_DEFAULT_STRATEGY);
assert(_zlibErr == Z_OK);
_stream.next_out = _buf;
_stream.avail_out = BUFSIZE;
_stream.avail_in = 0;
_stream.next_in = 0;
}
JNIEXPORT jlong JNICALL
Java_java_util_zip_Deflater_init(JNIEnv *env, jclass cls, jint level,
jint strategy, jboolean nowrap)
{
z_stream *strm = calloc(1, sizeof(z_stream));
if (strm == 0) {
JNU_ThrowOutOfMemoryError(env, 0);
return jlong_zero;
} else {
const char *msg;
int ret = deflateInit2(strm, level, Z_DEFLATED,
nowrap ? -MAX_WBITS : MAX_WBITS,
DEF_MEM_LEVEL, strategy);
switch (ret) {
case Z_OK:
return ptr_to_jlong(strm);
case Z_MEM_ERROR:
free(strm);
JNU_ThrowOutOfMemoryError(env, 0);
return jlong_zero;
case Z_STREAM_ERROR:
free(strm);
JNU_ThrowIllegalArgumentException(env, 0);
return jlong_zero;
default:
msg = ((strm->msg != NULL) ? strm->msg :
(ret == Z_VERSION_ERROR) ?
"zlib returned Z_VERSION_ERROR: "
"compile time and runtime zlib implementations differ" :
"unknown error initializing zlib library");
free(strm);
JNU_ThrowInternalError(env, msg);
return jlong_zero;
}
}
}
bool KGzipFilter::init(int mode, Flag flag)
{
if (d->isInitialized) {
terminate();
}
d->zStream.next_in = Z_NULL;
d->zStream.avail_in = 0;
if ( mode == QIODevice::ReadOnly )
{
const int windowBits = (flag == RawDeflate)
? -MAX_WBITS /*no zlib header*/
: (flag == GZipHeader) ?
MAX_WBITS + 32 /* auto-detect and eat gzip header */
: MAX_WBITS /*zlib header*/;
const int result = inflateInit2(&d->zStream, windowBits);
if ( result != Z_OK ) {
//qDebug() << "inflateInit2 returned " << result;
return false;
}
} else if ( mode == QIODevice::WriteOnly )
{
int result = deflateInit2(&d->zStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY); // same here
if ( result != Z_OK ) {
//qDebug() << "deflateInit returned " << result;
return false;
}
} else {
//qWarning() << "KGzipFilter: Unsupported mode " << mode << ". Only QIODevice::ReadOnly and QIODevice::WriteOnly supported";
return false;
}
d->mode = mode;
d->compressed = true;
d->headerWritten = false;
d->footerWritten = false;
d->isInitialized = true;
return true;
}
int ZLibCompressChunk(byte *in, int len_in, byte *out, int len_out, int method, int wbits)
{
z_stream zs;
int result;
zs.next_in = in;
zs.avail_in = len_in;
zs.total_in = 0;
zs.next_out = out;
zs.avail_out = len_out;
zs.total_out = 0;
zs.msg = NULL;
zs.state = NULL;
zs.zalloc = Z_NULL;
zs.zfree = Z_NULL;
zs.opaque = NULL;
zs.data_type = Z_BINARY;
zs.adler = 0;
zs.reserved = 0;
result = deflateInit2 (&zs, method, Z_DEFLATED, wbits, 9, Z_DEFAULT_STRATEGY);
if (result != Z_OK)
return -1;
result = deflate(&zs, Z_FINISH);
if (result != Z_STREAM_END)
return -1;
result = deflateEnd(&zs);
if (result != Z_OK)
return -1;
return zs.total_out;
}
struct _writer *deflatewriter_init(struct _writer *parent, unsigned char type) {
struct _deflatewriter *ret = (struct _deflatewriter*)_xalloc(sizeof(struct _deflatewriter));
int windowBits;
int err;
ret->base.destroy = &deflatewriter_destroy;
ret->base.write = &deflatewriter_write;
ret->base.resize = &deflatewriter_resize;
ret->base.seek = &deflatewriter_seek;
ret->base.tell = &deflatewriter_tell;
ret->base.error = 0;
ret->stream.zalloc = (alloc_func)&_deflatewriter_zalloc_func;
ret->stream.zfree = (free_func)&_deflatewriter_zfree_func;
ret->stream.opaque = (voidpf)ret;
ret->stream.next_in = Z_NULL;
ret->stream.avail_in = 0;
ret->parent = parent;
if (type == 255)
windowBits = -DEFLATEWRITER_WINDOW_BITS;
else
windowBits = DEFLATEWRITER_WINDOW_BITS + (int)type * 16;
err = deflateInit2(&ret->stream,
DEFLATEWRITER_LEVEL,
DEFLATEWRITER_METHOD,
windowBits,
DEFLATEWRITER_MEM_LEVEL,
DEFLATEWRITER_STRATEGY);
if (err != Z_OK) {
_deflatewriter_zerror("init",err);
ret->base.error = 1;
}
return(CAST_DOWN(ret,base));
}
std::vector<char> QblZlib::compress(std::vector<char> data)
{
std::vector<char> result;
int retval;
this->stream.next_in = Z_NULL;
this->stream.avail_in = 0;
retval = deflateInit2(&this->stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
windowBits | GZIP_ENCODING, 8, Z_DEFAULT_STRATEGY);
if (retval < 0) {
this->errorMessage(retval);
}
this->stream.next_in = (unsigned char *)data.data();
this->stream.avail_in = data.size();
do {
int n, retval;
unsigned char buffer[CHUNK];
std::vector<char> _buffer;
this->stream.avail_out = CHUNK;
this->stream.next_out = buffer;
retval = deflate(&this->stream, Z_FINISH);
if (retval < 0) {
(void)deflateEnd(&this->stream);
this->errorMessage(retval);
}
n = CHUNK - this->stream.avail_out;
_buffer.insert(_buffer.begin(), buffer, buffer + n);
result.insert(result.end(), _buffer.begin(), _buffer.end());
} while (this->stream.avail_out == 0);
(void)deflateEnd(&this->stream);
return (result);
}
ZStream *
CreateZStream (gint compress, guchar gzip, read_write_func func, void *gchandle)
{
z_stream *z;
gint retval;
ZStream *result;
if (func == NULL)
return NULL;
#if !defined(ZLIB_VERNUM) || (ZLIB_VERNUM < 0x1204)
/* Older versions of zlib do not support raw deflate or gzip */
return NULL;
#endif
z = g_new0 (z_stream, 1);
if (compress) {
retval = deflateInit2 (z, Z_DEFAULT_COMPRESSION, Z_DEFLATED, gzip ? 31 : -15, 8, Z_DEFAULT_STRATEGY);
} else {
retval = inflateInit2 (z, gzip ? 31 : -15);
}
if (retval != Z_OK) {
g_free (z);
return NULL;
}
z->zalloc = z_alloc;
z->zfree = z_free;
result = g_new0 (ZStream, 1);
result->stream = z;
result->func = func;
result->gchandle = gchandle;
result->compress = compress;
result->buffer = g_new (guchar, BUFFER_SIZE);
return result;
}
/**
* OCaml binding for _zlib_'s `deflateInit2` function.
*
* This creates a new stream and initializes it for deflate.
*
* This function may raise the following OCaml exceptions:
* - Out_of_memory exception
* - Failure exception: Invalid parameters
* - Failure exception: Invalid version
* - Failure exception: Unknown zlib return code
*
* See:
* https://github.com/madler/zlib/blob/cacf7f1d4e3d44d871b605da3b647f07d718623f/zlib.h#L538
*
* @param levelVal {value} OCaml `int`: the compression level, must be in the range 0..9.
* 0 gives no compression at all, 1 the best speed, 9 the best compression.
* @param windowBitsVal {value} OCaml `int`: base two logarithm of the window size (size of the
* history buffer) used by _zlib_. It should be in the range 9..15 for this version of _zlib_.
* It can also be in the range -15..-8 (the absolute value is used) for raw deflate.
* Finally, it can be greater than 15 for gzip encoding. See _zlib_'s documentation for
* `deflateInit2` for the exact documentation.
* @return {value} An OCaml value representing the new stream, initialized for deflate.
*/
CAMLprim value zlib_deflate_init2(value level_val, value window_bits_val) {
int level = Int_val(level_val);
int window_bits = Int_val(window_bits_val);
value z_streamp_val = zlib_new_stream();
z_streamp stream = ZStreamP_val(z_streamp_val);
int deflate_init2_result = deflateInit2(
stream,
level,
Z_DEFLATED, // method
window_bits,
8, // memLevel
Z_DEFAULT_STRATEGY // strategy
);
if (deflate_init2_result == Z_OK) {
return z_streamp_val;
}
switch (deflate_init2_result) {
case Z_MEM_ERROR:
caml_raise_out_of_memory();
break;
case Z_STREAM_ERROR:
// TODO: use stream->msg to get _zlib_'s text message
failwith("Error in `zlib_deflate_init2` (extc_stubs.c): call to `deflateInit2` failed: Z_STREAM_ERROR");
break;
case Z_VERSION_ERROR:
// TODO: use stream->msg to get _zlib_'s text message
failwith("Error in `zlib_deflate_init2` (extc_stubs.c): call to `deflateInit2` failed: Z_VERSION_ERROR");
break;
default:
failwith("Error in `zlib_deflate_init2` (extc_stubs.c): unknown return code from `deflateInit2`");
}
assert(0);
}
static int zlib_compress(grpc_slice_buffer* input, grpc_slice_buffer* output,
int gzip) {
z_stream zs;
int r;
size_t i;
size_t count_before = output->count;
size_t length_before = output->length;
memset(&zs, 0, sizeof(zs));
zs.zalloc = zalloc_gpr;
zs.zfree = zfree_gpr;
r = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | (gzip ? 16 : 0),
8, Z_DEFAULT_STRATEGY);
GPR_ASSERT(r == Z_OK);
r = zlib_body(&zs, input, output, deflate) && output->length < input->length;
if (!r) {
for (i = count_before; i < output->count; i++) {
grpc_slice_unref(output->slices[i]);
}
output->count = count_before;
output->length = length_before;
}
deflateEnd(&zs);
return r;
}
static int sc_compress_gzip(u8* out, size_t* outLen, const u8* in, size_t inLen) {
/* Since compress does not offer a way to make it compress gzip... manually set it up */
z_stream gz;
int err;
int window_size = 15 + 0x10;
memset(&gz, 0, sizeof(gz));
gz.next_in = (u8*)in;
gz.avail_in = inLen;
gz.next_out = out;
gz.avail_out = *outLen;
err = deflateInit2(&gz, Z_BEST_COMPRESSION, Z_DEFLATED, window_size, 9, Z_DEFAULT_STRATEGY);
if(err != Z_OK) return zerr_to_opensc(err);
err = deflate(&gz, Z_FINISH);
if(err != Z_STREAM_END) {
deflateEnd(&gz);
return zerr_to_opensc(err);
}
*outLen = gz.total_out;
err = deflateEnd(&gz);
return zerr_to_opensc(err);
}
/*
* GZIP compression.
*/
long
gz_compress (unsigned char *src, unsigned long slen,
unsigned char *dest, int level)
{
#define CHUNK 16384
int ret;
unsigned int have;
z_stream strm;
unsigned char out[CHUNK];
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit2 (&strm, level, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY);
if (ret != Z_OK)
return -1;
strm.avail_in = slen;
strm.next_in = src;
do
{
strm.avail_out = sizeof out;
strm.next_out = out;
ret = deflate (&strm, Z_FINISH);
if (ret == Z_STREAM_ERROR)
return -1;
have = sizeof (out) - strm.avail_out;
std::memcpy (dest, out, have);
dest += have;
}
while (strm.avail_out == 0);
deflateEnd (&strm);
return strm.total_out;
}
void CompressedBuffer::compressBytes(unsigned char *inputB, long long length, bool compress) {
if(compress) {
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
// int ret = deflateInit(&strm, 9); // compression level 9 (0-9, -1 auto)
int ret = deflateInit2(&strm, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); // Z_DEFAULT_STRATEGY or Z_HUFFMAN_ONLY
/* if(ret != Z_OK) {
printf("Error: DeflateInit failed.\n");
return;
}
*/ strm.avail_in = (unsigned int)length;
strm.next_in = inputB;
strm.avail_out = (unsigned int)(size - offset);
strm.next_out = buffer + offset;
ret = deflate(&strm, Z_FINISH); // we make it only once, so Z_FLUSH
offset += (size - offset) - strm.avail_out;
deflateEnd(&strm);
} else {
memcpy(buffer+offset, inputB, length);
offset += length;
}
}
/** Construct and return a tor_zlib_state_t object using <b>method</b>. If
* <b>compress</b>, it's for compression; otherwise it's for
* decompression. */
tor_zlib_state_t *
tor_zlib_new(int compress, compress_method_t method)
{
tor_zlib_state_t *out;
int bits;
if (method == GZIP_METHOD && !is_gzip_supported()) {
/* Old zlib version don't support gzip in inflateInit2 */
log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
return NULL;
}
out = tor_malloc_zero(sizeof(tor_zlib_state_t));
out->stream.zalloc = Z_NULL;
out->stream.zfree = Z_NULL;
out->stream.opaque = NULL;
out->compress = compress;
bits = method_bits(method);
if (compress) {
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)
goto err;
} else {
if (inflateInit2(&out->stream, bits) != Z_OK)
goto err;
}
out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);
total_zlib_allocation += out->allocation;
return out;
err:
tor_free(out);
return NULL;
}
nsresult nsCacheEntryDescriptor::
nsCompressOutputStreamWrapper::InitZstream()
{
// Determine compression level: Aggressive compression
// may impact performance on mobile devices, while a
// lower compression level still provides substantial
// space savings for many text streams.
int32_t compressionLevel = nsCacheService::CacheCompressionLevel();
// Initialize zlib deflate stream
mZstream.zalloc = Z_NULL;
mZstream.zfree = Z_NULL;
mZstream.opaque = Z_NULL;
if (deflateInit2(&mZstream, compressionLevel, Z_DEFLATED,
MAX_WBITS, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
return NS_ERROR_FAILURE;
}
mZstream.next_in = Z_NULL;
mZstream.avail_in = 0;
mStreamInitialized = true;
return NS_OK;
}
void
Compressor::compress(std::string & in, std::string & out)
{
switch (m_compression_codec) {
case CompressionCodec::UNCOMPRESSED:
{
// Just swap the input and output collections ...
out.swap(in);
}
break;
case CompressionCodec::SNAPPY:
{
size_t compressed_size;
m_tmp.resize(snappy::MaxCompressedLength(in.size()));
snappy::RawCompress((char const *) in.data(),
in.size(),
(char *) m_tmp.data(),
&compressed_size);
m_tmp.resize(compressed_size);
out.assign(m_tmp.begin(), m_tmp.end());
}
break;
case CompressionCodec::GZIP:
{
int window_bits = 15 + 16; // maximum window + GZIP
z_stream stream;
memset(&stream, '\0', sizeof(stream));
int rv = deflateInit2(&stream,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
window_bits,
9,
Z_DEFAULT_STRATEGY);
if (rv != Z_OK) {
cerr << "deflateInit2 failed: " << rv;
exit(1);
}
m_tmp.resize(deflateBound(&stream, in.size()));
stream.next_in =
const_cast<Bytef*>(reinterpret_cast<const Bytef*>(in.data()));
stream.avail_in = in.size();
stream.next_out = (Bytef*) m_tmp.data();
stream.avail_out = m_tmp.size();
rv = deflate(&stream, Z_FINISH);
if (rv != Z_STREAM_END) {
cerr << "gzip deflate failed: " << rv;
exit(1);
}
out.assign(m_tmp.begin(), m_tmp.begin() + stream.total_out);
deflateEnd(&stream);
}
break;
default:
cerr << "unsupported compression codec: " << int(m_compression_codec);
exit(1);
break;
}
}
static int
archive_write_zip_header(struct archive_write *a, struct archive_entry *entry)
{
struct zip *zip;
uint8_t h[SIZE_LOCAL_FILE_HEADER];
uint8_t e[SIZE_EXTRA_DATA_LOCAL];
uint8_t *d;
struct zip_file_header_link *l;
struct archive_string_conv *sconv;
int ret, ret2 = ARCHIVE_OK;
int64_t size;
mode_t type;
/* Entries other than a regular file or a folder are skipped. */
type = archive_entry_filetype(entry);
if (type != AE_IFREG && type != AE_IFDIR && type != AE_IFLNK) {
archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
"Filetype not supported");
return ARCHIVE_FAILED;
};
/* Directory entries should have a size of 0. */
if (type == AE_IFDIR)
archive_entry_set_size(entry, 0);
zip = a->format_data;
/* Setup default conversion. */
if (zip->opt_sconv == NULL && !zip->init_default_conversion) {
zip->sconv_default =
archive_string_default_conversion_for_write(&(a->archive));
zip->init_default_conversion = 1;
}
if (zip->flags == 0) {
/* Initialize the general purpose flags. */
zip->flags = ZIP_FLAGS;
if (zip->opt_sconv != NULL) {
if (strcmp(archive_string_conversion_charset_name(
zip->opt_sconv), "UTF-8") == 0)
zip->flags |= ZIP_FLAGS_UTF8_NAME;
#if HAVE_NL_LANGINFO
} else if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) {
zip->flags |= ZIP_FLAGS_UTF8_NAME;
#endif
}
}
d = zip->data_descriptor;
size = archive_entry_size(entry);
zip->remaining_data_bytes = size;
/* Append archive entry to the central directory data. */
l = (struct zip_file_header_link *) malloc(sizeof(*l));
if (l == NULL) {
archive_set_error(&a->archive, ENOMEM,
"Can't allocate zip header data");
return (ARCHIVE_FATAL);
}
#if defined(_WIN32) && !defined(__CYGWIN__)
/* Make sure the path separators in pahtname, hardlink and symlink
* are all slash '/', not the Windows path separator '\'. */
l->entry = __la_win_entry_in_posix_pathseparator(entry);
if (l->entry == entry)
l->entry = archive_entry_clone(entry);
#else
l->entry = archive_entry_clone(entry);
#endif
if (l->entry == NULL) {
archive_set_error(&a->archive, ENOMEM,
"Can't allocate zip header data");
free(l);
return (ARCHIVE_FATAL);
}
l->flags = zip->flags;
if (zip->opt_sconv != NULL)
sconv = zip->opt_sconv;
else
sconv = zip->sconv_default;
if (sconv != NULL) {
const char *p;
size_t len;
if (archive_entry_pathname_l(entry, &p, &len, sconv) != 0) {
if (errno == ENOMEM) {
archive_entry_free(l->entry);
free(l);
archive_set_error(&a->archive, ENOMEM,
"Can't allocate memory for Pathname");
return (ARCHIVE_FATAL);
}
archive_set_error(&a->archive,
ARCHIVE_ERRNO_FILE_FORMAT,
"Can't translate Pathname '%s' to %s",
archive_entry_pathname(entry),
archive_string_conversion_charset_name(sconv));
ret2 = ARCHIVE_WARN;
}
if (len > 0)
archive_entry_set_pathname(l->entry, p);
/*
* Although there is no character-set regulation for Symlink,
* it is suitable to convert a character-set of Symlinke to
* what those of the Pathname has been converted to.
*/
if (type == AE_IFLNK) {
if (archive_entry_symlink_l(entry, &p, &len, sconv)) {
if (errno == ENOMEM) {
archive_entry_free(l->entry);
free(l);
archive_set_error(&a->archive, ENOMEM,
"Can't allocate memory "
" for Symlink");
return (ARCHIVE_FATAL);
}
/*
* Even if the strng conversion failed,
* we should not report the error since
* thre is no regulation for.
*/
} else if (len > 0)
archive_entry_set_symlink(l->entry, p);
}
}
/* If all characters in a filename are ASCII, Reset UTF-8 Name flag. */
if ((l->flags & ZIP_FLAGS_UTF8_NAME) != 0 &&
is_all_ascii(archive_entry_pathname(l->entry)))
l->flags &= ~ZIP_FLAGS_UTF8_NAME;
/* Initialize the CRC variable and potentially the local crc32(). */
l->crc32 = crc32(0, NULL, 0);
if (type == AE_IFLNK) {
const char *p = archive_entry_symlink(l->entry);
if (p != NULL)
size = strlen(p);
else
size = 0;
zip->remaining_data_bytes = 0;
archive_entry_set_size(l->entry, size);
l->compression = COMPRESSION_STORE;
l->compressed_size = size;
} else {
l->compression = zip->compression;
l->compressed_size = 0;
}
l->next = NULL;
if (zip->central_directory == NULL) {
zip->central_directory = l;
} else {
zip->central_directory_end->next = l;
}
zip->central_directory_end = l;
/* Store the offset of this header for later use in central
* directory. */
l->offset = zip->written_bytes;
memset(h, 0, sizeof(h));
archive_le32enc(&h[LOCAL_FILE_HEADER_SIGNATURE],
ZIP_SIGNATURE_LOCAL_FILE_HEADER);
archive_le16enc(&h[LOCAL_FILE_HEADER_VERSION], ZIP_VERSION_EXTRACT);
archive_le16enc(&h[LOCAL_FILE_HEADER_FLAGS], l->flags);
archive_le16enc(&h[LOCAL_FILE_HEADER_COMPRESSION], l->compression);
archive_le32enc(&h[LOCAL_FILE_HEADER_TIMEDATE],
dos_time(archive_entry_mtime(entry)));
archive_le16enc(&h[LOCAL_FILE_HEADER_FILENAME_LENGTH],
(uint16_t)path_length(l->entry));
switch (l->compression) {
case COMPRESSION_STORE:
/* Setting compressed and uncompressed sizes even when
* specification says to set to zero when using data
* descriptors. Otherwise the end of the data for an
* entry is rather difficult to find. */
archive_le32enc(&h[LOCAL_FILE_HEADER_COMPRESSED_SIZE],
(uint32_t)size);
archive_le32enc(&h[LOCAL_FILE_HEADER_UNCOMPRESSED_SIZE],
(uint32_t)size);
break;
#ifdef HAVE_ZLIB_H
case COMPRESSION_DEFLATE:
archive_le32enc(&h[LOCAL_FILE_HEADER_UNCOMPRESSED_SIZE],
(uint32_t)size);
zip->stream.zalloc = Z_NULL;
zip->stream.zfree = Z_NULL;
zip->stream.opaque = Z_NULL;
zip->stream.next_out = zip->buf;
zip->stream.avail_out = (uInt)zip->len_buf;
if (deflateInit2(&zip->stream, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
archive_set_error(&a->archive, ENOMEM,
"Can't init deflate compressor");
return (ARCHIVE_FATAL);
}
break;
#endif
}
/* Formatting extra data. */
archive_le16enc(&h[LOCAL_FILE_HEADER_EXTRA_LENGTH], sizeof(e));
archive_le16enc(&e[EXTRA_DATA_LOCAL_TIME_ID],
ZIP_SIGNATURE_EXTRA_TIMESTAMP);
archive_le16enc(&e[EXTRA_DATA_LOCAL_TIME_SIZE], 1 + 4 * 3);
e[EXTRA_DATA_LOCAL_TIME_FLAG] = 0x07;
archive_le32enc(&e[EXTRA_DATA_LOCAL_MTIME],
(uint32_t)archive_entry_mtime(entry));
archive_le32enc(&e[EXTRA_DATA_LOCAL_ATIME],
(uint32_t)archive_entry_atime(entry));
archive_le32enc(&e[EXTRA_DATA_LOCAL_CTIME],
(uint32_t)archive_entry_ctime(entry));
archive_le16enc(&e[EXTRA_DATA_LOCAL_UNIX_ID],
ZIP_SIGNATURE_EXTRA_NEW_UNIX);
archive_le16enc(&e[EXTRA_DATA_LOCAL_UNIX_SIZE], 1 + (1 + 4) * 2);
e[EXTRA_DATA_LOCAL_UNIX_VERSION] = 1;
e[EXTRA_DATA_LOCAL_UNIX_UID_SIZE] = 4;
archive_le32enc(&e[EXTRA_DATA_LOCAL_UNIX_UID],
(uint32_t)archive_entry_uid(entry));
e[EXTRA_DATA_LOCAL_UNIX_GID_SIZE] = 4;
archive_le32enc(&e[EXTRA_DATA_LOCAL_UNIX_GID],
(uint32_t)archive_entry_gid(entry));
archive_le32enc(&d[DATA_DESCRIPTOR_UNCOMPRESSED_SIZE],
(uint32_t)size);
ret = __archive_write_output(a, h, sizeof(h));
if (ret != ARCHIVE_OK)
return (ARCHIVE_FATAL);
zip->written_bytes += sizeof(h);
ret = write_path(l->entry, a);
if (ret <= ARCHIVE_OK)
return (ARCHIVE_FATAL);
zip->written_bytes += ret;
ret = __archive_write_output(a, e, sizeof(e));
if (ret != ARCHIVE_OK)
return (ARCHIVE_FATAL);
zip->written_bytes += sizeof(e);
if (type == AE_IFLNK) {
const unsigned char *p;
p = (const unsigned char *)archive_entry_symlink(l->entry);
ret = __archive_write_output(a, p, (size_t)size);
if (ret != ARCHIVE_OK)
return (ARCHIVE_FATAL);
zip->written_bytes += size;
l->crc32 = crc32(l->crc32, p, (unsigned)size);
}
if (ret2 != ARCHIVE_OK)
return (ret2);
return (ARCHIVE_OK);
}
// Compress a character buffer using zlib in gzip format with given compression level
//
char* gzip( const char* inputBuffer, unsigned int inputSize, unsigned int* outlen, int compressionLevel)
{
if (inputBuffer == NULL || inputSize == 0)
throw MakeStringException(500, "gzip failed: input buffer is empty!");
/* Before we can begin compressing (aka "deflating") data using the zlib
functions, we must initialize zlib. Normally this is done by calling the
deflateInit() function; in this case, however, we'll use deflateInit2() so
that the compressed data will have gzip headers. This will make it easy to
decompress the data later using a tool like gunzip, WinZip, etc.
deflateInit2() accepts many parameters, the first of which is a C struct of
type "z_stream" defined in zlib.h. The properties of this struct are used to
control how the compression algorithms work. z_stream is also used to
maintain pointers to the "input" and "output" byte buffers (next_in/out) as
well as information about how many bytes have been processed, how many are
left to process, etc. */
z_stream zs; // z_stream is zlib's control structure
zs.zalloc = Z_NULL; // Set zalloc, zfree, and opaque to Z_NULL so
zs.zfree = Z_NULL; // that when we call deflateInit2 they will be
zs.opaque = Z_NULL; // updated to use default allocation functions.
zs.total_out = 0; // Total number of output bytes produced so far
/* Initialize the zlib deflation (i.e. compression) internals with deflateInit2().
The parameters are as follows:
z_streamp strm - Pointer to a zstream struct
int level - Compression level. Must be Z_DEFAULT_COMPRESSION, or between
0 and 9: 1 gives best speed, 9 gives best compression, 0 gives
no compression.
int method - Compression method. Only method supported is "Z_DEFLATED".
int windowBits - Base two logarithm of the maximum window size (the size of
the history buffer). It should be in the range 8..15. Add
16 to windowBits to write a simple gzip header and trailer
around the compressed data instead of a zlib wrapper. The
gzip header will have no file name, no extra data, no comment,
no modification time (set to zero), no header crc, and the
operating system will be set to 255 (unknown).
int memLevel - Amount of memory allocated for internal compression state.
1 uses minimum memory but is slow and reduces compression
ratio; 9 uses maximum memory for optimal speed. Default value
is 8.
int strategy - Used to tune the compression algorithm. Use the value
Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data
produced by a filter (or predictor), or Z_HUFFMAN_ONLY to
force Huffman encoding only (no string match) */
int ret = deflateInit2(&zs, compressionLevel, Z_DEFLATED, (15+16), 8, Z_DEFAULT_STRATEGY);
if (ret != Z_OK)
throwGZipException("initialization", ret);
// set the z_stream's input
zs.next_in = (Bytef*)inputBuffer;
zs.avail_in = inputSize;
// Create output memory buffer for compressed data. The zlib documentation states that
// destination buffer size must be at least 0.1% larger than avail_in plus 12 bytes.
const unsigned long outsize = (unsigned long)floorf((float)inputSize * 1.01f) + 12;
Bytef* outbuf = new Bytef[outsize];
do
{
// Store location where next byte should be put in next_out
zs.next_out = outbuf + zs.total_out;
// Calculate the amount of remaining free space in the output buffer
// by subtracting the number of bytes that have been written so far
// from the buffer's total capacity
zs.avail_out = outsize - zs.total_out;
/* deflate() compresses as much data as possible, and stops/returns when
the input buffer becomes empty or the output buffer becomes full. If
deflate() returns Z_OK, it means that there are more bytes left to
compress in the input buffer but the output buffer is full; the output
buffer should be expanded and deflate should be called again (i.e., the
loop should continue to rune). If deflate() returns Z_STREAM_END, the
end of the input stream was reached (i.e.g, all of the data has been
compressed) and the loop should stop. */
ret = deflate(&zs, Z_FINISH);
} while (ret == Z_OK);
if (ret != Z_STREAM_END) // an error occurred that was not EOS
{
// Free data structures that were dynamically created for the stream.
deflateEnd(&zs);
delete[] outbuf;
*outlen = 0;
throwGZipException("compression", ret);
}
// Free data structures that were dynamically created for the stream.
deflateEnd(&zs);
*outlen = zs.total_out;
return (char*) outbuf;
}
