int gzcompress(char **zdata, size_t *nzdata, char *data, size_t ndata)/*{{{*/
{
z_stream c_stream;
if(data && ndata > 0)
{
c_stream.zalloc = Z_NULL;
c_stream.zfree = Z_NULL;
c_stream.opaque = Z_NULL;
c_stream.next_in = data;
c_stream.avail_in = ndata;
if(
deflateInit2(&c_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 | MAX_WBITS, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK
)
return -1;
*nzdata = deflateBound(&c_stream, ndata);
*zdata = (char*) uws_malloc((*nzdata + 1) * sizeof(char));
c_stream.next_out = *zdata;
c_stream.avail_out = *nzdata;
if(Z_STREAM_END != deflate(&c_stream, Z_FINISH))
return -1;
if(deflateEnd(&c_stream) != Z_OK) return -1;
*nzdata -= c_stream.avail_out;
return 0;
}
return -1;
}/*}}}*/
int Compression::get_max_compressed_buffer_size(int p_src_size,Mode p_mode){
switch(p_mode) {
case MODE_FASTLZ: {
int ss = p_src_size+p_src_size*6/100;
if (ss<66)
ss=66;
return ss;
} break;
case MODE_DEFLATE: {
z_stream strm;
strm.zalloc = zipio_alloc;
strm.zfree = zipio_free;
strm.opaque = Z_NULL;
int err = deflateInit(&strm,Z_DEFAULT_COMPRESSION);
if (err!=Z_OK)
return -1;
int aout = deflateBound(&strm,p_src_size);
deflateEnd(&strm);
return aout;
} break;
}
ERR_FAIL_V(-1);
}
size_t MessageSet::get_worst_case_compressed_size(size_t size) const
{
switch (compression_)
{
case COMP_None:
return size;
case COMP_GZIP:
{
z_stream strm;
// Initialize gzip compression
memset(&strm, 0, sizeof(strm));
auto r = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, 15+16, // 15+16 forces zlib to add the gzip header and trailer
8, Z_DEFAULT_STRATEGY);
if (r != Z_OK) {
// Errr... not sure what else we can do in here
throw make_error_code(synkafka_error::compression_lib_error);
}
auto worst_case = deflateBound(&strm, size);
// Free memory
deflateEnd(&strm);
return worst_case;
}
case COMP_Snappy:
return snappy::MaxCompressedLength(size);
}
// GCC is apparently not smart enough to realise we implemented case for every defined value of enum above
// this is unreachable but to save the "control reaches end of non-void function" warning...
return 0;
}
long CZipCompression::EstimateCompressionBufferSize(size_t src_len)
{
#if !defined(ZLIB_VERNUM) || ZLIB_VERNUM < 0x1200
return -1;
#else
size_t header_len = 0;
int errcode = Z_OK;
if ( F_ISSET(fWriteGZipFormat) ) {
// Default empty GZIP header
header_len = 10;
}
STREAM->zalloc = (alloc_func)0;
STREAM->zfree = (free_func)0;
STREAM->opaque = (voidpf)0;
errcode = deflateInit2_(STREAM, GetLevel(), Z_DEFLATED,
header_len ? -m_WindowBits : m_WindowBits,
m_MemLevel, m_Strategy,
ZLIB_VERSION, (int)sizeof(z_stream));
if (errcode != Z_OK) {
SetError(errcode, zError(errcode));
return -1;
}
long n = (long)(deflateBound(STREAM, (unsigned long)src_len) + header_len);
deflateEnd(STREAM);
return n;
#endif
}
void deflateinit(int const rlevel)
{
level = rlevel;
if ( level >= Z_DEFAULT_COMPRESSION && level <= Z_BEST_COMPRESSION )
{
deflateinitz(&strm,level);
// search for number of bytes that will never produce more compressed space than we have
unsigned int bound = getBgzfMaxBlockSize();
while ( deflateBound(&strm,bound) > (getBgzfMaxBlockSize()-(getBgzfHeaderSize()+getBgzfFooterSize())) )
--bound;
deflbound = bound;
}
#if defined(LIBMAUS_HAVE_IGZIP)
else if ( level == libmaus::lz::IGzipDeflate::getCompressionLevel() )
{
// half a block should fit
deflbound = (getBgzfMaxBlockSize()-(getBgzfHeaderSize()+getBgzfFooterSize()))/2;
}
#endif
else
{
::libmaus::exception::LibMausException se;
se.getStream() << "BgzfDeflateZStreamBase::deflateinit(): unknown/unsupported compression level " << level << std::endl;
se.finish();
throw se;
}
}
#include <fcntl.h>
#include <math.h>
#include <time.h>
#include <zlib.h>
#include <openssl/bio.h>
#include <openssl/buffer.h>
#include "uws_utils.h"
#include "uws_memory.h"
#define OVECCOUNT 30
int wildcmp(const char* wild, const char* string){
const char* cp = NULL, *mp = NULL;
while((*string) && (*wild != '*'))
{
if((*wild != *string) && (*wild != '?'))
{
return 0;
}
wild++;
string++;
}
while(*string)
{
if(*wild == '*')
{
if(!*++wild){
return 1;
}
mp = wild;
cp = string + 1;
}
else if((*wild == *string) || (*wild == '?'))
{
wild++;
string++;
} else {
wild = mp;
string = cp++;
}
}
while(*wild == '*')
{
wild++;
}
return !*wild;
}
void setnonblocking(int sock)
{
int opts = fcntl(sock, F_GETFL);
if (opts < 0) {
printf("%d\n", sock);
exit_err("fcntl(F_GETFL)");
}
opts = (opts | O_NONBLOCK);
if (fcntl(sock, F_SETFL, opts) < 0) exit_err("fcntl(F_SETFL)");
return;
}
void setblocking(int sock)
{
int opts = fcntl(sock, F_GETFL);
if (opts < 0) exit_err("fcntl(F_GETFL)");
opts = (opts & ~O_NONBLOCK);
if (fcntl(sock, F_SETFL, opts) < 0) exit_err("fcntl(F_SETFL)");
return;
}
char *strlcat(const char *s1, const char *s2) {
char *new_str = (char*) uws_malloc((strlen(s1) + strlen(s2) + 1) * sizeof(char));
strcpy(new_str, s1);
strcat(new_str, s2);
return new_str;
}
char *itoa(const size_t data) {
size_t length = (size_t) pow(data, 0.1) + 2;
char *str = (char*) uws_malloc(length * sizeof(char));
sprintf(str, "%u", data);
return str;
}
char* get_time_string(time_t *tt) {
struct tm *cur_time;
time_t t;
char* buff = (char*) uws_malloc(sizeof(char) * 60);
if(tt == NULL) {
t = time(NULL);
tt = &t;
}
cur_time = gmtime(tt);
strftime(buff, 60, "%a, %d %b %Y %H:%M:%S GMT", cur_time);
return buff;
}
time_t parse_time_string(char *time_str) {
struct tm cur_time;
strptime(time_str, "%a, %d %b %Y %H:%M:%S GMT", &cur_time);
return mktime(&cur_time);
}
int in_int_array(int array[], int needle, int length) {
int i;
for(i = 0; i < length; i++) {
if(array[i] == needle) {
return i;
}
}
return -1;
}
int gzcompress(char **zdata, size_t *nzdata, char *data, size_t ndata)/*{{{*/
{
z_stream c_stream;
if(data && ndata > 0)
{
c_stream.zalloc = Z_NULL;
c_stream.zfree = Z_NULL;
c_stream.opaque = Z_NULL;
c_stream.next_in = data;
c_stream.avail_in = ndata;
if(
deflateInit2(&c_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 | MAX_WBITS, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK
)
return -1;
*nzdata = deflateBound(&c_stream, ndata);
*zdata = (char*) uws_malloc((*nzdata + 1) * sizeof(char));
c_stream.next_out = *zdata;
c_stream.avail_out = *nzdata;
if(Z_STREAM_END != deflate(&c_stream, Z_FINISH))
return -1;
if(deflateEnd(&c_stream) != Z_OK) return -1;
*nzdata -= c_stream.avail_out;
return 0;
}
return -1;
}/*}}}*/
int deflatecompress(char **zdata, size_t *nzdata, char *data, size_t ndata) {/*{{{*/
z_stream c_stream;
int err = 0;
if(data && ndata > 0)
{
c_stream.zalloc = Z_NULL;
c_stream.zfree = Z_NULL;
c_stream.opaque = Z_NULL;
c_stream.next_in = data;
c_stream.avail_in = ndata;
if(
deflateInit(&c_stream, Z_DEFAULT_COMPRESSION) != Z_OK
)
return -1;
*nzdata = deflateBound(&c_stream, ndata);
*zdata = (char*) uws_malloc((*nzdata + 1) * sizeof(char));
c_stream.next_out = *zdata;
c_stream.avail_out = *nzdata;
if(Z_STREAM_END != deflate(&c_stream, Z_FINISH))
return -1;
if(deflateEnd(&c_stream) != Z_OK) return -1;
*nzdata -= c_stream.avail_out;
return 0;
}
return -1;
}/*}}}*/
static void
ps_write_band(fz_context *ctx, fz_band_writer *writer_, int stride, int band_start, int band_height, const unsigned char *samples)
{
ps_band_writer *writer = (ps_band_writer *)writer_;
fz_output *out = writer->super.out;
int w = writer->super.w;
int h = writer->super.h;
int n = writer->super.n;
int x, y, i, err;
int required_input;
int required_output;
unsigned char *o;
if (band_start+band_height >= h)
band_height = h - band_start;
required_input = w*(n-1)*band_height;
required_output = (int)deflateBound(&writer->stream, required_input);
if (writer->input == NULL || writer->input_size < required_input)
{
fz_free(ctx, writer->input);
writer->input = NULL;
writer->input = fz_malloc(ctx, required_input);
writer->input_size = required_input;
}
if (writer->output == NULL || writer->output_size < required_output)
{
fz_free(ctx, writer->output);
writer->output = NULL;
writer->output = fz_malloc(ctx, required_output);
writer->output_size = required_output;
}
o = writer->input;
for (y = 0; y < band_height; y++)
{
for (x = 0; x < w; x++)
{
for (i = n-1; i > 0; i--)
*o++ = *samples++;
samples++;
}
samples += stride - w*n;
}
writer->stream.next_in = (Bytef*)writer->input;
writer->stream.avail_in = required_input;
writer->stream.next_out = (Bytef*)writer->output;
writer->stream.avail_out = (uInt)writer->output_size;
err = deflate(&writer->stream, Z_NO_FLUSH);
if (err != Z_OK)
fz_throw(ctx, FZ_ERROR_GENERIC, "compression error %d", err);
fz_write(ctx, out, writer->output, writer->output_size - writer->stream.avail_out);
}
bool pmix_compress_zlib_compress_block(char *instring,
uint8_t **outbytes,
size_t *nbytes)
{
z_stream strm;
size_t len, outlen;
uint8_t *tmp, *ptr;
uint32_t inlen;
/* set default output */
*outbytes = NULL;
/* setup the stream */
inlen = strlen(instring);
memset (&strm, 0, sizeof (strm));
deflateInit (&strm, 9);
/* get an upper bound on the required output storage */
len = deflateBound(&strm, inlen);
if (NULL == (tmp = (uint8_t*)malloc(len))) {
*outbytes = NULL;
return false;
}
strm.next_in = (uint8_t*)instring;
strm.avail_in = strlen(instring);
/* allocating the upper bound guarantees zlib will
* always successfully compress into the available space */
strm.avail_out = len;
strm.next_out = tmp;
deflate (&strm, Z_FINISH);
deflateEnd (&strm);
/* allocate 4 bytes beyond the size reqd by zlib so we
* can pass the size of the uncompressed string to the
* decompress side */
outlen = len - strm.avail_out + sizeof(uint32_t);
ptr = (uint8_t*)malloc(outlen);
if (NULL == ptr) {
free(tmp);
return false;
}
*outbytes = ptr;
*nbytes = outlen;
/* fold the uncompressed length into the buffer */
memcpy(ptr, &inlen, sizeof(uint32_t));
ptr += sizeof(uint32_t);
/* bring over the compressed data */
memcpy(ptr, tmp, outlen-sizeof(uint32_t));
free(tmp);
pmix_output_verbose(2, pmix_pcompress_base_framework.framework_output,
"COMPRESS INPUT STRING OF LEN %d OUTPUT SIZE %lu",
inlen, outlen-sizeof(uint32_t));
return true; // we did the compression
}
/*
* ZlibCompressObj
*
* Compresses data using the Zlib compression algorithm.
*
* Arguments:
* sourceObj - Pointer to a Tcl object containing the data to be compressed.
* destObj - Pointer to a Tcl object to receive the compressed data.
* level - Compression level.
* window - Maximum window size for Zlib.
*
* Return Value:
* A Zlib status code; Z_OK is returned if successful.
*/
static int
ZlibCompressObj(
Tcl_Obj *sourceObj,
Tcl_Obj *destObj,
int level,
int window
)
{
int status;
z_stream stream;
assert(sourceObj != NULL);
assert(destObj != NULL);
/*
* The next_in, opaque, zalloc, and zfree data structure members
* must be initialised prior to calling deflateInit2().
*/
stream.next_in = Tcl_GetByteArrayFromObj(sourceObj, (int *)&stream.avail_in);
stream.opaque = NULL;
stream.zalloc = ZlibAlloc;
stream.zfree = ZlibFree;
status = deflateInit2(&stream, level, Z_DEFLATED, window,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (status != Z_OK) {
return status;
}
stream.avail_out = deflateBound(&stream, stream.avail_in);
/*
* deflateBound() does not always return a sufficient buffer size when
* compressing data into the Gzip format. So this kludge will do...
*/
if (window > 15) {
stream.avail_out *= 2;
}
stream.next_out = Tcl_SetByteArrayLength(destObj, (int)stream.avail_out);
/*
* The Z_FINISH flag instructs Zlib to compress all data in a single
* pass, flush it to the output buffer, and return with Z_STREAM_END if
* successful.
*/
status = deflate(&stream, Z_FINISH);
deflateEnd(&stream);
if (status == Z_STREAM_END) {
Tcl_SetByteArrayLength(destObj, (int)stream.total_out);
return Z_OK;
}
return (status == Z_OK) ? Z_BUF_ERROR : status;
}
/*----------------------------------------------------------------------
| NPT_Zip::Deflate
+---------------------------------------------------------------------*/
NPT_Result
NPT_Zip::Deflate(const NPT_DataBuffer& in,
NPT_DataBuffer& out,
int compression_level,
Format format /* = ZLIB */)
{
// default return state
out.SetDataSize(0);
// check parameters
if (compression_level < NPT_ZIP_COMPRESSION_LEVEL_DEFAULT ||
compression_level > NPT_ZIP_COMPRESSION_LEVEL_MAX) {
return NPT_ERROR_INVALID_PARAMETERS;
}
// setup the stream
z_stream stream;
NPT_SetMemory(&stream, 0, sizeof(stream));
stream.next_in = (Bytef*)in.GetData();
stream.avail_in = (uInt)in.GetDataSize();
// setup the memory functions
stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;
// initialize the compressor
int err = deflateInit2(&stream,
compression_level,
Z_DEFLATED,
15 + (format == GZIP ? 16 : 0),
8,
Z_DEFAULT_STRATEGY);
if (err != Z_OK) return MapError(err);
// reserve an output buffer known to be large enough
out.Reserve(deflateBound(&stream, stream.avail_in) + (format==GZIP?10:0));
stream.next_out = out.UseData();
stream.avail_out = out.GetBufferSize();
// decompress
err = deflate(&stream, Z_FINISH);
if (err != Z_STREAM_END) {
deflateEnd(&stream);
return MapError(err);
}
// update the output size
out.SetDataSize(stream.total_out);
// cleanup
err = deflateEnd(&stream);
return MapError(err);
}
static TACommandVerdict deflateBound_cmd(TAThread thread,TAInputStream stream)
{
z_stream strm;
int res, is_null;
uLong sourceLen;
is_null = readZStream(&stream, &strm);
sourceLen = readULong(&stream);
/*ta_debug_printf( "next_in==%d\navail_in==%u\ntotal_in==%lu\nnext_out==%d\n"
"avail_out==%u\ntotal_out==%lu\n",
strm.next_in, strm.avail_in, strm.total_in, strm.next_out,
strm.avail_out,strm.total_out);
ta_debug_printf( "msg==%s\nstate==%d\nzalloc==%d\nzfree==%d\n"
"opaque==%d\ndata_type==%d\nadler==%lu\nreserved==%lu\n",
strm.msg, strm.state,
strm.zalloc, strm.zfree, strm.opaque, strm.data_type, strm.adler,
strm.reserved);*/
START_TARGET_OPERATION(thread);
if(!is_null)
res = deflateBound(&strm, sourceLen);
else
res = deflateBound(0, sourceLen);
END_TARGET_OPERATION(thread);
// Response
if(!is_null)
writeZStream(thread, &strm);
else
writeZStream(thread, 0);
writeInt(thread, res);
sendResponse(thread);
return taDefaultVerdict;
}
int main( int argc, char *argv[]) {
if (argc < 2) usage(argv[0]);
int rc;
size_t flen;
char *file = argv[1];
char *data = slurp(file, &flen);
/* minimal required initialization of z_stream prior to deflateInit2 */
z_stream zs = {.next_in = data, .zalloc=Z_NULL, .zfree=Z_NULL, .opaque=NULL};
#define want_gzip 16
#define def_windowbits (15 + want_gzip)
#define def_memlevel 8
rc = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, def_windowbits,
def_memlevel, Z_DEFAULT_STRATEGY);
if (rc != Z_OK) {
fprintf(stderr, "deflateInit failed: %s\n", zs.msg);
exit(-1);
}
/* calculate the max space needed to deflate this buffer in a single pass */
size_t gzmax = deflateBound(&zs, flen);
char *out = malloc(gzmax);
if (out == NULL) {
fprintf(stderr, "could not allocate %u bytes for compressed file\n",
(unsigned)gzmax);
exit(-1);
}
/* initialize the remaining parts of z_stream prior to actual deflate */
zs.avail_in = flen;
zs.next_out = out;
zs.avail_out = gzmax;
/* deflate it in one fell swoop */
rc = deflate(&zs, Z_FINISH);
if (rc != Z_STREAM_END) {
fprintf(stderr,"single-pass deflate failed: ");
if (rc == Z_OK) fprintf(stderr,"additional passes needed\n");
else if (rc == Z_STREAM_ERROR) fprintf(stderr,"stream error\n");
else if (rc == Z_BUF_ERROR) fprintf(stderr,"buffer unavailable\n");
else fprintf(stderr,"unknown error\n");
exit(-1);
}
rc = deflateEnd(&zs);
if (rc != Z_OK) fprintf(stderr,"deflateEnd error: %s\n", zs.msg);
fprintf(stderr,"Original size: %u\n", (unsigned)flen);
fprintf(stderr,"Deflated size: %u\n", (unsigned)zs.total_out);
if (write(STDOUT_FILENO, out, zs.total_out) != zs.total_out) {
fprintf(stderr,"error: partial write\n");
exit(-1);
}
return 0;
}
void fz_write_ps_band(fz_context *ctx, fz_output *out, fz_ps_output_context *psoc, int w, int h, int n, int stride, int band, int bandheight, unsigned char *samples)
{
int x, y, i, err;
int required_input;
int required_output;
unsigned char *o;
band *= bandheight;
if (band+bandheight >= h)
bandheight = h - band;
required_input = w*(n-1)*bandheight;
required_output = (int)deflateBound(&psoc->stream, required_input);
if (psoc->input == NULL || psoc->input_size < required_input)
{
fz_free(ctx, psoc->input);
psoc->input = NULL;
psoc->input = fz_malloc(ctx, required_input);
psoc->input_size = required_input;
}
if (psoc->output == NULL || psoc->output_size < required_output)
{
fz_free(ctx, psoc->output);
psoc->output = NULL;
psoc->output = fz_malloc(ctx, required_output);
psoc->output_size = required_output;
}
o = psoc->input;
for (y = 0; y < bandheight; y++)
{
for (x = 0; x < w; x++)
{
for (i = n-1; i > 0; i--)
*o++ = *samples++;
samples++;
}
samples += stride - w*n;
}
psoc->stream.next_in = (Bytef*)psoc->input;
psoc->stream.avail_in = required_input;
psoc->stream.next_out = (Bytef*)psoc->output;
psoc->stream.avail_out = (uInt)psoc->output_size;
err = deflate(&psoc->stream, Z_NO_FLUSH);
if (err != Z_OK)
fz_throw(ctx, FZ_ERROR_GENERIC, "compression error %d", err);
fz_write(ctx, out, psoc->output, psoc->output_size - psoc->stream.avail_out);
}
/*
*
* Encode a frame
*
*/
static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
const AVFrame *pict, int *got_packet)
{
LclEncContext *c = avctx->priv_data;
AVFrame * const p = &c->pic;
int i, ret;
int zret; // Zlib return code
int max_size = deflateBound(&c->zstream, avctx->width * avctx->height * 3);
if (!pkt->data &&
(ret = av_new_packet(pkt, max_size)) < 0) {
av_log(avctx, AV_LOG_ERROR, "Error allocating packet of size %d.\n", max_size);
return ret;
}
*p = *pict;
p->pict_type= AV_PICTURE_TYPE_I;
p->key_frame= 1;
if(avctx->pix_fmt != PIX_FMT_BGR24){
av_log(avctx, AV_LOG_ERROR, "Format not supported!\n");
return -1;
}
zret = deflateReset(&c->zstream);
if (zret != Z_OK) {
av_log(avctx, AV_LOG_ERROR, "Deflate reset error: %d\n", zret);
return -1;
}
c->zstream.next_out = pkt->data;
c->zstream.avail_out = pkt->size;
for(i = avctx->height - 1; i >= 0; i--) {
c->zstream.next_in = p->data[0]+p->linesize[0]*i;
c->zstream.avail_in = avctx->width*3;
zret = deflate(&c->zstream, Z_NO_FLUSH);
if (zret != Z_OK) {
av_log(avctx, AV_LOG_ERROR, "Deflate error: %d\n", zret);
return -1;
}
}
zret = deflate(&c->zstream, Z_FINISH);
if (zret != Z_STREAM_END) {
av_log(avctx, AV_LOG_ERROR, "Deflate error: %d\n", zret);
return -1;
}
pkt->size = c->zstream.total_out;
pkt->flags |= AV_PKT_FLAG_KEY;
*got_packet = 1;
return 0;
}
uint32 EstimateDeflateBuffer(uint32_t len) {
z_stream zstream;
memset(&zstream, 0, sizeof(zstream));
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
zstream.opaque = Z_NULL;
if (deflateInit(&zstream, Z_FINISH) != Z_OK)
return 0;
return deflateBound(&zstream, len);
}
WizardExport WizardBooleanType IncreaseZIP(ZIPInfo *zip_info,
const StringInfo *message,ExceptionInfo *exception)
{
int
status;
z_stream
stream;
/*
Increase the message entropy.
*/
(void) LogWizardEvent(TraceEvent,GetWizardModule(),"...");
WizardAssert(EntropyDomain,zip_info != (ZIPInfo *) NULL);
WizardAssert(EntropyDomain,zip_info->signature == WizardSignature);
WizardAssert(EntropyDomain,message != (const StringInfo *) NULL);
stream.zalloc=AcquireZIPMemory;
stream.zfree=RelinquishZIPMemory;
stream.opaque=(voidpf) NULL;
status=deflateInit(&stream,(int) zip_info->level);
if (status != Z_OK)
{
(void) ThrowWizardException(exception,GetWizardModule(),EntropyError,
"unable to increase entropy `%s'",strerror(errno));
return(WizardFalse);
}
stream.next_in=(Bytef *) GetStringInfoDatum(message);
stream.avail_in=(uInt) GetStringInfoLength(message);
SetStringInfoLength(zip_info->chaos,(size_t) deflateBound(&stream,
(unsigned long) GetStringInfoLength(message)));
stream.next_out=(Bytef *) GetStringInfoDatum(zip_info->chaos);
stream.avail_out=(uInt) GetStringInfoLength(zip_info->chaos);
status=deflate(&stream,Z_FINISH);
if (status != Z_STREAM_END)
{
(void) ThrowWizardException(exception,GetWizardModule(),EntropyError,
"unable to increase entropy `%s'",strerror(errno));
return(WizardFalse);
}
SetStringInfoLength(zip_info->chaos,(size_t) stream.total_out);
status=deflateEnd(&stream);
if (status != Z_OK)
{
(void) ThrowWizardException(exception,GetWizardModule(),EntropyError,
"unable to increase entropy `%s'",strerror(errno));
return(WizardFalse);
}
return(WizardTrue);
}
static uint64_t computeDeflateBound(int const rlevel)
{
z_stream strm;
deflateinitz(&strm,rlevel);
// search for number of bytes that will never produce more compressed space than we have
unsigned int bound = getBgzfMaxBlockSize();
while (
deflateBound(&strm,bound) >
(getBgzfMaxBlockSize()-(getBgzfHeaderSize()+getBgzfFooterSize()))
)
--bound;
return bound;
}
static bool page_begin(struct urf_context *ctx)
{
if (ctx->page_hdr->bpp != 24) {
URF_SET_ERROR(ctx, "unsupported bpp", -ctx->page_hdr->bpp);
return false;
}
IMPL(ctx)->zlen = 2 * deflateBound(&IMPL(ctx)->strm, ctx->page_line_bytes);
if (!buf_realloc(ctx, &IMPL(ctx)->zbuf, IMPL(ctx)->zlen)) {
return false;
}
IMPL(ctx)->idx = 0;
return xprintf(ctx,
"%%%%Page: %" PRIu32 " %" PRIu32 "\n"
"%%%%PageBoundingBox: 0 0 %" PRIu32 " %" PRIu32 "\n"
"save\n"
"/DeviceRGB setcolorspace\n"
"<<\n"
" /ImageType 1\n"
" /Width %" PRIu32 "\n"
" /Height %" PRIu32 "\n"
//" /ImageMatrix [ %" PRIu32 " 0 0 -%" PRIu32 " 0 %" PRIu32 " ]\n"
" /ImageMatrix [ 1 0 0 -1 0 %" PRIu32 " ]\n"
" /BitsPerComponent 8\n"
" /Interpolate true\n"
" /Decode [ 0 1 0 1 0 1 ]\n"
" /DataSource currentfile\n"
//" /ASCIIHexDecode filter\n"
#if ASCII85 == 1
" /ASCII85Decode filter\n"
#else
" /ASCIIHexDecode filter\n"
#endif
#ifndef NODEFLATE
" /FlateDecode filter\n"
#endif
">> image\n",
ctx->page_n, ctx->page_n, ctx->page_hdr->width,
ctx->page_hdr->height, ctx->page_hdr->width,
ctx->page_hdr->height,
// ctx->page_hdr->width, ctx->page_hdr->height,
ctx->page_hdr->height);
}
static SCM squeeze(SCM source, int method) {
z_stream strm;
DEFLATE_BLOB *blob;
int ret;
size_t inlen, outlen;
char *inbuf, *outbuf;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
if (method == SQUEEZE_GZIP) {
ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
}
else {
ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
}
if (ret != Z_OK) {
log_msg("zlib: deflate init failed\n");
return SCM_BOOL_F;
}
inbuf = scm_to_utf8_stringn(source, &inlen);
strm.total_in = strm.avail_in = inlen;
strm.next_in = (unsigned char*)inbuf;
outlen = deflateBound(&strm, inlen);
outbuf = (char *)malloc(outlen);
strm.total_out = 0;
strm.avail_out = outlen;
strm.next_out = (unsigned char *)outbuf;
ret = deflate(&strm, Z_FINISH);
free(inbuf);
blob = (DEFLATE_BLOB *)scm_gc_malloc(sizeof(DEFLATE_BLOB) +
strm.total_out, "gzip-blob");
blob->zip_len = strm.total_out;
blob->orig_len = inlen;
log_msg("compress %s %d -> %d\n",
(method == SQUEEZE_GZIP ? "gzip" : "deflate"),
blob->orig_len, blob->zip_len);
blob->method = method;
ret = deflateEnd(&strm);
if (ret != Z_OK) printf("deflateEnd: %d\n", ret);
memcpy(blob->payload, outbuf, blob->zip_len);
free(outbuf);
SCM_RETURN_NEWSMOB(deflate_tag, blob);
}
int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,Mode p_mode) {
switch(p_mode) {
case MODE_FASTLZ: {
if (p_src_size<16) {
uint8_t src[16];
zeromem(&src[p_src_size],16-p_src_size);
copymem(src,p_src,p_src_size);
return fastlz_compress(src,16,p_dst);
} else {
return fastlz_compress(p_src,p_src_size,p_dst);
}
} break;
case MODE_DEFLATE: {
z_stream strm;
strm.zalloc = zipio_alloc;
strm.zfree = zipio_free;
strm.opaque = Z_NULL;
int err = deflateInit(&strm,Z_DEFAULT_COMPRESSION);
if (err!=Z_OK)
return -1;
strm.avail_in=p_src_size;
int aout = deflateBound(&strm,p_src_size);;
/*if (aout>p_src_size) {
deflateEnd(&strm);
return -1;
}*/
strm.avail_out=aout;
strm.next_in=(Bytef*)p_src;
strm.next_out=p_dst;
deflate(&strm,Z_FINISH);
aout = aout - strm.avail_out;
deflateEnd(&strm);
return aout;
} break;
}
ERR_FAIL_V(-1);
}
static int Jim_Compress(Jim_Interp *interp, const char *in, int len, long level, int wbits)
{
z_stream strm = {0};
Bytef *buf;
if ((level != Z_DEFAULT_COMPRESSION) && ((level < Z_NO_COMPRESSION) || (level > Z_BEST_COMPRESSION))) {
Jim_SetResultString(interp, "level must be 0 to 9", -1);
return JIM_ERR;
}
if (deflateInit2(&strm, level, Z_DEFLATED, wbits, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
return JIM_ERR;
}
strm.avail_out = deflateBound(&strm, (uLong)len);
if (strm.avail_out > INT_MAX) {
deflateEnd(&strm);
return JIM_ERR;
}
buf = (Bytef *)Jim_Alloc((int)strm.avail_out);
strm.next_out = buf;
strm.next_in = (Bytef *)in;
strm.avail_in = (uInt)len;
/* always compress in one pass - the return value holds the entire
* decompressed data anyway, so there's no reason to do chunked
* decompression */
if (deflate(&strm, Z_FINISH) != Z_STREAM_END) {
Jim_Free(strm.next_out);
deflateEnd(&strm);
return JIM_ERR;
}
deflateEnd(&strm);
if (strm.total_out > INT_MAX) {
Jim_Free(strm.next_out);
return JIM_ERR;
}
Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, (char *)buf, (int)strm.total_out));
return JIM_OK;
}
bool WebSocketDeflater::addBytes(const char* data, size_t length)
{
if (!length)
return false;
// The estimation by deflateBound is not accurate if the zlib has some remaining input of the last compression.
size_t maxLength = deflateBound(m_stream.get(), length);
do {
size_t writePosition = m_buffer.size();
m_buffer.grow(writePosition + maxLength);
setStreamParameter(m_stream.get(), data, length, m_buffer.data() + writePosition, maxLength);
int result = deflate(m_stream.get(), Z_NO_FLUSH);
if (result != Z_OK)
return false;
m_buffer.shrink(writePosition + maxLength - m_stream->avail_out);
maxLength *= 2;
} while (m_stream->avail_in > 0);
m_isBytesAdded = true;
return true;
}
static int fpng_init_buffer(void *arg, gx_device *dev, gs_memory_t *mem, int w, int h, void **pbuffer)
{
/* Currently, we allocate a "worst case" buffer per band - this is
* slightly larger than the band itself. We do this, so that in banded
* mode, we never need to reallocate the buffer during operation.
* For paged mode, we don't care so much about not reallocating - we
* could spot paged mode by noting that w and h match that of the
* device (or that of the device after downscaling at least), and then
* allocate a smaller initial buffer. We could even output as we go
* in paged mode. For now we leave this as an exercise for the reader.
*/
fpng_buffer_t *buffer;
int size = deflateBound(NULL, (w*3+1)*h);
buffer = (fpng_buffer_t *)gs_alloc_bytes(mem, sizeof(fpng_buffer_t) + size, "fpng_init_buffer");
*pbuffer = (void *)buffer;
if (buffer == NULL)
return gs_error_VMerror;
buffer->size = size;
buffer->compressed = 0;
return 0;
}
std::vector<unsigned char> GZip::compress(
const std::vector<unsigned char> &data)
{
::z_stream zstream;
std::memset(&zstream, 0, sizeof(zstream));
zstream.next_in = const_cast<unsigned char *>(data.data());
zstream.avail_in = data.size();
auto rc = ::deflateInit2(
&zstream, LEVEL, Z_DEFLATED, WINDOW_SIZE + WINSIZE_OFFSET_GZIP,
MEM_LEVEL, Z_DEFAULT_STRATEGY);
kulloAssert(rc == Z_OK);
::gz_header header;
std::memset(&header, 0, sizeof(header));
rc = ::deflateSetHeader(&zstream, &header);
kulloAssert(rc == Z_OK);
std::vector<unsigned char> compressed;
compressed.resize(deflateBound(&zstream, data.size()));
zstream.next_out = compressed.data();
zstream.avail_out = compressed.size();
rc = ::deflate(&zstream, Z_FINISH);
if (rc != Z_STREAM_END)
{
throw GZipStreamError(
std::string("deflate error ") + std::to_string(rc) + ": "
+ zstream.msg);
}
compressed.resize(zstream.total_out);
rc = ::deflateEnd(&zstream);
kulloAssert(rc == Z_OK);
return compressed;
}
void *zCompress(void *indata, unsigned int len, unsigned int *outlen) {
void *zdata;
int zlen;
z_stream zstruct;
zstruct.zalloc = Z_NULL;
zstruct.zfree = Z_NULL;
zstruct.opaque = Z_NULL;
if (deflateInit(&zstruct, 9) != Z_OK) {
fprintf(stderr, "deflateInit() failed\n");
return NULL;
}
zlen = deflateBound(&zstruct, len);
if ((zdata = malloc(zlen)) == NULL) {
fprintf(stderr, "Malloc(%i) failed\n", len);
return NULL;
}
zstruct.avail_in = len;
zstruct.next_in = indata;
zstruct.avail_out = zlen;
zstruct.next_out = zdata;
if (deflate(&zstruct, Z_FINISH) != Z_STREAM_END) {
fprintf(stderr, "deflate() failed\n");
free(zdata);
return NULL;
}
*outlen = zlen - zstruct.avail_out;
zdata = realloc(zdata, *outlen);
return zdata;
}
BEGIN_INANITY_DATA
DeflateStream::DeflateStream(ptr<OutputStream> outputStream, CompressionLevel compressionLevel)
: inputFile(NEW(MemoryFile(inputBufferSize))), outputStream(outputStream), finalized(false)
{
try
{
//инициализировать поток zlib
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
zstream.opaque = Z_NULL;
switch(deflateInit(&zstream, compressionLevel))
{
case Z_OK:
break;
case Z_STREAM_ERROR:
THROW("Invalid compression level");
default:
THROW("Can't initialize deflation");
}
//выделить память под выходной буфер
unsigned outputBound = deflateBound(&zstream, inputBufferSize);
outputFile = NEW(MemoryFile(outputBound));
//указать параметры буферов для потока
zstream.next_in = (Bytef*)inputFile->GetData();
zstream.avail_in = 0;
zstream.next_out = (Bytef*)outputFile->GetData();
zstream.avail_out = outputBound;
}
catch(Exception* exception)
{
THROW_SECONDARY("Can't initialize compress stream", exception);
}
}
void
Compressor::compress(std::string & in, std::string & out)
{
switch (m_compression_codec) {
case CompressionCodec::UNCOMPRESSED:
{
// Just swap the input and output collections ...
out.swap(in);
}
break;
case CompressionCodec::SNAPPY:
{
size_t compressed_size;
m_tmp.resize(snappy::MaxCompressedLength(in.size()));
snappy::RawCompress((char const *) in.data(),
in.size(),
(char *) m_tmp.data(),
&compressed_size);
m_tmp.resize(compressed_size);
out.assign(m_tmp.begin(), m_tmp.end());
}
break;
case CompressionCodec::GZIP:
{
int window_bits = 15 + 16; // maximum window + GZIP
z_stream stream;
memset(&stream, '\0', sizeof(stream));
int rv = deflateInit2(&stream,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
window_bits,
9,
Z_DEFAULT_STRATEGY);
if (rv != Z_OK) {
cerr << "deflateInit2 failed: " << rv;
exit(1);
}
m_tmp.resize(deflateBound(&stream, in.size()));
stream.next_in =
const_cast<Bytef*>(reinterpret_cast<const Bytef*>(in.data()));
stream.avail_in = in.size();
stream.next_out = (Bytef*) m_tmp.data();
stream.avail_out = m_tmp.size();
rv = deflate(&stream, Z_FINISH);
if (rv != Z_STREAM_END) {
cerr << "gzip deflate failed: " << rv;
exit(1);
}
out.assign(m_tmp.begin(), m_tmp.begin() + stream.total_out);
deflateEnd(&stream);
}
break;
default:
cerr << "unsupported compression codec: " << int(m_compression_codec);
exit(1);
break;
}
}
HL_PRIM int HL_NAME(deflate_bound)( fmt_zip *zip, int size ) {
return deflateBound(zip->z,size);
}
/*
* Copies "from" to "to", compressing "to" on the fly
*/
int copy_and_zip_file (const char *from, const char *to, int force_overwrite, int must_exist)
{
struct stat sb;
struct utimbuf t;
int fdin, fdout;
z_stream zstr = {0};
int zstatus;
char buf[BUFSIZ*10];
char zbuf[BUFSIZ*20];
int n,zlen;
#ifdef UTIME_EXPECTS_WRITABLE
int change_it_back = 0;
#endif
TRACE(1,"copy_and_zip(%s,%s)",from,to);
if (noexec)
return 0;
if ((fdin = open (from, O_RDONLY | O_BINARY,0)) < 0)
{
if(must_exist)
error (1, errno, "cannot open %s for copying", from);
else
return -1;
}
if (fstat (fdin, &sb) < 0)
{
if(must_exist)
error (1, errno, "cannot fstat %s", from);
else
{
close(fdin);
return -1;
}
}
if (force_overwrite && unlink_file (to) && !existence_error (errno))
error (1, errno, "unable to remove %s", to);
if ((fdout = open (to, O_CREAT | O_TRUNC | O_RDWR | O_BINARY, 0600)) < 0)
error (1, errno, "cannot create %s for copying", to);
zstatus = deflateInit2 (&zstr, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
if(zstatus != Z_OK)
error(1, 0, "compression error (INIT): (%d)%s", zstatus,zstr.msg);
if (sb.st_size > 0)
{
for (;;)
{
n = read (fdin, buf, sizeof(buf));
if (n == -1)
{
#ifdef EINTR
if (errno == EINTR)
continue;
#endif
error (1, errno, "cannot read file %s for copying", from);
}
else if (n == 0)
break;
zlen = deflateBound(&zstr,n);
if(zlen>sizeof(zbuf))
error(1,0,"compression error: zlen=%d (> %d)",zlen,sizeof(zbuf));
zstr.next_in = buf;
zstr.avail_in = n;
zstr.next_out = zbuf;
zstr.avail_out = zlen;
zstatus = deflate (&zstr, Z_SYNC_FLUSH);
if(zstatus != Z_OK)
error(1,0, "compression error (deflate): (%d)%s", zstatus,zstr.msg);
n = zlen-zstr.avail_out;
if (n && write(fdout, zbuf, n) != n) {
error (1, errno, "cannot write file %s for copying", to);
}
}
#ifdef HAVE_FSYNC
if (fsync (fdout))
error (1, errno, "cannot fsync file %s after copying", to);
#endif
}
zstr.next_in = buf;
zstr.avail_in = 0;
zstr.next_out = zbuf;
zstr.avail_out = sizeof(zbuf);
zstatus = deflate (&zstr, Z_FINISH);
if(zstatus != Z_OK && zstatus != Z_STREAM_END)
error(1,0, "compression error (Z_FINISH): (%d)%s", zstatus,zstr.msg);
n = sizeof(zbuf)-zstr.avail_out;
if (n && write(fdout, zbuf, n) != n) {
error (1, errno, "cannot write file %s for copying", to);
}
zstr.next_in = buf;
zstr.avail_in = 0;
zstr.next_out = zbuf;
zstr.avail_out = sizeof(zbuf);
zstatus = deflateEnd(&zstr);
if(zstatus != Z_OK)
error(1,0, "compression error (deflateEnd): (%d) %s", zstatus, zstr.msg);
if (close (fdin) < 0)
error (0, errno, "cannot close %s", from);
if (close (fdout) < 0)
error (1, errno, "cannot close %s", to);
#ifdef UTIME_EXPECTS_WRITABLE
if (!iswritable (to))
{
xchmod (to, 1);
change_it_back = 1;
}
#endif /* UTIME_EXPECTS_WRITABLE */
/* now, set the times for the copied file to match those of the original */
memset ((char *) &t, 0, sizeof (t));
t.actime = sb.st_atime;
t.modtime = sb.st_mtime;
utime (to, &t);
chmod(to,sb.st_mode);
#ifdef UTIME_EXPECTS_WRITABLE
if (change_it_back)
xchmod (to, 0);
#endif /* UTIME_EXPECTS_WRITABLE */
return 0;
}
/*
* Read data of @inode from @block_start to @block_end and uncompress
* to one zisofs block. Store the data in the @pages array with @pcount
* entries. Start storing at offset @poffset of the first page.
*/
static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
loff_t block_end, int pcount,
struct page **pages, unsigned poffset,
int *errp)
{
unsigned int zisofs_block_shift = ISOFS_I(inode)->i_format_parm[1];
unsigned int bufsize = ISOFS_BUFFER_SIZE(inode);
unsigned int bufshift = ISOFS_BUFFER_BITS(inode);
unsigned int bufmask = bufsize - 1;
int i, block_size = block_end - block_start;
z_stream stream = { .total_out = 0,
.avail_in = 0,
.avail_out = 0, };
int zerr;
int needblocks = (block_size + (block_start & bufmask) + bufmask)
>> bufshift;
int haveblocks;
blkcnt_t blocknum;
struct buffer_head *bhs[needblocks + 1];
int curbh, curpage;
if (block_size > deflateBound(1UL << zisofs_block_shift)) {
*errp = -EIO;
return 0;
}
/* Empty block? */
if (block_size == 0) {
for ( i = 0 ; i < pcount ; i++ ) {
if (!pages[i])
continue;
memset(page_address(pages[i]), 0, PAGE_SIZE);
flush_dcache_page(pages[i]);
SetPageUptodate(pages[i]);
}
return ((loff_t)pcount) << PAGE_SHIFT;
}
/* Because zlib is not thread-safe, do all the I/O at the top. */
blocknum = block_start >> bufshift;
memset(bhs, 0, (needblocks + 1) * sizeof(struct buffer_head *));
haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks);
ll_rw_block(REQ_OP_READ, 0, haveblocks, bhs);
curbh = 0;
curpage = 0;
/*
* First block is special since it may be fractional. We also wait for
* it before grabbing the zlib mutex; odds are that the subsequent
* blocks are going to come in in short order so we don't hold the zlib
* mutex longer than necessary.
*/
if (!bhs[0])
goto b_eio;
wait_on_buffer(bhs[0]);
if (!buffer_uptodate(bhs[0])) {
*errp = -EIO;
goto b_eio;
}
stream.workspace = zisofs_zlib_workspace;
mutex_lock(&zisofs_zlib_lock);
zerr = zlib_inflateInit(&stream);
if (zerr != Z_OK) {
if (zerr == Z_MEM_ERROR)
*errp = -ENOMEM;
else
*errp = -EIO;
printk(KERN_DEBUG "zisofs: zisofs_inflateInit returned %d\n",
zerr);
goto z_eio;
}
while (curpage < pcount && curbh < haveblocks &&
zerr != Z_STREAM_END) {
if (!stream.avail_out) {
if (pages[curpage]) {
stream.next_out = page_address(pages[curpage])
+ poffset;
stream.avail_out = PAGE_SIZE - poffset;
poffset = 0;
} else {
stream.next_out = (void *)&zisofs_sink_page;
stream.avail_out = PAGE_SIZE;
}
}
if (!stream.avail_in) {
wait_on_buffer(bhs[curbh]);
if (!buffer_uptodate(bhs[curbh])) {
*errp = -EIO;
break;
}
stream.next_in = bhs[curbh]->b_data +
(block_start & bufmask);
stream.avail_in = min_t(unsigned, bufsize -
(block_start & bufmask),
block_size);
block_size -= stream.avail_in;
block_start = 0;
}
while (stream.avail_out && stream.avail_in) {
zerr = zlib_inflate(&stream, Z_SYNC_FLUSH);
if (zerr == Z_BUF_ERROR && stream.avail_in == 0)
break;
if (zerr == Z_STREAM_END)
break;
if (zerr != Z_OK) {
/* EOF, error, or trying to read beyond end of input */
if (zerr == Z_MEM_ERROR)
*errp = -ENOMEM;
else {
printk(KERN_DEBUG
"zisofs: zisofs_inflate returned"
" %d, inode = %lu,"
" page idx = %d, bh idx = %d,"
" avail_in = %ld,"
" avail_out = %ld\n",
zerr, inode->i_ino, curpage,
curbh, stream.avail_in,
stream.avail_out);
*errp = -EIO;
}
goto inflate_out;
}
}
if (!stream.avail_out) {
/* This page completed */
if (pages[curpage]) {
flush_dcache_page(pages[curpage]);
SetPageUptodate(pages[curpage]);
}
curpage++;
}
if (!stream.avail_in)
curbh++;
}
inflate_out:
zlib_inflateEnd(&stream);
z_eio:
mutex_unlock(&zisofs_zlib_lock);
b_eio:
for (i = 0; i < haveblocks; i++)
brelse(bhs[i]);
return stream.total_out;
}