int deflate_flush(struct comp_ctx *comp_ctx, struct buffer *out, int flag) { int ret; int out_len = 0; z_stream *strm = &comp_ctx->strm; strm->next_out = (unsigned char *)bi_end(out); strm->avail_out = out->size - buffer_len(out); ret = deflate(strm, flag); if (ret != Z_OK && ret != Z_STREAM_END) return -1; out_len = (out->size - buffer_len(out)) - strm->avail_out; out->i += out_len; /* compression limit */ if ((global.comp_rate_lim > 0 && (read_freq_ctr(&global.comp_bps_out) > global.comp_rate_lim)) || /* rate */ (idle_pct < compress_min_idle)) { /* idle */ /* decrease level */ if (comp_ctx->cur_lvl > 0) { comp_ctx->cur_lvl--; deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY); } } else if (comp_ctx->cur_lvl < global.tune.comp_maxlevel) { /* increase level */ comp_ctx->cur_lvl++ ; deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY); } return out_len; }
/* =========================================================================== * Test deflate() with large buffers and dynamic change of compression level */ void test_large_deflate( Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen) { z_stream c_stream; /* compression stream */ int err; c_stream.zalloc = (alloc_func)0; c_stream.zfree = (free_func)0; c_stream.opaque = (voidpf)0; err = deflateInit(&c_stream, Z_BEST_SPEED); CHECK_ERR(err, "deflateInit"); c_stream.next_out = compr; c_stream.avail_out = (uInt)comprLen; /* At this point, uncompr is still mostly zeroes, so it should compress * very well: */ c_stream.next_in = uncompr; c_stream.avail_in = (uInt)uncomprLen; err = deflate(&c_stream, Z_NO_FLUSH); CHECK_ERR(err, "deflate"); if (c_stream.avail_in != 0) { fprintf(stderr, "deflate not greedy\n"); exit(1); } /* Feed in already compressed data and switch to no compression: */ deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY); c_stream.next_in = compr; c_stream.avail_in = (uInt)comprLen/2; err = deflate(&c_stream, Z_NO_FLUSH); CHECK_ERR(err, "deflate"); /* Switch back to compressing mode: */ deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED); c_stream.next_in = uncompr; c_stream.avail_in = (uInt)uncomprLen; err = deflate(&c_stream, Z_NO_FLUSH); CHECK_ERR(err, "deflate"); err = deflate(&c_stream, Z_FINISH); if (err != Z_STREAM_END) { fprintf(stderr, "deflate should report Z_STREAM_END\n"); exit(1); } err = deflateEnd(&c_stream); CHECK_ERR(err, "deflateEnd"); }
bool gzstreambuf::setLevel( int level ) { #ifdef HAVE_LIBZ if ( level <= Z_BEST_COMPRESSION ) { sync(); // make sure there is room for the deflate to flush if ( level >= 0 ) { if( m_streams->M_compression_stream ) deflateParams( m_streams->M_compression_stream, level, Z_DEFAULT_STRATEGY ); sync(); // write any data that was flushed by deflateParams } M_level = level; return true; } else return false; #else if ( M_level >= 0 ) { return false; } else { return true; } #endif }
/* returns the total size of the encoded data */ int zlib_encode(ZlibEncoder *zlib, int level, int input_size, uint8_t *io_ptr, unsigned int num_io_bytes) { int flush; int enc_size = 0; int out_size = 0; int z_ret; z_ret = deflateReset(&zlib->strm); if (z_ret != Z_OK) { red_error("deflateReset failed"); } zlib->strm.next_out = io_ptr; zlib->strm.avail_out = num_io_bytes; if (level != zlib->last_level) { if (zlib->strm.avail_out == 0) { zlib->strm.avail_out = zlib->usr->more_space(zlib->usr, &zlib->strm.next_out); if (zlib->strm.avail_out == 0) { red_error("not enough space"); } } z_ret = deflateParams(&zlib->strm, level, Z_DEFAULT_STRATEGY); if (z_ret != Z_OK) { red_error("deflateParams failed"); } zlib->last_level = level; } do { zlib->strm.avail_in = zlib->usr->more_input(zlib->usr, &zlib->strm.next_in); if (zlib->strm.avail_in <= 0) { red_error("more input failed\n"); } enc_size += zlib->strm.avail_in; flush = (enc_size == input_size) ? Z_FINISH : Z_NO_FLUSH; while (1) { int deflate_size = zlib->strm.avail_out; z_ret = deflate(&zlib->strm, flush); ASSERT(z_ret != Z_STREAM_ERROR); out_size += deflate_size - zlib->strm.avail_out; if (zlib->strm.avail_out) { break; } zlib->strm.avail_out = zlib->usr->more_space(zlib->usr, &zlib->strm.next_out); if (zlib->strm.avail_out == 0) { red_error("not enough space"); } } } while (flush != Z_FINISH); ASSERT(z_ret == Z_STREAM_END); return out_size; }
static TACommandVerdict deflateParams_cmd(TAThread thread,TAInputStream stream) { z_stream strm; int res, is_null, level, strategy; is_null = readZStream(&stream, &strm); level = readInt(&stream); strategy = readInt(&stream); /*ta_debug_printf( "next_in==%d\navail_in==%u\ntotal_in==%lu\nnext_out==%d\n" "avail_out==%u\ntotal_out==%lu\n", strm.next_in, strm.avail_in, strm.total_in, strm.next_out, strm.avail_out,strm.total_out); ta_debug_printf( "msg==%s\nstate==%d\nzalloc==%d\nzfree==%d\n" "opaque==%d\ndata_type==%d\nadler==%lu\nreserved==%lu\n", strm.msg, strm.state, strm.zalloc, strm.zfree, strm.opaque, strm.data_type, strm.adler, strm.reserved);*/ START_TARGET_OPERATION(thread); if(!is_null) res = deflateParams(&strm, level, strategy); else res = deflateParams(0, level, strategy); END_TARGET_OPERATION(thread); // Response if(!is_null) writeZStream(thread, &strm); else writeZStream(thread, 0); writeInt(thread, res); sendResponse(thread); return taDefaultVerdict; }
bool ZFilter::operator()(const void* in, size_t& insize, void* out, size_t& outsize) { if(outsize == 0) return false; zs.next_in = (Bytef*)in; zs.next_out = (Bytef*)out; // Check if there's any use compressing; if not, save some cpu... if(compressing && insize > 0 && outsize > 16 && (totalIn > (64*1024)) && ((static_cast<double>(totalOut) / totalIn) > 0.95)) { zs.avail_in = 0; zs.avail_out = outsize; if(deflateParams(&zs, 0, Z_DEFAULT_STRATEGY) != Z_OK) { throw Exception(STRING(COMPRESSION_ERROR)); } zs.avail_in = insize; compressing = false; dcdebug("Dynamically disabled compression"); // Check if we ate all space already... if(zs.avail_out == 0) { outsize = outsize - zs.avail_out; insize = insize - zs.avail_in; totalOut += outsize; totalIn += insize; return true; } } else { zs.avail_in = insize; zs.avail_out = outsize; } if(insize == 0) { int err = ::deflate(&zs, Z_FINISH); if(err != Z_OK && err != Z_STREAM_END) throw Exception(STRING(COMPRESSION_ERROR)); outsize = outsize - zs.avail_out; insize = insize - zs.avail_in; totalOut += outsize; totalIn += insize; return err == Z_OK; } else { int err = ::deflate(&zs, Z_NO_FLUSH); if(err != Z_OK) throw Exception(STRING(COMPRESSION_ERROR)); outsize = outsize - zs.avail_out; insize = insize - zs.avail_in; totalOut += outsize; totalIn += insize; return true; } }
/* Output data to a compression buffer. * * GLOBALS * gzip_level If GZIP_LEVEL has changed to a value different from * CLOSURE->level, then set the compression level on the * stream to the new value. */ static int compress_buffer_output (void *closure, const char *data, size_t have, size_t *wrote) { struct compress_buffer *cb = closure; /* This is only used within the while loop below, but allocated here for * efficiency. */ static char *buffer = NULL; if (!buffer) buffer = pagealign_xalloc (BUFFER_DATA_SIZE); if (cb->level != gzip_level) { cb->level = gzip_level; deflateParams (&cb->zstr, gzip_level, Z_DEFAULT_STRATEGY); } cb->zstr.avail_in = have; cb->zstr.next_in = (unsigned char *) data; while (cb->zstr.avail_in > 0) { int zstatus; cb->zstr.avail_out = BUFFER_DATA_SIZE; cb->zstr.next_out = (unsigned char *) buffer; zstatus = deflate (&cb->zstr, Z_NO_FLUSH); if (zstatus != Z_OK) { compress_error (0, zstatus, &cb->zstr, "deflate"); return EIO; } if (cb->zstr.avail_out != BUFFER_DATA_SIZE) buf_output (cb->buf, buffer, BUFFER_DATA_SIZE - cb->zstr.avail_out); } *wrote = have; /* We will only be here because buf_send_output was called on the compression buffer. That means that we should now call buf_send_output on the underlying buffer. */ return buf_send_output (cb->buf); }
/* =========================================================================== * Update the compression level and strategy */ int lib_gzsetparams ( gzFile file, int level, int strategy) { gz_stream *s = (gz_stream*)file; if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; /* Make room to allow flushing */ if (s->stream.avail_out == 0) { s->stream.next_out = s->outbuf; if (F_WRITE(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { s->z_err = Z_ERRNO; } s->stream.avail_out = Z_BUFSIZE; } return deflateParams (&(s->stream), level, strategy); }
JNIEXPORT void JNICALL Java_java_util_zip_Deflater_setLevelsImpl (JNIEnv * env, jobject recv, int level, int strategy, jlong handle) { JCLZipStream *stream; jbyte b = 0; int err = 0; if (handle == -1) { throwNewIllegalStateException (env, ""); return; } stream = (JCLZipStream *) ((IDATA) handle); stream->stream->next_out = (Bytef *) & b; err = deflateParams (stream->stream, level, strategy); if (err != Z_OK) { THROW_ZIP_EXCEPTION(env, err, IllegalStateException); } }
/* -- see zlib.h -- */ int ZEXPORT gzsetparams( gzFile file, int level, int strategy) { gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return Z_STREAM_ERROR; /* if no change is requested, then do nothing */ if (level == state->level && strategy == state->strategy) return Z_OK; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return -1; } /* change compression parameters for subsequent input */ if (state->size) { /* flush previous input with previous parameters before changing */ if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1) return state->err; deflateParams(strm, level, strategy); } state->level = level; state->strategy = strategy; return Z_OK; }
static int ZIPVSetField(TIFF* tif, ttag_t tag, va_list ap) { ZIPState* sp = ZState(tif); static const char module[] = "ZIPVSetField"; switch (tag) { case TIFFTAG_ZIPQUALITY: sp->zipquality = va_arg(ap, int); if (tif->tif_mode != O_RDONLY && (sp->state & ZSTATE_INIT)) { if (deflateParams(&sp->stream, sp->zipquality, Z_DEFAULT_STRATEGY) != Z_OK) { TIFFError(module, "%s: zlib error: %s", tif->tif_name, sp->stream.msg); return (0); } } return (1); default: return (*sp->vsetparent)(tif, tag, ap); } /*NOTREACHED*/ }
static int ZIPVSetField(TIFF* tif, uint32 tag, va_list ap) { static const char module[] = "ZIPVSetField"; ZIPState* sp = ZState(tif); switch (tag) { case TIFFTAG_ZIPQUALITY: sp->zipquality = (int) va_arg(ap, int); if ( sp->state&ZSTATE_INIT_ENCODE ) { if (deflateParams(&sp->stream, sp->zipquality, Z_DEFAULT_STRATEGY) != Z_OK) { TIFFErrorExt(tif->tif_clientdata, module, "ZLib error: %s", SAFE_MSG(sp)); return (0); } } return (1); default: return (*sp->vsetparent)(tif, tag, ap); } /*NOTREACHED*/ }
/*! */ bool gzfilterstreambuf::setLevel( const int level ) { bool ret = false; #ifdef HAVE_LIBZ if ( level == DEFAULT_COMPRESSION || ( NO_COMPRESSION <= level && level <= BEST_COMPRESSION ) ) { //std::cerr << "set compression level = " << level << std::endl; // make sure there is room for the deflate to flush this->sync(); deflateParams( M_impl->comp_stream_, level, Z_DEFAULT_STRATEGY ); // write data flushed by deflateParams this->sync(); M_level = level; ret = true; } // Without zlib, this class cannot handle any compression level. #endif return ret; }
/* * Send data read from an already open file descriptor. * * We return 1 on sucess and 0 on errors. * * ***FIXME*** * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop * reading. * Currently this is not a problem as the only other stream, resource forks, * are not handled as sparse files. */ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signing_digest) { BSOCK *sd = jcr->store_bsock; uint64_t fileAddr = 0; /* file address */ char *rbuf, *wbuf; int32_t rsize = jcr->buf_size; /* read buffer size */ POOLMEM *msgsave; CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */ const uint8_t *cipher_input; uint32_t cipher_input_len; uint32_t cipher_block_size; uint32_t encrypted_len; #ifdef FD_NO_SEND_TEST return 1; #endif msgsave = sd->msg; rbuf = sd->msg; /* read buffer */ wbuf = sd->msg; /* write buffer */ cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */ Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); #ifdef HAVE_LIBZ uLong compress_len = 0; uLong max_compress_len = 0; const Bytef *cbuf = NULL; int zstat; if (ff_pkt->flags & FO_GZIP) { if (ff_pkt->flags & FO_SPARSE) { cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE; max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE; } else { cbuf = (Bytef *)jcr->compress_buf; max_compress_len = jcr->compress_buf_size; /* set max length */ } wbuf = jcr->compress_buf; /* compressed output here */ cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ /* * Only change zlib parameters if there is no pending operation. * This should never happen as deflatereset is called after each * deflate. */ if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) { /* set gzip compression level - must be done per file */ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } } } #else const uint32_t max_compress_len = 0; #endif if (ff_pkt->flags & FO_ENCRYPT) { if (ff_pkt->flags & FO_SPARSE) { Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n")); goto err; } /* Allocate the cipher context */ if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true, &cipher_block_size)) == NULL) { /* Shouldn't happen! */ Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n")); goto err; } /* * Grow the crypto buffer, if necessary. * crypto_cipher_update() will buffer up to (cipher_block_size - 1). * We grow crypto_buf to the maximum number of blocks that * could be returned for the given read buffer size. * (Using the larger of either rsize or max_compress_len) */ jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf, (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) + cipher_block_size - 1) / cipher_block_size * cipher_block_size); wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */ } /* * Send Data header to Storage daemon * <file-index> <stream> <info> */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(300, ">stored: datahdr %s\n", sd->msg); /* * Make space at beginning of buffer for fileAddr because this * same buffer will be used for writing if compression is off. */ if (ff_pkt->flags & FO_SPARSE) { rbuf += SPARSE_FADDR_SIZE; rsize -= SPARSE_FADDR_SIZE; #ifdef HAVE_FREEBSD_OS /* * To read FreeBSD partitions, the read size must be * a multiple of 512. */ rsize = (rsize/512) * 512; #endif } /* a RAW device read on win32 only works if the buffer is a multiple of 512 */ #ifdef HAVE_WIN32 if (S_ISBLK(ff_pkt->statp.st_mode)) rsize = (rsize/512) * 512; #endif /* * Read the file data */ while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { /* Check for sparse blocks */ if (ff_pkt->flags & FO_SPARSE) { ser_declare; bool allZeros = false; if ((sd->msglen == rsize && fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) || ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) && (uint64_t)ff_pkt->statp.st_size == 0)) { allZeros = is_buf_zero(rbuf, rsize); } if (!allZeros) { /* Put file address as first data in buffer */ ser_begin(wbuf, SPARSE_FADDR_SIZE); ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ } fileAddr += sd->msglen; /* update file address */ /* Skip block of all zeros */ if (allZeros) { continue; /* skip block of zeros */ } } jcr->ReadBytes += sd->msglen; /* count bytes read */ /* Uncompressed cipher input length */ cipher_input_len = sd->msglen; /* Update checksum if requested */ if (digest) { crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen); } /* Update signing digest if requested */ if (signing_digest) { crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen); } #ifdef HAVE_LIBZ /* Do compression if turned on */ if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) { Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf; ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen; ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf; ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len; if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) { Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out; /* reset zlib stream to be able to begin from scratch again */ if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len, sd->msglen); sd->msglen = compress_len; /* set compressed length */ cipher_input_len = compress_len; } #endif /* * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression * restore handling want records returned to them with exactly the * same number of bytes that were processed in the backup handling. * That is, both are block filters rather than a stream. When doing * compression, the compression routines may buffer data, so that for * any one record compressed, when it is decompressed the same size * will not be obtained. Of course, the buffered data eventually comes * out in subsequent crypto_cipher_update() calls or at least * when crypto_cipher_finalize() is called. Unfortunately, this * "feature" of encryption enormously complicates the restore code. */ if (ff_pkt->flags & FO_ENCRYPT) { uint32_t initial_len = 0; ser_declare; if (ff_pkt->flags & FO_SPARSE) { cipher_input_len += SPARSE_FADDR_SIZE; } /* Encrypt the length of the input block */ uint8_t packet_len[sizeof(uint32_t)]; ser_begin(packet_len, sizeof(uint32_t)); ser_uint32(cipher_input_len); /* store data len in begin of buffer */ Dmsg1(20, "Encrypt len=%d\n", cipher_input_len); if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len), (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) { /* Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } /* Encrypt the input block */ if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len, (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) { if ((initial_len + encrypted_len) == 0) { /* No full block of data available, read more data */ continue; } Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len, sd->msglen); sd->msglen = initial_len + encrypted_len; /* set encrypted length */ } else { /* Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } } /* Send the buffer to the Storage daemon */ if (ff_pkt->flags & FO_SPARSE) { sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */ } sd->msg = wbuf; /* set correct write buffer */ if (!sd->send()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); /* #endif */ jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */ sd->msg = msgsave; /* restore read buffer */ } /* end while read file data */ if (sd->msglen < 0) { /* error */ berrno be; Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno)); if (jcr->JobErrors++ > 1000) { /* insanity check */ Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n")); } } else if (ff_pkt->flags & FO_ENCRYPT) { /* * For encryption, we must call finalize to push out any * buffered data. */ if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf, &encrypted_len)) { /* Padding failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n")); goto err; } /* Note, on SSL pre-0.9.7, there is always some output */ if (encrypted_len > 0) { sd->msglen = encrypted_len; /* set encrypted length */ sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */ if (!sd->send()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */ sd->msg = msgsave; /* restore bnet buffer */ } } if (!sd->signal(BNET_EOD)) { /* indicate end of file data */ Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } /* Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } return 1; err: /* Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } sd->msg = msgsave; /* restore bnet buffer */ sd->msglen = 0; return 0; }
bool setup_compression_context(b_ctx &bctx) { bool retval = false; if ((bctx.ff_pkt->flags & FO_COMPRESS)) { /* * See if we need to be compatible with the old GZIP stream encoding. */ if (!me->compatible || bctx.ff_pkt->Compress_algo != COMPRESS_GZIP) { memset(&bctx.ch, 0, sizeof(comp_stream_header)); /* * Calculate buffer offsets. */ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) { bctx.chead = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE; bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE + sizeof(comp_stream_header); bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - (sizeof(comp_stream_header) + OFFSET_FADDR_SIZE); } else { bctx.chead = (uint8_t *)bctx.jcr->compress.deflate_buffer; bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + sizeof(comp_stream_header); bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - sizeof(comp_stream_header); } bctx.wbuf = bctx.jcr->compress.deflate_buffer; /* compressed output here */ bctx.cipher_input = (uint8_t *)bctx.jcr->compress.deflate_buffer; /* encrypt compressed data */ bctx.ch.magic = bctx.ff_pkt->Compress_algo; bctx.ch.version = COMP_HEAD_VERSION; } else { /* * Calculate buffer offsets. */ bctx.chead = NULL; if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) { bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE; bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - OFFSET_FADDR_SIZE; } else { bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer; bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size; } bctx.wbuf = bctx.jcr->compress.deflate_buffer; /* compressed output here */ bctx.cipher_input = (uint8_t *)bctx.jcr->compress.deflate_buffer; /* encrypt compressed data */ } /* * Do compression specific actions and set the magic, header version and compression level. */ switch (bctx.ff_pkt->Compress_algo) { #if defined(HAVE_LIBZ) case COMPRESS_GZIP: { z_stream *pZlibStream; /** * Only change zlib parameters if there is no pending operation. * This should never happen as deflateReset is called after each * deflate. */ pZlibStream = (z_stream *)bctx.jcr->compress.workset.pZLIB; if (pZlibStream->total_in == 0) { int zstat; /* * Set gzip compression level - must be done per file */ if ((zstat = deflateParams(pZlibStream, bctx.ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(bctx.jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); bctx.jcr->setJobStatus(JS_ErrorTerminated); goto bail_out; } } bctx.ch.level = bctx.ff_pkt->Compress_level; break; } #endif #if defined(HAVE_LZO) case COMPRESS_LZO1X: break; #endif #if defined(HAVE_FASTLZ) case COMPRESS_FZFZ: case COMPRESS_FZ4L: case COMPRESS_FZ4H: { int zstat; zfast_stream *pZfastStream; zfast_stream_compressor compressor = COMPRESSOR_FASTLZ; /** * Only change fastlz parameters if there is no pending operation. * This should never happen as fastlzlibCompressReset is called after each * fastlzlibCompress. */ pZfastStream = (zfast_stream *)bctx.jcr->compress.workset.pZFAST; if (pZfastStream->total_in == 0) { switch (bctx.ff_pkt->Compress_algo) { case COMPRESS_FZ4L: case COMPRESS_FZ4H: compressor = COMPRESSOR_LZ4; break; } if ((zstat = fastlzlibSetCompressor(pZfastStream, compressor)) != Z_OK) { Jmsg(bctx.jcr, M_FATAL, 0, _("Compression fastlzlibSetCompressor error: %d\n"), zstat); bctx.jcr->setJobStatus(JS_ErrorTerminated); goto bail_out; } } bctx.ch.level = bctx.ff_pkt->Compress_level; break; } #endif default: break; } } retval = true; bail_out: return retval; }
int vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen, int zlibLevel, int zlibStrategy) { if (dataLen < TIGHT_MIN_TO_COMPRESS) { memcpy(dest, m_buffer, dataLen); return dataLen; } z_streamp pz = &m_zsStruct[streamId]; // Initialize compression stream if needed. if (!m_zsActive[streamId]) { pz->zalloc = Z_NULL; pz->zfree = Z_NULL; pz->opaque = Z_NULL; vnclog.Print(LL_INTINFO, VNCLOG("calling deflateInit2 with zlib level:%d\n"), zlibLevel); int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS, MAX_MEM_LEVEL, zlibStrategy); if (err != Z_OK) { vnclog.Print(LL_INTINFO, VNCLOG("deflateInit2 returned error:%d:%s\n"), err, pz->msg); return -1; } m_zsActive[streamId] = true; m_zsLevel[streamId] = zlibLevel; } int outBufferSize = dataLen + dataLen / 100 + 16; // Prepare buffer pointers. pz->next_in = (Bytef *)m_buffer; pz->avail_in = dataLen; pz->next_out = (Bytef *)dest; pz->avail_out = outBufferSize; // Change compression parameters if needed. if (zlibLevel != m_zsLevel[streamId]) { vnclog.Print(LL_INTINFO, VNCLOG("calling deflateParams with zlib level:%d\n"), zlibLevel); int err = deflateParams (pz, zlibLevel, zlibStrategy); if (err != Z_OK) { vnclog.Print(LL_INTINFO, VNCLOG("deflateParams returned error:%d:%s\n"), err, pz->msg); return -1; } m_zsLevel[streamId] = zlibLevel; } // Actual compression. if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK || pz->avail_in != 0 || pz->avail_out == 0 ) { vnclog.Print(LL_INTINFO, VNCLOG("deflate() call failed.\n")); return -1; } return SendCompressedData(outBufferSize - pz->avail_out); }
/* * Setup deflate for auto deflate of data streams. */ static bool setup_auto_deflation(DCR *dcr) { JCR *jcr = dcr->jcr; bool retval = false; uint32_t compress_buf_size = 0; if (jcr->buf_size == 0) { jcr->buf_size = DEFAULT_NETWORK_BUFFER_SIZE; } if (!setup_compression_buffers(jcr, sd_enabled_compatible, dcr->device->autodeflate_algorithm, &compress_buf_size)) { goto bail_out; } /* * See if we need to create a new compression buffer or make sure the existing is big enough. */ if (!jcr->compress.deflate_buffer) { jcr->compress.deflate_buffer = get_memory(compress_buf_size); jcr->compress.deflate_buffer_size = compress_buf_size; } else { if (compress_buf_size > jcr->compress.deflate_buffer_size) { jcr->compress.deflate_buffer = realloc_pool_memory(jcr->compress.deflate_buffer, compress_buf_size); jcr->compress.deflate_buffer_size = compress_buf_size; } } switch (dcr->device->autodeflate_algorithm) { #if defined(HAVE_LIBZ) case COMPRESS_GZIP: { int zstat; z_stream *pZlibStream; pZlibStream = (z_stream *)jcr->compress.workset.pZLIB; if ((zstat = deflateParams(pZlibStream, dcr->device->autodeflate_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); jcr->setJobStatus(JS_ErrorTerminated); goto bail_out; } break; } #endif #if defined(HAVE_LZO) case COMPRESS_LZO1X: break; #endif #if defined(HAVE_FASTLZ) case COMPRESS_FZFZ: case COMPRESS_FZ4L: case COMPRESS_FZ4H: { int zstat; zfast_stream *pZfastStream; zfast_stream_compressor compressor = COMPRESSOR_FASTLZ; switch (dcr->device->autodeflate_algorithm) { case COMPRESS_FZ4L: case COMPRESS_FZ4H: compressor = COMPRESSOR_LZ4; break; } pZfastStream = (zfast_stream *)jcr->compress.workset.pZFAST; if ((zstat = fastlzlibSetCompressor(pZfastStream, compressor)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression fastlzlibSetCompressor error: %d\n"), zstat); jcr->setJobStatus(JS_ErrorTerminated); goto bail_out; } break; } #endif default: break; } retval = true; bail_out: return retval; }
static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { ZLibData* d = (ZLibData*)drv_data; int res; switch(command) { case DEFLATE_INIT: if (len != 4) goto badarg; if (d->state != ST_NONE) goto badarg; res = deflateInit(&d->s, i32(buf)); if (res == Z_OK) { d->state = ST_DEFLATE; d->want_crc = 0; d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); case DEFLATE_INIT2: { int wbits; if (len != 20) goto badarg; if (d->state != ST_NONE) goto badarg; wbits = i32(buf+8); res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits, i32(buf+12), i32(buf+16)); if (res == Z_OK) { d->state = ST_DEFLATE; d->want_crc = (wbits < 0); d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); } case DEFLATE_SETDICT: if (d->state != ST_DEFLATE) goto badarg; res = deflateSetDictionary(&d->s, (unsigned char*)buf, len); if (res == Z_OK) { return zlib_value(d->s.adler, rbuf, rlen); } else { return zlib_return(res, rbuf, rlen); } case DEFLATE_RESET: if (len != 0) goto badarg; if (d->state != ST_DEFLATE) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = deflateReset(&d->s); return zlib_return(res, rbuf, rlen); case DEFLATE_END: if (len != 0) goto badarg; if (d->state != ST_DEFLATE) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = deflateEnd(&d->s); d->state = ST_NONE; return zlib_return(res, rbuf, rlen); case DEFLATE_PARAMS: if (len != 8) goto badarg; if (d->state != ST_DEFLATE) goto badarg; res = deflateParams(&d->s, i32(buf), i32(buf+4)); return zlib_return(res, rbuf, rlen); case DEFLATE: if (d->state != ST_DEFLATE) goto badarg; if (len != 4) goto badarg; res = zlib_deflate(d, i32(buf)); return zlib_return(res, rbuf, rlen); case INFLATE_INIT: if (len != 0) goto badarg; if (d->state != ST_NONE) goto badarg; res = inflateInit(&d->s); if (res == Z_OK) { d->state = ST_INFLATE; d->inflate_eos_seen = 0; d->want_crc = 0; d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); case INFLATE_INIT2: { int wbits; if (len != 4) goto badarg; if (d->state != ST_NONE) goto badarg; wbits = i32(buf); res = inflateInit2(&d->s, wbits); if (res == Z_OK) { d->state = ST_INFLATE; d->inflate_eos_seen = 0; d->want_crc = (wbits < 0); d->crc = crc32(0L, Z_NULL, 0); } return zlib_return(res, rbuf, rlen); } case INFLATE_SETDICT: if (d->state != ST_INFLATE) goto badarg; res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); return zlib_return(res, rbuf, rlen); case INFLATE_SYNC: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; if (driver_sizeq(d->port) == 0) { res = Z_BUF_ERROR; } else { int vlen; SysIOVec* iov = driver_peekq(d->port, &vlen); d->s.next_in = iov[0].iov_base; d->s.avail_in = iov[0].iov_len; res = inflateSync(&d->s); } return zlib_return(res, rbuf, rlen); case INFLATE_RESET: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = inflateReset(&d->s); d->inflate_eos_seen = 0; return zlib_return(res, rbuf, rlen); case INFLATE_END: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; driver_deq(d->port, driver_sizeq(d->port)); res = inflateEnd(&d->s); if (res == Z_OK && d->inflate_eos_seen == 0) { res = Z_DATA_ERROR; } d->state = ST_NONE; return zlib_return(res, rbuf, rlen); case INFLATE: if (d->state != ST_INFLATE) goto badarg; if (len != 4) goto badarg; res = zlib_inflate(d, i32(buf)); if (res == Z_NEED_DICT) { return zlib_value2(3, d->s.adler, rbuf, rlen); } else { return zlib_return(res, rbuf, rlen); } case GET_QSIZE: return zlib_value(driver_sizeq(d->port), rbuf, rlen); case GET_BUFSZ: return zlib_value(d->binsz_need, rbuf, rlen); case SET_BUFSZ: { int need; if (len != 4) goto badarg; need = i32(buf); if ((need < 16) || (need > 0x00ffffff)) goto badarg; if (d->binsz_need != need) { d->binsz_need = need; if (d->bin != NULL) { if (d->s.avail_out == d->binsz) { driver_free_binary(d->bin); d->bin = NULL; d->binsz = 0; } else zlib_output(d); } } return zlib_return(Z_OK, rbuf, rlen); } case CRC32_0: return zlib_value(d->crc, rbuf, rlen); case CRC32_1: { uLong crc = crc32(0L, Z_NULL, 0); crc = crc32(crc, (unsigned char*) buf, len); return zlib_value(crc, rbuf, rlen); } case CRC32_2: { uLong crc; if (len < 4) goto badarg; crc = (unsigned int) i32(buf); crc = crc32(crc, (unsigned char*) buf+4, len-4); return zlib_value(crc, rbuf, rlen); } case ADLER32_1: { uLong adler = adler32(0L, Z_NULL, 0); adler = adler32(adler, (unsigned char*) buf, len); return zlib_value(adler, rbuf, rlen); } case ADLER32_2: { uLong adler; if (len < 4) goto badarg; adler = (unsigned int) i32(buf); adler = adler32(adler, (unsigned char*) buf+4, len-4); return zlib_value(adler, rbuf, rlen); } case CRC32_COMBINE: { uLong crc, crc1, crc2, len2; if (len != 12) goto badarg; crc1 = (unsigned int) i32(buf); crc2 = (unsigned int) i32(buf+4); len2 = (unsigned int) i32(buf+8); crc = crc32_combine(crc1, crc2, len2); return zlib_value(crc, rbuf, rlen); } case ADLER32_COMBINE: { uLong adler, adler1, adler2, len2; if (len != 12) goto badarg; adler1 = (unsigned int) i32(buf); adler2 = (unsigned int) i32(buf+4); len2 = (unsigned int) i32(buf+8); adler = adler32_combine(adler1, adler2, len2); return zlib_value(adler, rbuf, rlen); } } badarg: errno = EINVAL; return zlib_return(Z_ERRNO, rbuf, rlen); }