int deflate_flush(struct comp_ctx *comp_ctx, struct buffer *out, int flag)
{
int ret;
int out_len = 0;
z_stream *strm = &comp_ctx->strm;
strm->next_out = (unsigned char *)bi_end(out);
strm->avail_out = out->size - buffer_len(out);
ret = deflate(strm, flag);
if (ret != Z_OK && ret != Z_STREAM_END)
return -1;
out_len = (out->size - buffer_len(out)) - strm->avail_out;
out->i += out_len;
/* compression limit */
if ((global.comp_rate_lim > 0 && (read_freq_ctr(&global.comp_bps_out) > global.comp_rate_lim)) || /* rate */
(idle_pct < compress_min_idle)) { /* idle */
/* decrease level */
if (comp_ctx->cur_lvl > 0) {
comp_ctx->cur_lvl--;
deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY);
}
} else if (comp_ctx->cur_lvl < global.tune.comp_maxlevel) {
/* increase level */
comp_ctx->cur_lvl++ ;
deflateParams(&comp_ctx->strm, comp_ctx->cur_lvl, Z_DEFAULT_STRATEGY);
}
return out_len;
}
/* ===========================================================================
* Test deflate() with large buffers and dynamic change of compression level
*/
void test_large_deflate(
Byte *compr,
uLong comprLen,
Byte *uncompr,
uLong uncomprLen)
{
z_stream c_stream; /* compression stream */
int err;
c_stream.zalloc = (alloc_func)0;
c_stream.zfree = (free_func)0;
c_stream.opaque = (voidpf)0;
err = deflateInit(&c_stream, Z_BEST_SPEED);
CHECK_ERR(err, "deflateInit");
c_stream.next_out = compr;
c_stream.avail_out = (uInt)comprLen;
/* At this point, uncompr is still mostly zeroes, so it should compress
* very well:
*/
c_stream.next_in = uncompr;
c_stream.avail_in = (uInt)uncomprLen;
err = deflate(&c_stream, Z_NO_FLUSH);
CHECK_ERR(err, "deflate");
if (c_stream.avail_in != 0) {
fprintf(stderr, "deflate not greedy\n");
exit(1);
}
/* Feed in already compressed data and switch to no compression: */
deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY);
c_stream.next_in = compr;
c_stream.avail_in = (uInt)comprLen/2;
err = deflate(&c_stream, Z_NO_FLUSH);
CHECK_ERR(err, "deflate");
/* Switch back to compressing mode: */
deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED);
c_stream.next_in = uncompr;
c_stream.avail_in = (uInt)uncomprLen;
err = deflate(&c_stream, Z_NO_FLUSH);
CHECK_ERR(err, "deflate");
err = deflate(&c_stream, Z_FINISH);
if (err != Z_STREAM_END) {
fprintf(stderr, "deflate should report Z_STREAM_END\n");
exit(1);
}
err = deflateEnd(&c_stream);
CHECK_ERR(err, "deflateEnd");
}
bool
gzstreambuf::setLevel( int level )
{
#ifdef HAVE_LIBZ
if ( level <= Z_BEST_COMPRESSION )
{
sync(); // make sure there is room for the deflate to flush
if ( level >= 0 )
{
if( m_streams->M_compression_stream )
deflateParams( m_streams->M_compression_stream, level, Z_DEFAULT_STRATEGY );
sync(); // write any data that was flushed by deflateParams
}
M_level = level;
return true;
}
else
return false;
#else
if ( M_level >= 0 )
{
return false;
}
else
{
return true;
}
#endif
}
/* returns the total size of the encoded data */
int zlib_encode(ZlibEncoder *zlib, int level, int input_size,
uint8_t *io_ptr, unsigned int num_io_bytes)
{
int flush;
int enc_size = 0;
int out_size = 0;
int z_ret;
z_ret = deflateReset(&zlib->strm);
if (z_ret != Z_OK) {
red_error("deflateReset failed");
}
zlib->strm.next_out = io_ptr;
zlib->strm.avail_out = num_io_bytes;
if (level != zlib->last_level) {
if (zlib->strm.avail_out == 0) {
zlib->strm.avail_out = zlib->usr->more_space(zlib->usr, &zlib->strm.next_out);
if (zlib->strm.avail_out == 0) {
red_error("not enough space");
}
}
z_ret = deflateParams(&zlib->strm, level, Z_DEFAULT_STRATEGY);
if (z_ret != Z_OK) {
red_error("deflateParams failed");
}
zlib->last_level = level;
}
do {
zlib->strm.avail_in = zlib->usr->more_input(zlib->usr, &zlib->strm.next_in);
if (zlib->strm.avail_in <= 0) {
red_error("more input failed\n");
}
enc_size += zlib->strm.avail_in;
flush = (enc_size == input_size) ? Z_FINISH : Z_NO_FLUSH;
while (1) {
int deflate_size = zlib->strm.avail_out;
z_ret = deflate(&zlib->strm, flush);
ASSERT(z_ret != Z_STREAM_ERROR);
out_size += deflate_size - zlib->strm.avail_out;
if (zlib->strm.avail_out) {
break;
}
zlib->strm.avail_out = zlib->usr->more_space(zlib->usr, &zlib->strm.next_out);
if (zlib->strm.avail_out == 0) {
red_error("not enough space");
}
}
} while (flush != Z_FINISH);
ASSERT(z_ret == Z_STREAM_END);
return out_size;
}
static TACommandVerdict deflateParams_cmd(TAThread thread,TAInputStream stream)
{
z_stream strm;
int res, is_null, level, strategy;
is_null = readZStream(&stream, &strm);
level = readInt(&stream);
strategy = readInt(&stream);
/*ta_debug_printf( "next_in==%d\navail_in==%u\ntotal_in==%lu\nnext_out==%d\n"
"avail_out==%u\ntotal_out==%lu\n",
strm.next_in, strm.avail_in, strm.total_in, strm.next_out,
strm.avail_out,strm.total_out);
ta_debug_printf( "msg==%s\nstate==%d\nzalloc==%d\nzfree==%d\n"
"opaque==%d\ndata_type==%d\nadler==%lu\nreserved==%lu\n",
strm.msg, strm.state,
strm.zalloc, strm.zfree, strm.opaque, strm.data_type, strm.adler,
strm.reserved);*/
START_TARGET_OPERATION(thread);
if(!is_null)
res = deflateParams(&strm, level, strategy);
else
res = deflateParams(0, level, strategy);
END_TARGET_OPERATION(thread);
// Response
if(!is_null)
writeZStream(thread, &strm);
else
writeZStream(thread, 0);
writeInt(thread, res);
sendResponse(thread);
return taDefaultVerdict;
}
bool ZFilter::operator()(const void* in, size_t& insize, void* out, size_t& outsize) {
if(outsize == 0)
return false;
zs.next_in = (Bytef*)in;
zs.next_out = (Bytef*)out;
// Check if there's any use compressing; if not, save some cpu...
if(compressing && insize > 0 && outsize > 16 && (totalIn > (64*1024)) && ((static_cast<double>(totalOut) / totalIn) > 0.95)) {
zs.avail_in = 0;
zs.avail_out = outsize;
if(deflateParams(&zs, 0, Z_DEFAULT_STRATEGY) != Z_OK) {
throw Exception(STRING(COMPRESSION_ERROR));
}
zs.avail_in = insize;
compressing = false;
dcdebug("Dynamically disabled compression");
// Check if we ate all space already...
if(zs.avail_out == 0) {
outsize = outsize - zs.avail_out;
insize = insize - zs.avail_in;
totalOut += outsize;
totalIn += insize;
return true;
}
} else {
zs.avail_in = insize;
zs.avail_out = outsize;
}
if(insize == 0) {
int err = ::deflate(&zs, Z_FINISH);
if(err != Z_OK && err != Z_STREAM_END)
throw Exception(STRING(COMPRESSION_ERROR));
outsize = outsize - zs.avail_out;
insize = insize - zs.avail_in;
totalOut += outsize;
totalIn += insize;
return err == Z_OK;
} else {
int err = ::deflate(&zs, Z_NO_FLUSH);
if(err != Z_OK)
throw Exception(STRING(COMPRESSION_ERROR));
outsize = outsize - zs.avail_out;
insize = insize - zs.avail_in;
totalOut += outsize;
totalIn += insize;
return true;
}
}
/* Output data to a compression buffer.
*
* GLOBALS
* gzip_level If GZIP_LEVEL has changed to a value different from
* CLOSURE->level, then set the compression level on the
* stream to the new value.
*/
static int
compress_buffer_output (void *closure, const char *data, size_t have,
size_t *wrote)
{
struct compress_buffer *cb = closure;
/* This is only used within the while loop below, but allocated here for
* efficiency.
*/
static char *buffer = NULL;
if (!buffer)
buffer = pagealign_xalloc (BUFFER_DATA_SIZE);
if (cb->level != gzip_level)
{
cb->level = gzip_level;
deflateParams (&cb->zstr, gzip_level, Z_DEFAULT_STRATEGY);
}
cb->zstr.avail_in = have;
cb->zstr.next_in = (unsigned char *) data;
while (cb->zstr.avail_in > 0)
{
int zstatus;
cb->zstr.avail_out = BUFFER_DATA_SIZE;
cb->zstr.next_out = (unsigned char *) buffer;
zstatus = deflate (&cb->zstr, Z_NO_FLUSH);
if (zstatus != Z_OK)
{
compress_error (0, zstatus, &cb->zstr, "deflate");
return EIO;
}
if (cb->zstr.avail_out != BUFFER_DATA_SIZE)
buf_output (cb->buf, buffer,
BUFFER_DATA_SIZE - cb->zstr.avail_out);
}
*wrote = have;
/* We will only be here because buf_send_output was called on the
compression buffer. That means that we should now call
buf_send_output on the underlying buffer. */
return buf_send_output (cb->buf);
}
/* ===========================================================================
* Update the compression level and strategy
*/
int lib_gzsetparams (
gzFile file,
int level,
int strategy)
{
gz_stream *s = (gz_stream*)file;
if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR;
/* Make room to allow flushing */
if (s->stream.avail_out == 0) {
s->stream.next_out = s->outbuf;
if (F_WRITE(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) {
s->z_err = Z_ERRNO;
}
s->stream.avail_out = Z_BUFSIZE;
}
return deflateParams (&(s->stream), level, strategy);
}
JNIEXPORT void JNICALL
Java_java_util_zip_Deflater_setLevelsImpl (JNIEnv * env, jobject recv,
int level, int strategy,
jlong handle)
{
JCLZipStream *stream;
jbyte b = 0;
int err = 0;
if (handle == -1)
{
throwNewIllegalStateException (env, "");
return;
}
stream = (JCLZipStream *) ((IDATA) handle);
stream->stream->next_out = (Bytef *) & b;
err = deflateParams (stream->stream, level, strategy);
if (err != Z_OK) {
THROW_ZIP_EXCEPTION(env, err, IllegalStateException);
}
}
/* -- see zlib.h -- */
int ZEXPORT gzsetparams(
gzFile file,
int level,
int strategy)
{
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return Z_STREAM_ERROR;
/* if no change is requested, then do nothing */
if (level == state->level && strategy == state->strategy)
return Z_OK;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return -1;
}
/* change compression parameters for subsequent input */
if (state->size) {
/* flush previous input with previous parameters before changing */
if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1)
return state->err;
deflateParams(strm, level, strategy);
}
state->level = level;
state->strategy = strategy;
return Z_OK;
}
static int
ZIPVSetField(TIFF* tif, ttag_t tag, va_list ap) {
ZIPState* sp = ZState(tif);
static const char module[] = "ZIPVSetField";
switch (tag) {
case TIFFTAG_ZIPQUALITY:
sp->zipquality = va_arg(ap, int);
if (tif->tif_mode != O_RDONLY && (sp->state & ZSTATE_INIT)) {
if (deflateParams(&sp->stream,
sp->zipquality, Z_DEFAULT_STRATEGY) != Z_OK) {
TIFFError(module, "%s: zlib error: %s",
tif->tif_name, sp->stream.msg);
return (0);
}
}
return (1);
default:
return (*sp->vsetparent)(tif, tag, ap);
}
/*NOTREACHED*/
}
static int
ZIPVSetField(TIFF* tif, uint32 tag, va_list ap)
{
static const char module[] = "ZIPVSetField";
ZIPState* sp = ZState(tif);
switch (tag) {
case TIFFTAG_ZIPQUALITY:
sp->zipquality = (int) va_arg(ap, int);
if ( sp->state&ZSTATE_INIT_ENCODE ) {
if (deflateParams(&sp->stream,
sp->zipquality, Z_DEFAULT_STRATEGY) != Z_OK) {
TIFFErrorExt(tif->tif_clientdata, module, "ZLib error: %s",
SAFE_MSG(sp));
return (0);
}
}
return (1);
default:
return (*sp->vsetparent)(tif, tag, ap);
}
/*NOTREACHED*/
}
/*!
*/
bool
gzfilterstreambuf::setLevel( const int level )
{
bool ret = false;
#ifdef HAVE_LIBZ
if ( level == DEFAULT_COMPRESSION
|| ( NO_COMPRESSION <= level
&& level <= BEST_COMPRESSION )
)
{
//std::cerr << "set compression level = " << level << std::endl;
// make sure there is room for the deflate to flush
this->sync();
deflateParams( M_impl->comp_stream_,
level, Z_DEFAULT_STRATEGY );
// write data flushed by deflateParams
this->sync();
M_level = level;
ret = true;
}
// Without zlib, this class cannot handle any compression level.
#endif
return ret;
}
/*
* Send data read from an already open file descriptor.
*
* We return 1 on sucess and 0 on errors.
*
* ***FIXME***
* We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
* reading.
* Currently this is not a problem as the only other stream, resource forks,
* are not handled as sparse files.
*/
static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
DIGEST *signing_digest)
{
BSOCK *sd = jcr->store_bsock;
uint64_t fileAddr = 0; /* file address */
char *rbuf, *wbuf;
int32_t rsize = jcr->buf_size; /* read buffer size */
POOLMEM *msgsave;
CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
const uint8_t *cipher_input;
uint32_t cipher_input_len;
uint32_t cipher_block_size;
uint32_t encrypted_len;
#ifdef FD_NO_SEND_TEST
return 1;
#endif
msgsave = sd->msg;
rbuf = sd->msg; /* read buffer */
wbuf = sd->msg; /* write buffer */
cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
#ifdef HAVE_LIBZ
uLong compress_len = 0;
uLong max_compress_len = 0;
const Bytef *cbuf = NULL;
int zstat;
if (ff_pkt->flags & FO_GZIP) {
if (ff_pkt->flags & FO_SPARSE) {
cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
} else {
cbuf = (Bytef *)jcr->compress_buf;
max_compress_len = jcr->compress_buf_size; /* set max length */
}
wbuf = jcr->compress_buf; /* compressed output here */
cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
/*
* Only change zlib parameters if there is no pending operation.
* This should never happen as deflatereset is called after each
* deflate.
*/
if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
/* set gzip compression level - must be done per file */
if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
}
}
#else
const uint32_t max_compress_len = 0;
#endif
if (ff_pkt->flags & FO_ENCRYPT) {
if (ff_pkt->flags & FO_SPARSE) {
Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
goto err;
}
/* Allocate the cipher context */
if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
&cipher_block_size)) == NULL) {
/* Shouldn't happen! */
Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
goto err;
}
/*
* Grow the crypto buffer, if necessary.
* crypto_cipher_update() will buffer up to (cipher_block_size - 1).
* We grow crypto_buf to the maximum number of blocks that
* could be returned for the given read buffer size.
* (Using the larger of either rsize or max_compress_len)
*/
jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
(MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
cipher_block_size - 1) / cipher_block_size * cipher_block_size);
wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
}
/*
* Send Data header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
/*
* Make space at beginning of buffer for fileAddr because this
* same buffer will be used for writing if compression is off.
*/
if (ff_pkt->flags & FO_SPARSE) {
rbuf += SPARSE_FADDR_SIZE;
rsize -= SPARSE_FADDR_SIZE;
#ifdef HAVE_FREEBSD_OS
/*
* To read FreeBSD partitions, the read size must be
* a multiple of 512.
*/
rsize = (rsize/512) * 512;
#endif
}
/* a RAW device read on win32 only works if the buffer is a multiple of 512 */
#ifdef HAVE_WIN32
if (S_ISBLK(ff_pkt->statp.st_mode))
rsize = (rsize/512) * 512;
#endif
/*
* Read the file data
*/
while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
/* Check for sparse blocks */
if (ff_pkt->flags & FO_SPARSE) {
ser_declare;
bool allZeros = false;
if ((sd->msglen == rsize &&
fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
(uint64_t)ff_pkt->statp.st_size == 0)) {
allZeros = is_buf_zero(rbuf, rsize);
}
if (!allZeros) {
/* Put file address as first data in buffer */
ser_begin(wbuf, SPARSE_FADDR_SIZE);
ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
}
fileAddr += sd->msglen; /* update file address */
/* Skip block of all zeros */
if (allZeros) {
continue; /* skip block of zeros */
}
}
jcr->ReadBytes += sd->msglen; /* count bytes read */
/* Uncompressed cipher input length */
cipher_input_len = sd->msglen;
/* Update checksum if requested */
if (digest) {
crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
}
/* Update signing digest if requested */
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
}
#ifdef HAVE_LIBZ
/* Do compression if turned on */
if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
/* reset zlib stream to be able to begin from scratch again */
if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
sd->msglen);
sd->msglen = compress_len; /* set compressed length */
cipher_input_len = compress_len;
}
#endif
/*
* Note, here we prepend the current record length to the beginning
* of the encrypted data. This is because both sparse and compression
* restore handling want records returned to them with exactly the
* same number of bytes that were processed in the backup handling.
* That is, both are block filters rather than a stream. When doing
* compression, the compression routines may buffer data, so that for
* any one record compressed, when it is decompressed the same size
* will not be obtained. Of course, the buffered data eventually comes
* out in subsequent crypto_cipher_update() calls or at least
* when crypto_cipher_finalize() is called. Unfortunately, this
* "feature" of encryption enormously complicates the restore code.
*/
if (ff_pkt->flags & FO_ENCRYPT) {
uint32_t initial_len = 0;
ser_declare;
if (ff_pkt->flags & FO_SPARSE) {
cipher_input_len += SPARSE_FADDR_SIZE;
}
/* Encrypt the length of the input block */
uint8_t packet_len[sizeof(uint32_t)];
ser_begin(packet_len, sizeof(uint32_t));
ser_uint32(cipher_input_len); /* store data len in begin of buffer */
Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
(uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
/* Encrypt the input block */
if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
(uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
if ((initial_len + encrypted_len) == 0) {
/* No full block of data available, read more data */
continue;
}
Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
sd->msglen);
sd->msglen = initial_len + encrypted_len; /* set encrypted length */
} else {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
}
/* Send the buffer to the Storage daemon */
if (ff_pkt->flags & FO_SPARSE) {
sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
}
sd->msg = wbuf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
/* #endif */
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore read buffer */
} /* end while read file data */
if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
if (jcr->JobErrors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
}
} else if (ff_pkt->flags & FO_ENCRYPT) {
/*
* For encryption, we must call finalize to push out any
* buffered data.
*/
if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
&encrypted_len)) {
/* Padding failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
goto err;
}
/* Note, on SSL pre-0.9.7, there is always some output */
if (encrypted_len > 0) {
sd->msglen = encrypted_len; /* set encrypted length */
sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore bnet buffer */
}
}
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
return 1;
err:
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
sd->msg = msgsave; /* restore bnet buffer */
sd->msglen = 0;
return 0;
}
bool setup_compression_context(b_ctx &bctx)
{
bool retval = false;
if ((bctx.ff_pkt->flags & FO_COMPRESS)) {
/*
* See if we need to be compatible with the old GZIP stream encoding.
*/
if (!me->compatible || bctx.ff_pkt->Compress_algo != COMPRESS_GZIP) {
memset(&bctx.ch, 0, sizeof(comp_stream_header));
/*
* Calculate buffer offsets.
*/
if ((bctx.ff_pkt->flags & FO_SPARSE) ||
(bctx.ff_pkt->flags & FO_OFFSETS)) {
bctx.chead = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE;
bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - (sizeof(comp_stream_header) + OFFSET_FADDR_SIZE);
} else {
bctx.chead = (uint8_t *)bctx.jcr->compress.deflate_buffer;
bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + sizeof(comp_stream_header);
bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - sizeof(comp_stream_header);
}
bctx.wbuf = bctx.jcr->compress.deflate_buffer; /* compressed output here */
bctx.cipher_input = (uint8_t *)bctx.jcr->compress.deflate_buffer; /* encrypt compressed data */
bctx.ch.magic = bctx.ff_pkt->Compress_algo;
bctx.ch.version = COMP_HEAD_VERSION;
} else {
/*
* Calculate buffer offsets.
*/
bctx.chead = NULL;
if ((bctx.ff_pkt->flags & FO_SPARSE) ||
(bctx.ff_pkt->flags & FO_OFFSETS)) {
bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer + OFFSET_FADDR_SIZE;
bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size - OFFSET_FADDR_SIZE;
} else {
bctx.cbuf = (uint8_t *)bctx.jcr->compress.deflate_buffer;
bctx.max_compress_len = bctx.jcr->compress.deflate_buffer_size;
}
bctx.wbuf = bctx.jcr->compress.deflate_buffer; /* compressed output here */
bctx.cipher_input = (uint8_t *)bctx.jcr->compress.deflate_buffer; /* encrypt compressed data */
}
/*
* Do compression specific actions and set the magic, header version and compression level.
*/
switch (bctx.ff_pkt->Compress_algo) {
#if defined(HAVE_LIBZ)
case COMPRESS_GZIP: {
z_stream *pZlibStream;
/**
* Only change zlib parameters if there is no pending operation.
* This should never happen as deflateReset is called after each
* deflate.
*/
pZlibStream = (z_stream *)bctx.jcr->compress.workset.pZLIB;
if (pZlibStream->total_in == 0) {
int zstat;
/*
* Set gzip compression level - must be done per file
*/
if ((zstat = deflateParams(pZlibStream, bctx.ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
Jmsg(bctx.jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
bctx.jcr->setJobStatus(JS_ErrorTerminated);
goto bail_out;
}
}
bctx.ch.level = bctx.ff_pkt->Compress_level;
break;
}
#endif
#if defined(HAVE_LZO)
case COMPRESS_LZO1X:
break;
#endif
#if defined(HAVE_FASTLZ)
case COMPRESS_FZFZ:
case COMPRESS_FZ4L:
case COMPRESS_FZ4H: {
int zstat;
zfast_stream *pZfastStream;
zfast_stream_compressor compressor = COMPRESSOR_FASTLZ;
/**
* Only change fastlz parameters if there is no pending operation.
* This should never happen as fastlzlibCompressReset is called after each
* fastlzlibCompress.
*/
pZfastStream = (zfast_stream *)bctx.jcr->compress.workset.pZFAST;
if (pZfastStream->total_in == 0) {
switch (bctx.ff_pkt->Compress_algo) {
case COMPRESS_FZ4L:
case COMPRESS_FZ4H:
compressor = COMPRESSOR_LZ4;
break;
}
if ((zstat = fastlzlibSetCompressor(pZfastStream, compressor)) != Z_OK) {
Jmsg(bctx.jcr, M_FATAL, 0, _("Compression fastlzlibSetCompressor error: %d\n"), zstat);
bctx.jcr->setJobStatus(JS_ErrorTerminated);
goto bail_out;
}
}
bctx.ch.level = bctx.ff_pkt->Compress_level;
break;
}
#endif
default:
break;
}
}
retval = true;
bail_out:
return retval;
}
int
vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen,
int zlibLevel, int zlibStrategy)
{
if (dataLen < TIGHT_MIN_TO_COMPRESS) {
memcpy(dest, m_buffer, dataLen);
return dataLen;
}
z_streamp pz = &m_zsStruct[streamId];
// Initialize compression stream if needed.
if (!m_zsActive[streamId]) {
pz->zalloc = Z_NULL;
pz->zfree = Z_NULL;
pz->opaque = Z_NULL;
vnclog.Print(LL_INTINFO,
VNCLOG("calling deflateInit2 with zlib level:%d\n"),
zlibLevel);
int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, zlibStrategy);
if (err != Z_OK) {
vnclog.Print(LL_INTINFO,
VNCLOG("deflateInit2 returned error:%d:%s\n"),
err, pz->msg);
return -1;
}
m_zsActive[streamId] = true;
m_zsLevel[streamId] = zlibLevel;
}
int outBufferSize = dataLen + dataLen / 100 + 16;
// Prepare buffer pointers.
pz->next_in = (Bytef *)m_buffer;
pz->avail_in = dataLen;
pz->next_out = (Bytef *)dest;
pz->avail_out = outBufferSize;
// Change compression parameters if needed.
if (zlibLevel != m_zsLevel[streamId]) {
vnclog.Print(LL_INTINFO,
VNCLOG("calling deflateParams with zlib level:%d\n"),
zlibLevel);
int err = deflateParams (pz, zlibLevel, zlibStrategy);
if (err != Z_OK) {
vnclog.Print(LL_INTINFO,
VNCLOG("deflateParams returned error:%d:%s\n"),
err, pz->msg);
return -1;
}
m_zsLevel[streamId] = zlibLevel;
}
// Actual compression.
if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
pz->avail_in != 0 || pz->avail_out == 0 ) {
vnclog.Print(LL_INTINFO, VNCLOG("deflate() call failed.\n"));
return -1;
}
return SendCompressedData(outBufferSize - pz->avail_out);
}
/*
* Setup deflate for auto deflate of data streams.
*/
static bool setup_auto_deflation(DCR *dcr)
{
JCR *jcr = dcr->jcr;
bool retval = false;
uint32_t compress_buf_size = 0;
if (jcr->buf_size == 0) {
jcr->buf_size = DEFAULT_NETWORK_BUFFER_SIZE;
}
if (!setup_compression_buffers(jcr, sd_enabled_compatible,
dcr->device->autodeflate_algorithm,
&compress_buf_size)) {
goto bail_out;
}
/*
* See if we need to create a new compression buffer or make sure the existing is big enough.
*/
if (!jcr->compress.deflate_buffer) {
jcr->compress.deflate_buffer = get_memory(compress_buf_size);
jcr->compress.deflate_buffer_size = compress_buf_size;
} else {
if (compress_buf_size > jcr->compress.deflate_buffer_size) {
jcr->compress.deflate_buffer = realloc_pool_memory(jcr->compress.deflate_buffer, compress_buf_size);
jcr->compress.deflate_buffer_size = compress_buf_size;
}
}
switch (dcr->device->autodeflate_algorithm) {
#if defined(HAVE_LIBZ)
case COMPRESS_GZIP: {
int zstat;
z_stream *pZlibStream;
pZlibStream = (z_stream *)jcr->compress.workset.pZLIB;
if ((zstat = deflateParams(pZlibStream, dcr->device->autodeflate_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
jcr->setJobStatus(JS_ErrorTerminated);
goto bail_out;
}
break;
}
#endif
#if defined(HAVE_LZO)
case COMPRESS_LZO1X:
break;
#endif
#if defined(HAVE_FASTLZ)
case COMPRESS_FZFZ:
case COMPRESS_FZ4L:
case COMPRESS_FZ4H: {
int zstat;
zfast_stream *pZfastStream;
zfast_stream_compressor compressor = COMPRESSOR_FASTLZ;
switch (dcr->device->autodeflate_algorithm) {
case COMPRESS_FZ4L:
case COMPRESS_FZ4H:
compressor = COMPRESSOR_LZ4;
break;
}
pZfastStream = (zfast_stream *)jcr->compress.workset.pZFAST;
if ((zstat = fastlzlibSetCompressor(pZfastStream, compressor)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression fastlzlibSetCompressor error: %d\n"), zstat);
jcr->setJobStatus(JS_ErrorTerminated);
goto bail_out;
}
break;
}
#endif
default:
break;
}
retval = true;
bail_out:
return retval;
}
static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf,
ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
{
ZLibData* d = (ZLibData*)drv_data;
int res;
switch(command) {
case DEFLATE_INIT:
if (len != 4) goto badarg;
if (d->state != ST_NONE) goto badarg;
res = deflateInit(&d->s, i32(buf));
if (res == Z_OK) {
d->state = ST_DEFLATE;
d->want_crc = 0;
d->crc = crc32(0L, Z_NULL, 0);
}
return zlib_return(res, rbuf, rlen);
case DEFLATE_INIT2: {
int wbits;
if (len != 20) goto badarg;
if (d->state != ST_NONE) goto badarg;
wbits = i32(buf+8);
res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits,
i32(buf+12), i32(buf+16));
if (res == Z_OK) {
d->state = ST_DEFLATE;
d->want_crc = (wbits < 0);
d->crc = crc32(0L, Z_NULL, 0);
}
return zlib_return(res, rbuf, rlen);
}
case DEFLATE_SETDICT:
if (d->state != ST_DEFLATE) goto badarg;
res = deflateSetDictionary(&d->s, (unsigned char*)buf, len);
if (res == Z_OK) {
return zlib_value(d->s.adler, rbuf, rlen);
} else {
return zlib_return(res, rbuf, rlen);
}
case DEFLATE_RESET:
if (len != 0) goto badarg;
if (d->state != ST_DEFLATE) goto badarg;
driver_deq(d->port, driver_sizeq(d->port));
res = deflateReset(&d->s);
return zlib_return(res, rbuf, rlen);
case DEFLATE_END:
if (len != 0) goto badarg;
if (d->state != ST_DEFLATE) goto badarg;
driver_deq(d->port, driver_sizeq(d->port));
res = deflateEnd(&d->s);
d->state = ST_NONE;
return zlib_return(res, rbuf, rlen);
case DEFLATE_PARAMS:
if (len != 8) goto badarg;
if (d->state != ST_DEFLATE) goto badarg;
res = deflateParams(&d->s, i32(buf), i32(buf+4));
return zlib_return(res, rbuf, rlen);
case DEFLATE:
if (d->state != ST_DEFLATE) goto badarg;
if (len != 4) goto badarg;
res = zlib_deflate(d, i32(buf));
return zlib_return(res, rbuf, rlen);
case INFLATE_INIT:
if (len != 0) goto badarg;
if (d->state != ST_NONE) goto badarg;
res = inflateInit(&d->s);
if (res == Z_OK) {
d->state = ST_INFLATE;
d->inflate_eos_seen = 0;
d->want_crc = 0;
d->crc = crc32(0L, Z_NULL, 0);
}
return zlib_return(res, rbuf, rlen);
case INFLATE_INIT2: {
int wbits;
if (len != 4) goto badarg;
if (d->state != ST_NONE) goto badarg;
wbits = i32(buf);
res = inflateInit2(&d->s, wbits);
if (res == Z_OK) {
d->state = ST_INFLATE;
d->inflate_eos_seen = 0;
d->want_crc = (wbits < 0);
d->crc = crc32(0L, Z_NULL, 0);
}
return zlib_return(res, rbuf, rlen);
}
case INFLATE_SETDICT:
if (d->state != ST_INFLATE) goto badarg;
res = inflateSetDictionary(&d->s, (unsigned char*)buf, len);
return zlib_return(res, rbuf, rlen);
case INFLATE_SYNC:
if (d->state != ST_INFLATE) goto badarg;
if (len != 0) goto badarg;
if (driver_sizeq(d->port) == 0) {
res = Z_BUF_ERROR;
} else {
int vlen;
SysIOVec* iov = driver_peekq(d->port, &vlen);
d->s.next_in = iov[0].iov_base;
d->s.avail_in = iov[0].iov_len;
res = inflateSync(&d->s);
}
return zlib_return(res, rbuf, rlen);
case INFLATE_RESET:
if (d->state != ST_INFLATE) goto badarg;
if (len != 0) goto badarg;
driver_deq(d->port, driver_sizeq(d->port));
res = inflateReset(&d->s);
d->inflate_eos_seen = 0;
return zlib_return(res, rbuf, rlen);
case INFLATE_END:
if (d->state != ST_INFLATE) goto badarg;
if (len != 0) goto badarg;
driver_deq(d->port, driver_sizeq(d->port));
res = inflateEnd(&d->s);
if (res == Z_OK && d->inflate_eos_seen == 0) {
res = Z_DATA_ERROR;
}
d->state = ST_NONE;
return zlib_return(res, rbuf, rlen);
case INFLATE:
if (d->state != ST_INFLATE) goto badarg;
if (len != 4) goto badarg;
res = zlib_inflate(d, i32(buf));
if (res == Z_NEED_DICT) {
return zlib_value2(3, d->s.adler, rbuf, rlen);
} else {
return zlib_return(res, rbuf, rlen);
}
case GET_QSIZE:
return zlib_value(driver_sizeq(d->port), rbuf, rlen);
case GET_BUFSZ:
return zlib_value(d->binsz_need, rbuf, rlen);
case SET_BUFSZ: {
int need;
if (len != 4) goto badarg;
need = i32(buf);
if ((need < 16) || (need > 0x00ffffff))
goto badarg;
if (d->binsz_need != need) {
d->binsz_need = need;
if (d->bin != NULL) {
if (d->s.avail_out == d->binsz) {
driver_free_binary(d->bin);
d->bin = NULL;
d->binsz = 0;
}
else
zlib_output(d);
}
}
return zlib_return(Z_OK, rbuf, rlen);
}
case CRC32_0:
return zlib_value(d->crc, rbuf, rlen);
case CRC32_1: {
uLong crc = crc32(0L, Z_NULL, 0);
crc = crc32(crc, (unsigned char*) buf, len);
return zlib_value(crc, rbuf, rlen);
}
case CRC32_2: {
uLong crc;
if (len < 4) goto badarg;
crc = (unsigned int) i32(buf);
crc = crc32(crc, (unsigned char*) buf+4, len-4);
return zlib_value(crc, rbuf, rlen);
}
case ADLER32_1: {
uLong adler = adler32(0L, Z_NULL, 0);
adler = adler32(adler, (unsigned char*) buf, len);
return zlib_value(adler, rbuf, rlen);
}
case ADLER32_2: {
uLong adler;
if (len < 4) goto badarg;
adler = (unsigned int) i32(buf);
adler = adler32(adler, (unsigned char*) buf+4, len-4);
return zlib_value(adler, rbuf, rlen);
}
case CRC32_COMBINE: {
uLong crc, crc1, crc2, len2;
if (len != 12) goto badarg;
crc1 = (unsigned int) i32(buf);
crc2 = (unsigned int) i32(buf+4);
len2 = (unsigned int) i32(buf+8);
crc = crc32_combine(crc1, crc2, len2);
return zlib_value(crc, rbuf, rlen);
}
case ADLER32_COMBINE: {
uLong adler, adler1, adler2, len2;
if (len != 12) goto badarg;
adler1 = (unsigned int) i32(buf);
adler2 = (unsigned int) i32(buf+4);
len2 = (unsigned int) i32(buf+8);
adler = adler32_combine(adler1, adler2, len2);
return zlib_value(adler, rbuf, rlen);
}
}
badarg:
errno = EINVAL;
return zlib_return(Z_ERRNO, rbuf, rlen);
}
