int testStreamingAPI(void)
{
size_t const outBuffSize = ZSTD_DStreamOutSize();
char* const outBuff = malloc(outBuffSize);
ZSTD_DStream* const stream = ZSTD_createDStream();
ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 };
size_t outputPos = 0;
int needsInit = 1;
if (outBuff == NULL) {
DISPLAY("ERROR: Could not allocate memory\n");
return 1;
}
if (stream == NULL) {
DISPLAY("ERROR: Could not create dstream\n");
return 1;
}
while (1) {
ZSTD_outBuffer output = {outBuff, outBuffSize, 0};
if (needsInit) {
size_t const ret = ZSTD_initDStream(stream);
if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
return 1;
}
}
{
size_t const ret = ZSTD_decompressStream(stream, &output, &input);
if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
return 1;
}
if (ret == 0) {
needsInit = 1;
}
}
if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
return 1;
}
outputPos += output.pos;
if (input.pos == input.size && output.pos < output.size) {
break;
}
}
free(outBuff);
ZSTD_freeDStream(stream);
DISPLAY("Streaming API OK\n");
return 0;
}
int testSimpleAPI(void)
{
size_t const size = strlen(EXPECTED);
char* const output = malloc(size);
if (!output) {
DISPLAY("ERROR: Not enough memory!\n");
return 1;
}
{
size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE);
if (ZSTD_isError(ret)) {
if (ret == ZSTD_error_prefix_unknown) {
DISPLAY("ERROR: Invalid frame magic number, was this compiled "
"without legacy support?\n");
} else {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
}
return 1;
}
if (ret != size) {
DISPLAY("ERROR: Wrong decoded size\n");
}
}
if (memcmp(EXPECTED, output, size) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
return 1;
}
free(output);
DISPLAY("Simple API OK\n");
return 0;
}
void zstdThrowIfError(size_t rc) {
if (!ZSTD_isError(rc)) {
return;
}
throw std::runtime_error(
to<std::string>("ZSTD returned an error: ", ZSTD_getErrorName(rc)));
}
static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFileX(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown \n", fname);
exit(5);
}
void* const rBuff = mallocX(rSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
ZSTD_freeDCtx(dctx);
free(rBuff);
free(cBuff);
}
void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
size_t res = ZSTD_decompress(dest, uncompressed_size, source, source_size);
if (ZSTD_isError(res))
throw Exception("Cannot ZSTD_decompress: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_DECOMPRESS);
}
const char* up_error_msg( int err_code ) {
if (err_code == ZSTD_NO_ERROR) {
return NULL;
} else {
return ZSTD_getErrorName( (size_t)err_code );
}
}
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
exit(5);
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
fprintf(stderr,
"%s : original size unknown. Use streaming decompression instead.\n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
exit(5);
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
fprintf(stderr, "%s : original size unknown \n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
if (dSize != rSize) {
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
ZSTD_freeDCtx(dctx);
free(rBuff);
free(cBuff);
}
static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#", &source, &source_size))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#", &source, &source_size))
return NULL;
#endif
header_size = sizeof(dest_size);
memcpy(&dest_size, source, header_size);
result = PyBytes_FromStringAndSize(NULL, dest_size);
source += header_size;
if (result != NULL && dest_size > 0) {
char *dest = PyBytes_AS_STRING(result);
cSize = ZSTD_decompress(dest, dest_size, source, source_size - header_size);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Decompression error: %s", ZSTD_getErrorName(cSize));
}
return result;
}
static ezResult DecompressZStd(ezArrayPtr<const ezUInt8> pCompressedData, ezDynamicArray<ezUInt8>& out_Data)
{
size_t uiSize = ZSTD_findDecompressedSize(pCompressedData.GetPtr(), pCompressedData.GetCount());
if (uiSize == ZSTD_CONTENTSIZE_ERROR)
{
ezLog::Error("Can't decompress since it wasn't compressed with ZStd");
return EZ_FAILURE;
}
else if (uiSize == ZSTD_CONTENTSIZE_UNKNOWN)
{
ezLog::Error("Can't decompress since the original size can't be determined, was the data compressed using the streaming variant?");
return EZ_FAILURE;
}
if (uiSize > std::numeric_limits<ezUInt32>::max())
{
ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSize));
return EZ_FAILURE;
}
out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSize));
size_t const uiActualSize = ZSTD_decompress(out_Data.GetData(), uiSize, pCompressedData.GetPtr(), pCompressedData.GetCount());
if (uiActualSize != uiSize)
{
ezLog::Error("Error during ZStd decompression: '{0}'.", ZSTD_getErrorName(uiActualSize));
return EZ_FAILURE;
}
return EZ_SUCCESS;
}
UInt32 CompressionCodecZSTD::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
size_t compressed_size = ZSTD_compress(dest, ZSTD_compressBound(source_size), source, source_size, level);
if (ZSTD_isError(compressed_size))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(compressed_size)), ErrorCodes::CANNOT_COMPRESS);
return compressed_size;
}
static int decompress_zstd(const char *inbuf, char *outbuf, u64 compress_len,
u64 decompress_len)
{
#if !BTRFSRESTORE_ZSTD
error("btrfs not compiled with zstd support");
return -1;
#else
ZSTD_DStream *strm;
size_t zret;
int ret = 0;
ZSTD_inBuffer in = {inbuf, compress_len, 0};
ZSTD_outBuffer out = {outbuf, decompress_len, 0};
strm = ZSTD_createDStream();
if (!strm) {
error("zstd create failed");
return -1;
}
zret = ZSTD_initDStream(strm);
if (ZSTD_isError(zret)) {
error("zstd init failed: %s", ZSTD_getErrorName(zret));
ret = -1;
goto out;
}
zret = ZSTD_decompressStream(strm, &out, &in);
if (ZSTD_isError(zret)) {
error("zstd decompress failed %s\n", ZSTD_getErrorName(zret));
ret = -1;
goto out;
}
if (zret != 0) {
error("zstd frame incomplete");
ret = -1;
goto out;
}
out:
ZSTD_freeDStream(strm);
return ret;
#endif
}
static PyObject* ZstdCompressionWriter_exit(ZstdCompressionWriter* self, PyObject* args) {
PyObject* exc_type;
PyObject* exc_value;
PyObject* exc_tb;
size_t zresult;
ZSTD_outBuffer output;
PyObject* res;
if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
return NULL;
}
self->entered = 0;
if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) {
ZSTD_inBuffer inBuffer;
inBuffer.src = NULL;
inBuffer.size = 0;
inBuffer.pos = 0;
output.dst = PyMem_Malloc(self->outSize);
if (!output.dst) {
return PyErr_NoMemory();
}
output.size = self->outSize;
output.pos = 0;
while (1) {
zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &inBuffer, ZSTD_e_end);
if (ZSTD_isError(zresult)) {
PyErr_Format(ZstdError, "error ending compression stream: %s",
ZSTD_getErrorName(zresult));
PyMem_Free(output.dst);
return NULL;
}
if (output.pos) {
#if PY_MAJOR_VERSION >= 3
res = PyObject_CallMethod(self->writer, "write", "y#",
#else
res = PyObject_CallMethod(self->writer, "write", "s#",
#endif
output.dst, output.pos);
Py_XDECREF(res);
}
if (!zresult) {
break;
}
output.pos = 0;
}
static PyObject *py_zstd_compress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
char *dest;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
uint32_t level = 0;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#|i", &source, &source_size, &level))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#|i", &source, &source_size, &level))
return NULL;
#endif
if (level <= 0) level=1;
if (level > 20) level=20;
header_size = sizeof(source_size);
dest_size = ZSTD_compressBound(source_size);
result = PyBytes_FromStringAndSize(NULL, header_size + dest_size);
if (result == NULL) {
return NULL;
}
dest = PyBytes_AS_STRING(result);
memcpy(dest, &source_size, header_size);
dest += header_size;
if (source_size > 0) {
// Low level == old version
cSize = ZSTD_compress(dest, dest_size, source, source_size, level);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Compression error: %s", ZSTD_getErrorName(cSize));
Py_SIZE(result) = cSize + sizeof(source_size);
}
return result;
}
static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) {
size_t zresult;
if (self->entered) {
PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
return NULL;
}
zresult = ZSTD_CCtx_setPledgedSrcSize(self->compressor->cctx, self->sourceSize);
if (ZSTD_isError(zresult)) {
PyErr_Format(ZstdError, "error setting source size: %s",
ZSTD_getErrorName(zresult));
return NULL;
}
self->entered = 1;
Py_INCREF(self);
return (PyObject*)self;
}
/* ZSTDMT_compressChunk() : POOL_function type */
void ZSTDMT_compressChunk(void* jobDescription)
{
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t const dstBuff = job->dstBuff;
DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
if (job->cdict) { /* should only happen for first segment */
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
if (job->cdict) DEBUGLOG(3, "using CDict ");
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
{ size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */
size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
} }
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
ZSTD_invalidateRepCodes(job->cctx);
}
DEBUGLOG(4, "Compressing : ");
DEBUG_PRINTHEX(4, job->srcStart, 12);
job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
(unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
_endJob:
PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
job->jobCompleted = 1;
job->jobScanned = 0;
pthread_cond_signal(job->jobCompleted_cond);
pthread_mutex_unlock(job->jobCompleted_mutex);
}
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(const IOBuf* data,
uint64_t uncompressedLength) {
size_t rc;
auto out = IOBuf::createCombined(uncompressedLength);
CHECK_GE(out->capacity(), uncompressedLength);
CHECK_EQ(out->length(), 0);
rc = ZSTD_decompress(
out->writableTail(), out->capacity(), data->data(), data->length());
if (ZSTD_isError(rc)) {
throw std::runtime_error(to<std::string>(
"ZSTD decompression returned an error: ",
ZSTD_getErrorName(rc)));
}
out->append(rc);
CHECK_EQ(out->length(), rc);
return out;
}
static ezResult CompressZStd(ezArrayPtr<const ezUInt8> pUncompressedData, ezDynamicArray<ezUInt8>& out_Data)
{
size_t uiSizeBound = ZSTD_compressBound(pUncompressedData.GetCount());
if(uiSizeBound > std::numeric_limits<ezUInt32>::max())
{
ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSizeBound));
return EZ_FAILURE;
}
out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSizeBound));
size_t const cSize = ZSTD_compress(out_Data.GetData(), uiSizeBound, pUncompressedData.GetPtr(), pUncompressedData.GetCount(), 1);
if (ZSTD_isError(cSize))
{
ezLog::Error("Compression failed with error: '{0}'.", ZSTD_getErrorName(cSize));
return EZ_FAILURE;
}
out_Data.SetCount(static_cast<ezUInt32>(cSize));
return EZ_SUCCESS;
}
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
{
size_t fSize;
void* const fBuff = loadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(7);
}
saveFile_orDie(oname, cBuff, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
ZSTD_freeCCtx(cctx);
free(fBuff);
free(cBuff);
}
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown \n", fname);
exit(5);
}
void* const rBuff = malloc_X(rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
static ssize_t
zstd_filter_read(struct archive_read_filter *self, const void **p)
{
struct private_data *state;
size_t decompressed;
ssize_t avail_in;
ZSTD_outBuffer out;
ZSTD_inBuffer in;
state = (struct private_data *)self->data;
out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 };
/* Try to fill the output buffer. */
while (out.pos < out.size && !state->eof) {
if (!state->in_frame) {
const size_t ret = ZSTD_initDStream(state->dstream);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Error initializing zstd decompressor: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
}
in.src = __archive_read_filter_ahead(self->upstream, 1,
&avail_in);
if (avail_in < 0) {
return avail_in;
}
if (in.src == NULL && avail_in == 0) {
if (!state->in_frame) {
/* end of stream */
state->eof = 1;
break;
} else {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Truncated zstd input");
return (ARCHIVE_FATAL);
}
}
in.size = avail_in;
in.pos = 0;
{
const size_t ret =
ZSTD_decompressStream(state->dstream, &out, &in);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Zstd decompression failed: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
/* Decompressor made some progress */
__archive_read_filter_consume(self->upstream, in.pos);
/* ret guaranteed to be > 0 if frame isn't done yet */
state->in_frame = (ret != 0);
}
}
decompressed = out.pos;
state->total_out += decompressed;
if (decompressed == 0)
*p = NULL;
else
*p = state->out_block;
return (decompressed);
}
/*
* Setup state for decoding a strip.
*/
static int
LERCPreDecode(TIFF* tif, uint16 s)
{
static const char module[] = "LERCPreDecode";
lerc_status lerc_ret;
TIFFDirectory *td = &tif->tif_dir;
LERCState* sp = DecoderState(tif);
int lerc_data_type;
unsigned int infoArray[8];
unsigned nomask_bands = td->td_samplesperpixel;
int ndims;
int use_mask = 0;
uint8* lerc_data = tif->tif_rawcp;
unsigned int lerc_data_size = (unsigned int)tif->tif_rawcc;
(void) s;
assert(sp != NULL);
lerc_data_type = GetLercDataType(tif);
if( lerc_data_type < 0 )
return 0;
if( !SetupUncompressedBuffer(tif, sp, module) )
return 0;
if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
if( sp->compressed_size < sp->uncompressed_alloc )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(sp->uncompressed_alloc);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = sp->uncompressed_alloc;
}
}
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = inflateInit(&strm);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflateInit() failed");
inflateEnd(&strm);
return 0;
}
strm.avail_in = (uInt)tif->tif_rawcc;
strm.next_in = tif->tif_rawcp;
strm.avail_out = sp->compressed_size;
strm.next_out = sp->compressed_buffer;
zlib_ret = inflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END && zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflate() failed");
inflateEnd(&strm);
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = sp->compressed_size - strm.avail_out;
inflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret;
zstd_ret = ZSTD_decompress(sp->compressed_buffer,
sp->compressed_size,
tif->tif_rawcp,
tif->tif_rawcc);
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_decompress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = (unsigned int)zstd_ret;
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
lerc_ret = lerc_getBlobInfo(
lerc_data,
lerc_data_size,
infoArray,
NULL,
8,
0);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_getBlobInfo() failed");
return 0;
}
/* If the configuration is compatible of a LERC mask, and that the */
/* LERC info has dim == samplesperpixel - 1, then there is a LERC */
/* mask. */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 &&
infoArray[2] == td->td_samplesperpixel - 1U )
{
use_mask = 1;
nomask_bands --;
}
ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ?
nomask_bands : 1;
/* Info returned in infoArray is { version, dataType, nDim, nCols,
nRows, nBands, nValidPixels, blobSize } */
if( infoArray[0] != (unsigned)sp->lerc_version )
{
TIFFWarningExt(tif->tif_clientdata, module,
"Unexpected version number: %d. Expected: %d",
infoArray[0], sp->lerc_version);
}
if( infoArray[1] != (unsigned)lerc_data_type )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected dataType: %d. Expected: %d",
infoArray[1], lerc_data_type);
return 0;
}
if( infoArray[2] != (unsigned)ndims )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nDim: %d. Expected: %d",
infoArray[2], ndims);
return 0;
}
if( infoArray[3] != sp->segment_width )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nCols: %d. Expected: %du",
infoArray[3], sp->segment_width);
return 0;
}
if( infoArray[4] != sp->segment_height )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nRows: %d. Expected: %u",
infoArray[4], sp->segment_height);
return 0;
}
if( infoArray[5] != 1 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nBands: %d. Expected: %d",
infoArray[5], 1);
return 0;
}
if( infoArray[7] != lerc_data_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected blobSize: %d. Expected: %u",
infoArray[7],
lerc_data_size);
return 0;
}
lerc_ret = lerc_decode(
lerc_data,
lerc_data_size,
use_mask ? sp->mask_buffer : NULL,
ndims,
sp->segment_width,
sp->segment_height,
1,
lerc_data_type,
sp->uncompressed_buffer);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_decode() failed");
return 0;
}
/* Interleave alpha mask with other samples. */
if( use_mask )
{
unsigned src_stride =
(td->td_samplesperpixel - 1) * (td->td_bitspersample / 8);
unsigned dst_stride =
td->td_samplesperpixel * (td->td_bitspersample / 8);
unsigned i = sp->segment_width * sp->segment_height;
/* Operate from end to begin to be able to move in place */
while( i > 0 && i > nomask_bands )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
/* First pixels must use memmove due to overlapping areas */
while( i > 0 )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
}
return 1;
}
/*
* Finish off an encoded strip by flushing it.
*/
static int
LERCPostEncode(TIFF* tif)
{
lerc_status lerc_ret;
static const char module[] = "LERCPostEncode";
LERCState *sp = EncoderState(tif);
unsigned int numBytes = 0;
unsigned int numBytesWritten = 0;
TIFFDirectory *td = &tif->tif_dir;
int use_mask = 0;
unsigned dst_nbands = td->td_samplesperpixel;
if( sp->uncompressed_offset != sp->uncompressed_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected number of bytes in the buffer");
return 0;
}
/* Extract alpha mask (if containing only 0 and 255 values, */
/* and compact array of regular bands */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 )
{
unsigned dst_stride = (td->td_samplesperpixel - 1) *
(td->td_bitspersample / 8);
unsigned src_stride = td->td_samplesperpixel *
(td->td_bitspersample / 8);
unsigned i = 0;
unsigned nb_pixels = sp->segment_width * sp->segment_height;
use_mask = 1;
for( i = 0 ; i < nb_pixels; i++)
{
int v = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
if( v != 0 && v != 255 )
{
use_mask = 0;
break;
}
}
if( use_mask )
{
dst_nbands --;
/* First pixels must use memmove due to overlapping areas */
for( i = 0 ;i < dst_nbands && i < nb_pixels; i++)
{
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
for(; i < nb_pixels; i++)
{
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
}
}
#if 0
lerc_ret = lerc_computeCompressedSize(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
&numBytes);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_computeCompressedSize() failed");
return 0;
}
#else
numBytes = sp->uncompressed_alloc;
#endif
if( sp->compressed_size < numBytes )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(numBytes);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = numBytes;
}
lerc_ret = lerc_encodeForVersion(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
sp->compressed_buffer,
sp->compressed_size,
&numBytesWritten);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_encode() failed");
return 0;
}
assert( numBytesWritten < numBytes );
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = deflateInit(&strm, sp->zipquality);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflateInit() failed");
return 0;
}
strm.avail_in = numBytesWritten;
strm.next_in = sp->compressed_buffer;
strm.avail_out = sp->uncompressed_alloc;
strm.next_out = sp->uncompressed_buffer;
zlib_ret = deflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflate() failed");
deflateEnd(&strm);
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = sp->uncompressed_alloc - strm.avail_out;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
deflateEnd(&strm);
return 0;
}
}
deflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret = ZSTD_compress( sp->uncompressed_buffer,
sp->uncompressed_alloc,
sp->compressed_buffer,
numBytesWritten,
sp->zstd_compress_level );
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_compress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = zstd_ret;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
return 0;
}
}
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
else
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->compressed_buffer;
tif->tif_rawcc = numBytesWritten;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
return 0;
}
return 1;
}
static gint
rspamd_client_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_client_request *req =
(struct rspamd_client_request *)conn->ud;
struct rspamd_client_connection *c;
struct ucl_parser *parser;
GError *err;
const rspamd_ftok_t *tok;
c = req->conn;
if (!c->req_sent) {
c->req_sent = TRUE;
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
c->fd,
&c->timeout,
c->ev_base);
return 0;
}
else {
if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) {
err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
msg->code,
(gint)msg->status->len, msg->status->str);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
tok = rspamd_http_message_find_header (msg, "compression");
if (tok) {
/* Need to uncompress */
rspamd_ftok_t t;
t.begin = "zstd";
t.len = 4;
if (rspamd_ftok_casecmp (tok, &t) == 0) {
ZSTD_DStream *zstream;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
guchar *out;
gsize outlen, r;
zstream = ZSTD_createDStream ();
ZSTD_initDStream (zstream);
zin.pos = 0;
zin.src = msg->body_buf.begin;
zin.size = msg->body_buf.len;
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
outlen = ZSTD_DStreamOutSize ();
}
out = g_malloc (outlen);
zout.dst = out;
zout.pos = 0;
zout.size = outlen;
while (zin.pos < zin.size) {
r = ZSTD_decompressStream (zstream, &zout, &zin);
if (ZSTD_isError (r)) {
err = g_error_new (RCLIENT_ERROR, 500,
"Decompression error: %s",
ZSTD_getErrorName (r));
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
ZSTD_freeDStream (zstream);
g_free (out);
return 0;
}
if (zout.pos == zout.size) {
/* We need to extend output buffer */
zout.size = zout.size * 1.5 + 1.0;
zout.dst = g_realloc (zout.dst, zout.size);
}
}
ZSTD_freeDStream (zstream);
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, zout.dst, zout.pos)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
g_free (zout.dst);
return 0;
}
g_free (zout.dst);
}
else {
err = g_error_new (RCLIENT_ERROR, 500,
"Invalid compression method");
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
else {
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
req->cb (c, msg, c->server_name->str, ucl_parser_get_object (
parser), req->input, req->ud, NULL);
ucl_parser_free (parser);
}
return 0;
}
static int
pkg_archive_cookie_read(void *p, char *buf, int size)
{
struct pkg_archive_stdio_cookie *cookie = p;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
ssize_t r, ret;
r = MIN(size, cookie->remain);
if (cookie->zstream == NULL) {
/* Not a compressed payload */
r = fread(buf, 1, r, cookie->fs);
if (r > 0) {
cookie->remain -= r;
}
return (r);
}
else {
if (cookie->rd_offset > 0 && cookie->rd_offset < cookie->rd_len / 2) {
/* Move rd_offset bytes and read more */
r = fread(cookie->rdbuf + cookie->rd_offset, 1,
MIN(cookie->rd_len - cookie->rd_offset, cookie->remain),
cookie->fs);
if (r > 0) {
cookie->remain -= r;
r += cookie->rd_offset;
cookie->rd_offset = r;
}
else {
r = cookie->rd_offset;
}
}
else if (cookie->rd_offset > 0) {
/* Just process the remaining buffer */
r = cookie->rd_len - cookie->rd_offset;
}
else {
/* Buffer is empty, read chunk */
r = fread(cookie->rdbuf, 1,
MIN(cookie->rd_len, cookie->remain), cookie->fs);
if (r > 0) {
cookie->remain -= r;
cookie->rd_offset = r;
}
else {
return (r);
}
}
zin.size = r;
zin.src = cookie->rdbuf;
zin.pos = 0;
zout.size = size;
zout.dst = buf;
zout.pos = 0;
while (zin.pos < zin.size) {
ret = ZSTD_decompressStream(cookie->zstream, &zout, &zin);
if (ZSTD_isError(ret)) {
pkg_emit_error("cannot decompress data: %s",
ZSTD_getErrorName(ret));
return (-1);
}
if (zout.pos == zout.size) {
/* We are done */
cookie->rd_offset = zin.size - zin.pos;
/* Move leftover */
memmove (cookie->rdbuf, cookie->rdbuf + cookie->rd_offset,
cookie->rd_len - cookie->rd_offset);
break;
}
}
return (zout.pos);
}
/* Not reached */
return (-1);
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** Формат сжатого блока - см. CompressedStream.h
*/
switch (method)
{
case CompressionMethod::QuickLZ:
{
#ifdef USE_QUICKLZ
compressed_buffer.resize(uncompressed_size + QUICKLZ_ADDITIONAL_SPACE);
compressed_size = qlz_compress(
working_buffer.begin(),
&compressed_buffer[0],
uncompressed_size,
qlz_state.get());
compressed_buffer[0] &= 3;
compressed_buffer_ptr = &compressed_buffer[0];
break;
#else
throw Exception("QuickLZ compression method is disabled", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
#endif
}
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
else
compressed_size = header_size + LZ4_compressHC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
uint128 checksum = CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* refCtx;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
refCtx = ZSTD_createCCtx();
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
int cLevel;
BYTE* sampleBuffer;
const BYTE* dict;
size_t dictSize;
/* init */
if (nbTests >= testNb)
{ DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2;
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
/* create sample buffer (to catch read error with valgrind & sanitizers) */
sampleBuffer = (BYTE*)malloc(sampleSize);
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression test */
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
}
/* successfull decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
free(cBufferTooSmall);
}
/* too small dst decompression test */
if (sampleSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9;
dstBuffer[tooSmallSize] = token;
errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
/* noisy src decompression test */
if (cSize > 6)
{
const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
size_t pos = 4; /* preserve magic number (too easy to detect) */
U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
size_t mask = (1<<nbBits) - 1;
size_t skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
while (pos < cSize)
{
/* add noise */
size_t noiseStart, noiseLength;
nbBits = FUZ_rand(&lseed) % maxNbBits;
if (nbBits>0) nbBits--;
mask = (1<<nbBits) - 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
/* keep some original src */
nbBits = FUZ_rand(&lseed) % maxNbBits;
mask = (1<<nbBits) - 1;
skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
}
/* decompress noisy source */
{
U32 noiseSrc = FUZ_rand(&lseed) % 5;
const U32 endMark = 0xA9B1C3D6;
U32 endCheck;
srcBuffer = cNoiseBuffer[noiseSrc];
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
}
}
/* Streaming compression of scattered segments test */
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */
break;
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
(void)startTest; (void)compressibility;
/* allocation */
srcBuffer = malloc (srcBufferSize);
dstBuffer = malloc (dstBufferSize);
cBuffer = malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */
FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed);
/* catch up testNb */
for (testNb=0; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for (testNb=startTest; testNb < nbTests; testNb++)
{
size_t sampleSize, sampleStart;
size_t cSize, dSize, dSupSize;
U32 sampleSizeLog;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1<<sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
/* compression tests*/
cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
/* decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize));
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
}
DISPLAY("\rAll fuzzer tests completed \n");
_cleanup:
free(srcBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** The format of compressed block - see CompressedStream.h
*/
switch (method)
{
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress_default(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size));
else
compressed_size = header_size + LZ4_compress_HC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size),
0);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */
void* const buffOut = malloc_orDie(buffOutSize);
ZSTD_seekable_CStream* const cstream = ZSTD_seekable_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_seekable_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, frameSize);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t read, toRead = buffInSize;
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_seekable_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_seekable_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
fwrite_orDie(buffOut, output.pos, fout);
}
}
while (1) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remainingToFlush = ZSTD_seekable_endStream(cstream, &output); /* close stream */
if (ZSTD_isError(remainingToFlush)) { fprintf(stderr, "ZSTD_seekable_endStream() error : %s \n", ZSTD_getErrorName(remainingToFlush)); exit(13); }
fwrite_orDie(buffOut, output.pos, fout);
if (!remainingToFlush) break;
}
ZSTD_seekable_freeCStream(cstream);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
