int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
seed = FUZZ_seed(&src, &size);
/* Allocate all buffers and contexts if not already allocated */
if (!buf) {
buf = malloc(kBufSize);
FUZZ_ASSERT(buf);
}
if (!dstream) {
dstream = ZSTD_createDStream();
FUZZ_ASSERT(dstream);
FUZZ_ASSERT(!ZSTD_isError(ZSTD_initDStream(dstream)));
} else {
FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream)));
}
while (size > 0) {
ZSTD_inBuffer in = makeInBuffer(&src, &size);
while (in.pos != in.size) {
ZSTD_outBuffer out = makeOutBuffer();
size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
if (ZSTD_isError(rc)) goto error;
if (rc == 0) FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream)));
}
}
error:
#ifndef STATEFUL_FUZZING
ZSTD_freeDStream(dstream); dstream = NULL;
#endif
return 0;
}
ByteArray DecompressFile(const FilePath& path)
{
BinaryReader reader(path);
if (!reader)
{
return ByteArray();
}
const size_t inputBufferSize = ZSTD_DStreamInSize();
const auto pInputBuffer = std::make_unique<Byte[]>(inputBufferSize);
const size_t outputBufferSize = ZSTD_DStreamOutSize();
const auto pOutputBuffer = std::make_unique<Byte[]>(outputBufferSize);
ZSTD_DStream* const dStream = ZSTD_createDStream();
if (!dStream)
{
return ByteArray();
}
const size_t initResult = ZSTD_initDStream(dStream);
if (ZSTD_isError(initResult))
{
ZSTD_freeDStream(dStream);
return ByteArray();
}
size_t toRead = initResult;
Array<Byte> buffer;
while (const size_t read = static_cast<size_t>(reader.read(pInputBuffer.get(), toRead)))
{
ZSTD_inBuffer input = { pInputBuffer.get(), read, 0 };
while (input.pos < input.size)
{
ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 };
toRead = ZSTD_decompressStream(dStream, &output, &input);
if (ZSTD_isError(toRead))
{
ZSTD_freeDStream(dStream);
return ByteArray();
}
buffer.insert(buffer.end(), pOutputBuffer.get(), pOutputBuffer.get() + output.pos);
}
}
ZSTD_freeDStream(dStream);
return ByteArray(std::move(buffer));
}
int testStreamingAPI(void)
{
size_t const outBuffSize = ZSTD_DStreamOutSize();
char* const outBuff = malloc(outBuffSize);
ZSTD_DStream* const stream = ZSTD_createDStream();
ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 };
size_t outputPos = 0;
int needsInit = 1;
if (outBuff == NULL) {
DISPLAY("ERROR: Could not allocate memory\n");
return 1;
}
if (stream == NULL) {
DISPLAY("ERROR: Could not create dstream\n");
return 1;
}
while (1) {
ZSTD_outBuffer output = {outBuff, outBuffSize, 0};
if (needsInit) {
size_t const ret = ZSTD_initDStream(stream);
if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
return 1;
}
}
{
size_t const ret = ZSTD_decompressStream(stream, &output, &input);
if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
return 1;
}
if (ret == 0) {
needsInit = 1;
}
}
if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
return 1;
}
outputPos += output.pos;
if (input.pos == input.size && output.pos < output.size) {
break;
}
}
free(outBuff);
ZSTD_freeDStream(stream);
DISPLAY("Streaming API OK\n");
return 0;
}
static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#", &source, &source_size))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#", &source, &source_size))
return NULL;
#endif
header_size = sizeof(dest_size);
memcpy(&dest_size, source, header_size);
result = PyBytes_FromStringAndSize(NULL, dest_size);
source += header_size;
if (result != NULL && dest_size > 0) {
char *dest = PyBytes_AS_STRING(result);
cSize = ZSTD_decompress(dest, dest_size, source, source_size - header_size);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Decompression error: %s", ZSTD_getErrorName(cSize));
}
return result;
}
int testSimpleAPI(void)
{
size_t const size = strlen(EXPECTED);
char* const output = malloc(size);
if (!output) {
DISPLAY("ERROR: Not enough memory!\n");
return 1;
}
{
size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE);
if (ZSTD_isError(ret)) {
if (ret == ZSTD_error_prefix_unknown) {
DISPLAY("ERROR: Invalid frame magic number, was this compiled "
"without legacy support?\n");
} else {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
}
return 1;
}
if (ret != size) {
DISPLAY("ERROR: Wrong decoded size\n");
}
}
if (memcmp(EXPECTED, output, size) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
return 1;
}
free(output);
DISPLAY("Simple API OK\n");
return 0;
}
void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
size_t res = ZSTD_decompress(dest, uncompressed_size, source, source_size);
if (ZSTD_isError(res))
throw Exception("Cannot ZSTD_decompress: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_DECOMPRESS);
}
void zstdThrowIfError(size_t rc) {
if (!ZSTD_isError(rc)) {
return;
}
throw std::runtime_error(
to<std::string>("ZSTD returned an error: ", ZSTD_getErrorName(rc)));
}
/* *** Initialization *** */
#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
{
size_t neededInBuffSize;
ZSTD_validateParams(¶ms);
neededInBuffSize = (size_t)1 << params.windowLog;
/* allocate buffers */
if (zbc->inBuffSize < neededInBuffSize)
{
zbc->inBuffSize = neededInBuffSize;
free(zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)malloc(neededInBuffSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1)
{
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
free(zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)malloc(zbc->outBuffSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, params);
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
zbc->inToCompress = 0;
zbc->inBuffPos = 0;
zbc->inBuffTarget = zbc->blockSize;
zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
return 0; /* ready to go */
}
static void write_zstd(int f, int fd, const char *arg)
{
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
size_t const inbufsz = ZSTD_CStreamInSize();
zin.src = malloc(inbufsz);
zout.size = ZSTD_CStreamOutSize();
zout.dst = malloc(zout.size);
if (!zin.src || !zout.dst)
goto zstd_w_no_stream;
ZSTD_CStream* const stream = ZSTD_createCStream();
if (!stream)
goto zstd_w_no_stream;
if (ZSTD_isError(ZSTD_initCStream(stream, 3)))
goto zstd_w_error;
size_t s;
while ((s = read(fd, (void*)zin.src, inbufsz)) > 0)
{
zin.size = s;
zin.pos = 0;
while (zin.pos < zin.size)
{
zout.pos = 0;
size_t w = ZSTD_compressStream(stream, &zout, &zin);
if (ZSTD_isError(w))
goto zstd_w_error;
if (write(f, zout.dst, zout.pos) != (ssize_t)zout.pos)
goto zstd_w_error;
}
}
zout.pos = 0;
ZSTD_endStream(stream, &zout);
// no way to handle an error here
write(f, zout.dst, zout.pos);
zstd_w_error:
ZSTD_freeCStream(stream);
zstd_w_no_stream:
free((void*)zin.src);
free(zout.dst);
close(f);
close(fd);
}
/* ZSTDMT_flushNextJob() :
* output : will be updated with amount of data flushed .
* blockToFlush : if >0, the function will block and wait if there is no data available to flush .
* @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */
static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
{
unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */
PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
while (zcs->jobs[wJobID].jobCompleted==0) {
DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */
pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */
}
pthread_mutex_unlock(&zcs->jobCompleted_mutex);
/* compression job completed : output can be flushed */
{ ZSTDMT_jobDescription job = zcs->jobs[wJobID];
if (!job.jobScanned) {
if (ZSTD_isError(job.cSize)) {
DEBUGLOG(5, "compression error detected ");
ZSTDMT_waitForAllJobsCompleted(zcs);
ZSTDMT_releaseAllJobResources(zcs);
return job.cSize;
}
ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx);
zcs->jobs[wJobID].cctx = NULL;
DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
if (zcs->params.fParams.checksumFlag) {
XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */
U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
DEBUGLOG(4, "writing checksum : %08X \n", checksum);
MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
job.cSize += 4;
zcs->jobs[wJobID].cSize += 4;
} }
ZSTDMT_releaseBuffer(zcs->buffPool, job.src);
zcs->jobs[wJobID].srcStart = NULL;
zcs->jobs[wJobID].src = g_nullBuffer;
zcs->jobs[wJobID].jobScanned = 1;
}
{ size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
output->pos += toWrite;
job.dstFlushed += toWrite;
}
if (job.dstFlushed == job.cSize) { /* output buffer fully flushed => move to next one */
ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff);
zcs->jobs[wJobID].dstBuff = g_nullBuffer;
zcs->jobs[wJobID].jobCompleted = 0;
zcs->doneJobID++;
} else {
zcs->jobs[wJobID].dstFlushed = job.dstFlushed;
}
/* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */
if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed);
if (zcs->doneJobID < zcs->nextJobID) return 1; /* still some buffer to flush */
zcs->allJobsCompleted = zcs->frameEnded; /* frame completed and entirely flushed */
return 0; /* everything flushed */
} }
UInt32 CompressionCodecZSTD::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
size_t compressed_size = ZSTD_compress(dest, ZSTD_compressBound(source_size), source, source_size, level);
if (ZSTD_isError(compressed_size))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(compressed_size)), ErrorCodes::CANNOT_COMPRESS);
return compressed_size;
}
static int decompress_zstd(const char *inbuf, char *outbuf, u64 compress_len,
u64 decompress_len)
{
#if !BTRFSRESTORE_ZSTD
error("btrfs not compiled with zstd support");
return -1;
#else
ZSTD_DStream *strm;
size_t zret;
int ret = 0;
ZSTD_inBuffer in = {inbuf, compress_len, 0};
ZSTD_outBuffer out = {outbuf, decompress_len, 0};
strm = ZSTD_createDStream();
if (!strm) {
error("zstd create failed");
return -1;
}
zret = ZSTD_initDStream(strm);
if (ZSTD_isError(zret)) {
error("zstd init failed: %s", ZSTD_getErrorName(zret));
ret = -1;
goto out;
}
zret = ZSTD_decompressStream(strm, &out, &in);
if (ZSTD_isError(zret)) {
error("zstd decompress failed %s\n", ZSTD_getErrorName(zret));
ret = -1;
goto out;
}
if (zret != 0) {
error("zstd frame incomplete");
ret = -1;
goto out;
}
out:
ZSTD_freeDStream(strm);
return ret;
#endif
}
static PyObject* ZstdCompressionWriter_exit(ZstdCompressionWriter* self, PyObject* args) {
PyObject* exc_type;
PyObject* exc_value;
PyObject* exc_tb;
size_t zresult;
ZSTD_outBuffer output;
PyObject* res;
if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
return NULL;
}
self->entered = 0;
if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) {
ZSTD_inBuffer inBuffer;
inBuffer.src = NULL;
inBuffer.size = 0;
inBuffer.pos = 0;
output.dst = PyMem_Malloc(self->outSize);
if (!output.dst) {
return PyErr_NoMemory();
}
output.size = self->outSize;
output.pos = 0;
while (1) {
zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &inBuffer, ZSTD_e_end);
if (ZSTD_isError(zresult)) {
PyErr_Format(ZstdError, "error ending compression stream: %s",
ZSTD_getErrorName(zresult));
PyMem_Free(output.dst);
return NULL;
}
if (output.pos) {
#if PY_MAJOR_VERSION >= 3
res = PyObject_CallMethod(self->writer, "write", "y#",
#else
res = PyObject_CallMethod(self->writer, "write", "s#",
#endif
output.dst, output.pos);
Py_XDECREF(res);
}
if (!zresult) {
break;
}
output.pos = 0;
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */
void* const buffOut = malloc_orDie(buffOutSize);
ZSTD_seekable_CStream* const cstream = ZSTD_seekable_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_seekable_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, frameSize);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t read, toRead = buffInSize;
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_seekable_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_seekable_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
fwrite_orDie(buffOut, output.pos, fout);
}
}
while (1) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remainingToFlush = ZSTD_seekable_endStream(cstream, &output); /* close stream */
if (ZSTD_isError(remainingToFlush)) { fprintf(stderr, "ZSTD_seekable_endStream() error : %s \n", ZSTD_getErrorName(remainingToFlush)); exit(13); }
fwrite_orDie(buffOut, output.pos, fout);
if (!remainingToFlush) break;
}
ZSTD_seekable_freeCStream(cstream);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
/* ZSTDMT_compressChunk() : POOL_function type */
void ZSTDMT_compressChunk(void* jobDescription)
{
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t const dstBuff = job->dstBuff;
DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
if (job->cdict) { /* should only happen for first segment */
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
if (job->cdict) DEBUGLOG(3, "using CDict ");
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
{ size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */
size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
} }
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
ZSTD_invalidateRepCodes(job->cctx);
}
DEBUGLOG(4, "Compressing : ");
DEBUG_PRINTHEX(4, job->srcStart, 12);
job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
(unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
_endJob:
PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
job->jobCompleted = 1;
job->jobScanned = 0;
pthread_cond_signal(job->jobCompleted_cond);
pthread_mutex_unlock(job->jobCompleted_mutex);
}
size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
{
if (!zcs->writingSeekTable && zcs->frameDSize) {
const size_t endFrame = ZSTD_seekable_endFrame(zcs, output);
if (ZSTD_isError(endFrame)) return endFrame;
/* return an accurate size hint */
if (endFrame) return endFrame + ZSTD_seekable_seekTableSize(&zcs->framelog);
}
zcs->writingSeekTable = 1;
return ZSTD_seekable_writeSeekTable(&zcs->framelog, output);
}
static int zstd_uncompress(void *dest, void *src, int size, int outsize,
int *error)
{
const size_t res = ZSTD_decompress(dest, outsize, src, size);
if (ZSTD_isError(res)) {
fprintf(stderr, "\t%d %d\n", outsize, size);
*error = (int)ZSTD_getErrorCode(res);
return -1;
}
return (int)res;
}
size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
const BYTE* const inBase = (const BYTE*) input->src + input->pos;
size_t inLen = input->size - input->pos;
inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize));
/* if we haven't finished flushing the last frame, don't start writing a new one */
if (inLen > 0) {
ZSTD_inBuffer inTmp = { inBase, inLen, 0 };
size_t const prevOutPos = output->pos;
size_t const ret = ZSTD_compressStream(zcs->cstream, output, &inTmp);
if (zcs->framelog.checksumFlag) {
XXH64_update(&zcs->xxhState, inBase, inTmp.pos);
}
zcs->frameCSize += output->pos - prevOutPos;
zcs->frameDSize += inTmp.pos;
input->pos += inTmp.pos;
if (ZSTD_isError(ret)) return ret;
}
if (zcs->maxFrameSize == zcs->frameDSize) {
/* log the frame and start over */
size_t const ret = ZSTD_seekable_endFrame(zcs, output);
if (ZSTD_isError(ret)) return ret;
/* get the client ready for the next frame */
return (size_t)zcs->maxFrameSize;
}
return (size_t)(zcs->maxFrameSize - zcs->frameDSize);
}
ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
{
ZSTD_frameLog* fl = malloc(sizeof(ZSTD_frameLog));
if (fl == NULL) return NULL;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
free(fl);
return NULL;
}
fl->checksumFlag = checksumFlag;
fl->seekTablePos = 0;
fl->seekTableIndex = 0;
fl->size = 0;
return fl;
}
static PyObject *py_zstd_compress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
char *dest;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
uint32_t level = 0;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#|i", &source, &source_size, &level))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#|i", &source, &source_size, &level))
return NULL;
#endif
if (level <= 0) level=1;
if (level > 20) level=20;
header_size = sizeof(source_size);
dest_size = ZSTD_compressBound(source_size);
result = PyBytes_FromStringAndSize(NULL, header_size + dest_size);
if (result == NULL) {
return NULL;
}
dest = PyBytes_AS_STRING(result);
memcpy(dest, &source_size, header_size);
dest += header_size;
if (source_size > 0) {
// Low level == old version
cSize = ZSTD_compress(dest, dest_size, source, source_size, level);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Compression error: %s", ZSTD_getErrorName(cSize));
Py_SIZE(result) = cSize + sizeof(source_size);
}
return result;
}
ByteArray Compress(const ByteArrayView view, const int32 compressionLevel)
{
const size_t bufferSize = ZSTD_compressBound(view.size());
Array<Byte> buffer(bufferSize);
const size_t result = ZSTD_compress(buffer.data(), buffer.size(), view.data(), view.size(), compressionLevel);
if (ZSTD_isError(result))
{
return ByteArray();
}
buffer.resize(result);
buffer.shrink_to_fit();
return ByteArray(std::move(buffer));
}
static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) {
size_t zresult;
if (self->entered) {
PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
return NULL;
}
zresult = ZSTD_CCtx_setPledgedSrcSize(self->compressor->cctx, self->sourceSize);
if (ZSTD_isError(zresult)) {
PyErr_Format(ZstdError, "error setting source size: %s",
ZSTD_getErrorName(zresult));
return NULL;
}
self->entered = 1;
Py_INCREF(self);
return (PyObject*)self;
}
static int zstd_compress(void *strm, void *dest, void *src, int size,
int block_size, int *error)
{
const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
src, size, compression_level);
if (ZSTD_isError(res)) {
/* FIXME:
* zstd does not expose stable error codes. The error enum may
* change between versions. Until upstream zstd stablizes the
* error codes, we have no way of knowing why the error occurs.
* zstd shouldn't fail to compress any input unless there isn't
* enough output space. We assume that is the cause and return
* the special error code for not enough output space.
*/
return 0;
}
return (int)res;
}
ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
{
ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream));
if (zcs == NULL) return NULL;
memset(zcs, 0, sizeof(*zcs));
zcs->cstream = ZSTD_createCStream();
if (zcs->cstream == NULL) goto failed1;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(&zcs->framelog))) goto failed2;
return zcs;
failed2:
ZSTD_freeCStream(zcs->cstream);
failed1:
free(zcs);
return NULL;
}
static ezResult CompressZStd(ezArrayPtr<const ezUInt8> pUncompressedData, ezDynamicArray<ezUInt8>& out_Data)
{
size_t uiSizeBound = ZSTD_compressBound(pUncompressedData.GetCount());
if(uiSizeBound > std::numeric_limits<ezUInt32>::max())
{
ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSizeBound));
return EZ_FAILURE;
}
out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSizeBound));
size_t const cSize = ZSTD_compress(out_Data.GetData(), uiSizeBound, pUncompressedData.GetPtr(), pUncompressedData.GetCount(), 1);
if (ZSTD_isError(cSize))
{
ezLog::Error("Compression failed with error: '{0}'.", ZSTD_getErrorName(cSize));
return EZ_FAILURE;
}
out_Data.SetCount(static_cast<ezUInt32>(cSize));
return EZ_SUCCESS;
}
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(const IOBuf* data,
uint64_t uncompressedLength) {
size_t rc;
auto out = IOBuf::createCombined(uncompressedLength);
CHECK_GE(out->capacity(), uncompressedLength);
CHECK_EQ(out->length(), 0);
rc = ZSTD_decompress(
out->writableTail(), out->capacity(), data->data(), data->length());
if (ZSTD_isError(rc)) {
throw std::runtime_error(to<std::string>(
"ZSTD decompression returned an error: ",
ZSTD_getErrorName(rc)));
}
out->append(rc);
CHECK_EQ(out->length(), rc);
return out;
}
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
{
size_t fSize;
void* const fBuff = loadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(7);
}
saveFile_orDie(oname, cBuff, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
ZSTD_freeCCtx(cctx);
free(fBuff);
free(cBuff);
}
static ssize_t
zstd_filter_read(struct archive_read_filter *self, const void **p)
{
struct private_data *state;
size_t decompressed;
ssize_t avail_in;
ZSTD_outBuffer out;
ZSTD_inBuffer in;
state = (struct private_data *)self->data;
out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 };
/* Try to fill the output buffer. */
while (out.pos < out.size && !state->eof) {
if (!state->in_frame) {
const size_t ret = ZSTD_initDStream(state->dstream);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Error initializing zstd decompressor: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
}
in.src = __archive_read_filter_ahead(self->upstream, 1,
&avail_in);
if (avail_in < 0) {
return avail_in;
}
if (in.src == NULL && avail_in == 0) {
if (!state->in_frame) {
/* end of stream */
state->eof = 1;
break;
} else {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Truncated zstd input");
return (ARCHIVE_FATAL);
}
}
in.size = avail_in;
in.pos = 0;
{
const size_t ret =
ZSTD_decompressStream(state->dstream, &out, &in);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Zstd decompression failed: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
/* Decompressor made some progress */
__archive_read_filter_consume(self->upstream, in.pos);
/* ret guaranteed to be > 0 if frame isn't done yet */
state->in_frame = (ret != 0);
}
}
decompressed = out.pos;
state->total_out += decompressed;
if (decompressed == 0)
*p = NULL;
else
*p = state->out_block;
return (decompressed);
}
/*
* Finish off an encoded strip by flushing it.
*/
static int
LERCPostEncode(TIFF* tif)
{
lerc_status lerc_ret;
static const char module[] = "LERCPostEncode";
LERCState *sp = EncoderState(tif);
unsigned int numBytes = 0;
unsigned int numBytesWritten = 0;
TIFFDirectory *td = &tif->tif_dir;
int use_mask = 0;
unsigned dst_nbands = td->td_samplesperpixel;
if( sp->uncompressed_offset != sp->uncompressed_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected number of bytes in the buffer");
return 0;
}
/* Extract alpha mask (if containing only 0 and 255 values, */
/* and compact array of regular bands */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 )
{
unsigned dst_stride = (td->td_samplesperpixel - 1) *
(td->td_bitspersample / 8);
unsigned src_stride = td->td_samplesperpixel *
(td->td_bitspersample / 8);
unsigned i = 0;
unsigned nb_pixels = sp->segment_width * sp->segment_height;
use_mask = 1;
for( i = 0 ; i < nb_pixels; i++)
{
int v = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
if( v != 0 && v != 255 )
{
use_mask = 0;
break;
}
}
if( use_mask )
{
dst_nbands --;
/* First pixels must use memmove due to overlapping areas */
for( i = 0 ;i < dst_nbands && i < nb_pixels; i++)
{
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
for(; i < nb_pixels; i++)
{
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
}
}
#if 0
lerc_ret = lerc_computeCompressedSize(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
&numBytes);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_computeCompressedSize() failed");
return 0;
}
#else
numBytes = sp->uncompressed_alloc;
#endif
if( sp->compressed_size < numBytes )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(numBytes);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = numBytes;
}
lerc_ret = lerc_encodeForVersion(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
sp->compressed_buffer,
sp->compressed_size,
&numBytesWritten);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_encode() failed");
return 0;
}
assert( numBytesWritten < numBytes );
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = deflateInit(&strm, sp->zipquality);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflateInit() failed");
return 0;
}
strm.avail_in = numBytesWritten;
strm.next_in = sp->compressed_buffer;
strm.avail_out = sp->uncompressed_alloc;
strm.next_out = sp->uncompressed_buffer;
zlib_ret = deflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflate() failed");
deflateEnd(&strm);
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = sp->uncompressed_alloc - strm.avail_out;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
deflateEnd(&strm);
return 0;
}
}
deflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret = ZSTD_compress( sp->uncompressed_buffer,
sp->uncompressed_alloc,
sp->compressed_buffer,
numBytesWritten,
sp->zstd_compress_level );
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_compress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = zstd_ret;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
return 0;
}
}
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
else
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->compressed_buffer;
tif->tif_rawcc = numBytesWritten;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
return 0;
}
return 1;
}
/*
* Setup state for decoding a strip.
*/
static int
LERCPreDecode(TIFF* tif, uint16 s)
{
static const char module[] = "LERCPreDecode";
lerc_status lerc_ret;
TIFFDirectory *td = &tif->tif_dir;
LERCState* sp = DecoderState(tif);
int lerc_data_type;
unsigned int infoArray[8];
unsigned nomask_bands = td->td_samplesperpixel;
int ndims;
int use_mask = 0;
uint8* lerc_data = tif->tif_rawcp;
unsigned int lerc_data_size = (unsigned int)tif->tif_rawcc;
(void) s;
assert(sp != NULL);
lerc_data_type = GetLercDataType(tif);
if( lerc_data_type < 0 )
return 0;
if( !SetupUncompressedBuffer(tif, sp, module) )
return 0;
if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
if( sp->compressed_size < sp->uncompressed_alloc )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(sp->uncompressed_alloc);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = sp->uncompressed_alloc;
}
}
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = inflateInit(&strm);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflateInit() failed");
inflateEnd(&strm);
return 0;
}
strm.avail_in = (uInt)tif->tif_rawcc;
strm.next_in = tif->tif_rawcp;
strm.avail_out = sp->compressed_size;
strm.next_out = sp->compressed_buffer;
zlib_ret = inflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END && zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflate() failed");
inflateEnd(&strm);
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = sp->compressed_size - strm.avail_out;
inflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret;
zstd_ret = ZSTD_decompress(sp->compressed_buffer,
sp->compressed_size,
tif->tif_rawcp,
tif->tif_rawcc);
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_decompress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = (unsigned int)zstd_ret;
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
lerc_ret = lerc_getBlobInfo(
lerc_data,
lerc_data_size,
infoArray,
NULL,
8,
0);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_getBlobInfo() failed");
return 0;
}
/* If the configuration is compatible of a LERC mask, and that the */
/* LERC info has dim == samplesperpixel - 1, then there is a LERC */
/* mask. */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 &&
infoArray[2] == td->td_samplesperpixel - 1U )
{
use_mask = 1;
nomask_bands --;
}
ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ?
nomask_bands : 1;
/* Info returned in infoArray is { version, dataType, nDim, nCols,
nRows, nBands, nValidPixels, blobSize } */
if( infoArray[0] != (unsigned)sp->lerc_version )
{
TIFFWarningExt(tif->tif_clientdata, module,
"Unexpected version number: %d. Expected: %d",
infoArray[0], sp->lerc_version);
}
if( infoArray[1] != (unsigned)lerc_data_type )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected dataType: %d. Expected: %d",
infoArray[1], lerc_data_type);
return 0;
}
if( infoArray[2] != (unsigned)ndims )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nDim: %d. Expected: %d",
infoArray[2], ndims);
return 0;
}
if( infoArray[3] != sp->segment_width )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nCols: %d. Expected: %du",
infoArray[3], sp->segment_width);
return 0;
}
if( infoArray[4] != sp->segment_height )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nRows: %d. Expected: %u",
infoArray[4], sp->segment_height);
return 0;
}
if( infoArray[5] != 1 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nBands: %d. Expected: %d",
infoArray[5], 1);
return 0;
}
if( infoArray[7] != lerc_data_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected blobSize: %d. Expected: %u",
infoArray[7],
lerc_data_size);
return 0;
}
lerc_ret = lerc_decode(
lerc_data,
lerc_data_size,
use_mask ? sp->mask_buffer : NULL,
ndims,
sp->segment_width,
sp->segment_height,
1,
lerc_data_type,
sp->uncompressed_buffer);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_decode() failed");
return 0;
}
/* Interleave alpha mask with other samples. */
if( use_mask )
{
unsigned src_stride =
(td->td_samplesperpixel - 1) * (td->td_bitspersample / 8);
unsigned dst_stride =
td->td_samplesperpixel * (td->td_bitspersample / 8);
unsigned i = sp->segment_width * sp->segment_height;
/* Operate from end to begin to be able to move in place */
while( i > 0 && i > nomask_bands )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
/* First pixels must use memmove due to overlapping areas */
while( i > 0 )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
}
return 1;
}
