ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
{
ZSTDMT_CCtx* cctx;
U32 const minNbJobs = nbThreads + 2;
U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
U32 const nbJobs = 1 << nbJobsLog2;
DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n",
nbThreads, minNbJobs, nbJobsLog2, nbJobs);
if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
if (!cctx) return NULL;
cctx->nbThreads = nbThreads;
cctx->jobIDMask = nbJobs - 1;
cctx->allJobsCompleted = 1;
cctx->sectionSize = 0;
cctx->overlapRLog = 3;
cctx->factory = POOL_create(nbThreads, 1);
cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) { /* one object was not created */
ZSTDMT_freeCCtx(cctx);
return NULL;
}
if (nbThreads==1) {
cctx->cstream = ZSTD_createCStream();
if (!cctx->cstream) {
ZSTDMT_freeCCtx(cctx); return NULL;
} }
pthread_mutex_init(&cctx->jobCompleted_mutex, NULL); /* Todo : check init function return */
pthread_cond_init(&cctx->jobCompleted_cond, NULL);
DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
return cctx;
}
static size_t Init(ZSTD_CStream **stream,
const BZstdCompressionParameters* parameters)
{
int32 compressionLevel = B_ZSTD_COMPRESSION_DEFAULT;
if (parameters != NULL) {
compressionLevel = parameters->CompressionLevel();
}
*stream = ZSTD_createCStream();
return ZSTD_initCStream(*stream, compressionLevel);
}
std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) {
// Support earlier versions of the codec (working with a single IOBuf,
// and using ZSTD_decompress which requires ZSTD frame to contain size,
// which isn't populated by streaming API).
if (!data->isChained()) {
auto out = IOBuf::createCombined(ZSTD_compressBound(data->length()));
const auto rc = ZSTD_compress(
out->writableData(),
out->capacity(),
data->data(),
data->length(),
level_);
zstdThrowIfError(rc);
out->append(rc);
return out;
}
auto zcs = ZSTD_createCStream();
SCOPE_EXIT {
ZSTD_freeCStream(zcs);
};
auto rc = ZSTD_initCStream(zcs, level_);
zstdThrowIfError(rc);
Cursor cursor(data);
auto result = IOBuf::createCombined(ZSTD_compressBound(cursor.totalLength()));
ZSTD_outBuffer out;
out.dst = result->writableTail();
out.size = result->capacity();
out.pos = 0;
for (auto buffer = cursor.peekBytes(); !buffer.empty();) {
ZSTD_inBuffer in;
in.src = buffer.data();
in.size = buffer.size();
for (in.pos = 0; in.pos != in.size;) {
rc = ZSTD_compressStream(zcs, &out, &in);
zstdThrowIfError(rc);
}
cursor.skip(in.size);
buffer = cursor.peekBytes();
}
rc = ZSTD_endStream(zcs, &out);
zstdThrowIfError(rc);
CHECK_EQ(rc, 0);
result->append(out.pos);
return result;
}
static void write_zstd(int f, int fd, const char *arg)
{
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
size_t const inbufsz = ZSTD_CStreamInSize();
zin.src = malloc(inbufsz);
zout.size = ZSTD_CStreamOutSize();
zout.dst = malloc(zout.size);
if (!zin.src || !zout.dst)
goto zstd_w_no_stream;
ZSTD_CStream* const stream = ZSTD_createCStream();
if (!stream)
goto zstd_w_no_stream;
if (ZSTD_isError(ZSTD_initCStream(stream, 3)))
goto zstd_w_error;
size_t s;
while ((s = read(fd, (void*)zin.src, inbufsz)) > 0)
{
zin.size = s;
zin.pos = 0;
while (zin.pos < zin.size)
{
zout.pos = 0;
size_t w = ZSTD_compressStream(stream, &zout, &zin);
if (ZSTD_isError(w))
goto zstd_w_error;
if (write(f, zout.dst, zout.pos) != (ssize_t)zout.pos)
goto zstd_w_error;
}
}
zout.pos = 0;
ZSTD_endStream(stream, &zout);
// no way to handle an error here
write(f, zout.dst, zout.pos);
zstd_w_error:
ZSTD_freeCStream(stream);
zstd_w_no_stream:
free((void*)zin.src);
free(zout.dst);
close(f);
close(fd);
}
ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
{
ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream));
if (zcs == NULL) return NULL;
memset(zcs, 0, sizeof(*zcs));
zcs->cstream = ZSTD_createCStream();
if (zcs->cstream == NULL) goto failed1;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(&zcs->framelog))) goto failed2;
return zcs;
failed2:
ZSTD_freeCStream(zcs->cstream);
failed1:
free(zcs);
return NULL;
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */
void* const buffOut = malloc_orDie(buffOutSize);
ZSTD_CStream* const cstream = ZSTD_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t read, toRead = buffInSize;
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */
fwrite_orDie(buffOut, output.pos, fout);
}
}
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remainingToFlush = ZSTD_endStream(cstream, &output); /* close frame */
if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
fwrite_orDie(buffOut, output.pos, fout);
ZSTD_freeCStream(cstream);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
ByteArray CompressFile(const FilePath& path, const int32 compressionLevel)
{
BinaryReader reader(path);
if (!reader)
{
return ByteArray();
}
const size_t inputBufferSize = ZSTD_CStreamInSize();
const auto pInputBuffer = std::make_unique<Byte[]>(inputBufferSize);
const size_t outputBufferSize = ZSTD_CStreamOutSize();
const auto pOutputBuffer = std::make_unique<Byte[]>(outputBufferSize);
ZSTD_CStream* const cStream = ZSTD_createCStream();
if (!cStream)
{
return ByteArray();
}
const size_t initResult = ZSTD_initCStream(cStream, compressionLevel);
if (ZSTD_isError(initResult))
{
ZSTD_freeCStream(cStream);
return ByteArray();
}
size_t toRead = inputBufferSize;
Array<Byte> buffer;
while (const size_t read = static_cast<size_t>(reader.read(pInputBuffer.get(), toRead)))
{
ZSTD_inBuffer input = { pInputBuffer.get(), read, 0 };
while (input.pos < input.size)
{
ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 };
toRead = ZSTD_compressStream(cStream, &output, &input);
if (ZSTD_isError(toRead))
{
ZSTD_freeCStream(cStream);
return ByteArray();
}
if (toRead > inputBufferSize)
{
toRead = inputBufferSize;
}
buffer.insert(buffer.end(), pOutputBuffer.get(), pOutputBuffer.get() + output.pos);
}
}
ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 };
const size_t remainingToFlush = ZSTD_endStream(cStream, &output);
ZSTD_freeCStream(cStream);
if (remainingToFlush)
{
return ByteArray();
}
buffer.insert(buffer.end(), pOutputBuffer.get(), pOutputBuffer.get() + output.pos);
return ByteArray(std::move(buffer));
}
bool CompressFileToFile(const FilePath& inputPath, const FilePath& outputPath, const int32 compressionLevel)
{
BinaryReader reader(inputPath);
if (!reader)
{
return false;
}
const size_t inputBufferSize = ZSTD_CStreamInSize();
const auto pInputBuffer = std::make_unique<Byte[]>(inputBufferSize);
const size_t outputBufferSize = ZSTD_CStreamOutSize();
const auto pOutputBuffer = std::make_unique<Byte[]>(outputBufferSize);
ZSTD_CStream* const cStream = ZSTD_createCStream();
if (!cStream)
{
return false;
}
const size_t initResult = ZSTD_initCStream(cStream, compressionLevel);
if (ZSTD_isError(initResult))
{
ZSTD_freeCStream(cStream);
return false;
}
size_t toRead = inputBufferSize;
BinaryWriter writer(outputPath);
if (!writer)
{
ZSTD_freeCStream(cStream);
return false;
}
while (const size_t read = static_cast<size_t>(reader.read(pInputBuffer.get(), toRead)))
{
ZSTD_inBuffer input = { pInputBuffer.get(), read, 0 };
while (input.pos < input.size)
{
ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 };
toRead = ZSTD_compressStream(cStream, &output, &input);
if (ZSTD_isError(toRead))
{
writer.clear();
ZSTD_freeCStream(cStream);
return false;
}
if (toRead > inputBufferSize)
{
toRead = inputBufferSize;
}
writer.write(pOutputBuffer.get(), output.pos);
}
}
ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 };
const size_t remainingToFlush = ZSTD_endStream(cStream, &output);
ZSTD_freeCStream(cStream);
if (remainingToFlush)
{
writer.clear();
return false;
}
writer.write(pOutputBuffer.get(), output.pos);
return true;
}
ZBUFF_CCtx* ZBUFF_createCCtx(void)
{
return ZSTD_createCStream();
}
