size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) { size_t const prevOutPos = output->pos; /* end the frame */ size_t ret = ZSTD_endStream(zcs->cstream, output); zcs->frameCSize += output->pos - prevOutPos; /* need to flush before doing the rest */ if (ret) return ret; /* frame done */ /* store the frame data for later */ ret = ZSTD_seekable_logFrame( &zcs->framelog, zcs->frameCSize, zcs->frameDSize, zcs->framelog.checksumFlag ? XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU : 0); if (ret) return ret; /* reset for the next frame */ zcs->frameCSize = 0; zcs->frameDSize = 0; ZSTD_resetCStream(zcs->cstream, 0); if (zcs->framelog.checksumFlag) XXH64_reset(&zcs->xxhState, 0); return 0; }
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { size_t result; ZSTD_outBuffer outBuff; outBuff.dst = dst; outBuff.pos = 0; outBuff.size = *dstCapacityPtr; result = ZSTD_endStream(zbc, &outBuff); *dstCapacityPtr = outBuff.pos; return result; }
std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) { // Support earlier versions of the codec (working with a single IOBuf, // and using ZSTD_decompress which requires ZSTD frame to contain size, // which isn't populated by streaming API). if (!data->isChained()) { auto out = IOBuf::createCombined(ZSTD_compressBound(data->length())); const auto rc = ZSTD_compress( out->writableData(), out->capacity(), data->data(), data->length(), level_); zstdThrowIfError(rc); out->append(rc); return out; } auto zcs = ZSTD_createCStream(); SCOPE_EXIT { ZSTD_freeCStream(zcs); }; auto rc = ZSTD_initCStream(zcs, level_); zstdThrowIfError(rc); Cursor cursor(data); auto result = IOBuf::createCombined(ZSTD_compressBound(cursor.totalLength())); ZSTD_outBuffer out; out.dst = result->writableTail(); out.size = result->capacity(); out.pos = 0; for (auto buffer = cursor.peekBytes(); !buffer.empty();) { ZSTD_inBuffer in; in.src = buffer.data(); in.size = buffer.size(); for (in.pos = 0; in.pos != in.size;) { rc = ZSTD_compressStream(zcs, &out, &in); zstdThrowIfError(rc); } cursor.skip(in.size); buffer = cursor.peekBytes(); } rc = ZSTD_endStream(zcs, &out); zstdThrowIfError(rc); CHECK_EQ(rc, 0); result->append(out.pos); return result; }
static void write_zstd(int f, int fd, const char *arg) { ZSTD_inBuffer zin; ZSTD_outBuffer zout; size_t const inbufsz = ZSTD_CStreamInSize(); zin.src = malloc(inbufsz); zout.size = ZSTD_CStreamOutSize(); zout.dst = malloc(zout.size); if (!zin.src || !zout.dst) goto zstd_w_no_stream; ZSTD_CStream* const stream = ZSTD_createCStream(); if (!stream) goto zstd_w_no_stream; if (ZSTD_isError(ZSTD_initCStream(stream, 3))) goto zstd_w_error; size_t s; while ((s = read(fd, (void*)zin.src, inbufsz)) > 0) { zin.size = s; zin.pos = 0; while (zin.pos < zin.size) { zout.pos = 0; size_t w = ZSTD_compressStream(stream, &zout, &zin); if (ZSTD_isError(w)) goto zstd_w_error; if (write(f, zout.dst, zout.pos) != (ssize_t)zout.pos) goto zstd_w_error; } } zout.pos = 0; ZSTD_endStream(stream, &zout); // no way to handle an error here write(f, zout.dst, zout.pos); zstd_w_error: ZSTD_freeCStream(stream); zstd_w_no_stream: free((void*)zin.src); free(zout.dst); close(f); close(fd); }
static void compressFile_orDie(const char* fname, const char* outName, int cLevel) { FILE* const fin = fopen_orDie(fname, "rb"); FILE* const fout = fopen_orDie(outName, "wb"); size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ void* const buffIn = malloc_orDie(buffInSize); size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */ void* const buffOut = malloc_orDie(buffOutSize); ZSTD_CStream* const cstream = ZSTD_createCStream(); if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } size_t const initResult = ZSTD_initCStream(cstream, cLevel); if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } size_t read, toRead = buffInSize; while( (read = fread_orDie(buffIn, toRead, fin)) ) { ZSTD_inBuffer input = { buffIn, read, 0 }; while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ fwrite_orDie(buffOut, output.pos, fout); } } ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; size_t const remainingToFlush = ZSTD_endStream(cstream, &output); /* close frame */ if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); } fwrite_orDie(buffOut, output.pos, fout); ZSTD_freeCStream(cstream); fclose_orDie(fout); fclose_orDie(fin); free(buffIn); free(buffOut); }
ByteArray CompressFile(const FilePath& path, const int32 compressionLevel) { BinaryReader reader(path); if (!reader) { return ByteArray(); } const size_t inputBufferSize = ZSTD_CStreamInSize(); const auto pInputBuffer = std::make_unique<Byte[]>(inputBufferSize); const size_t outputBufferSize = ZSTD_CStreamOutSize(); const auto pOutputBuffer = std::make_unique<Byte[]>(outputBufferSize); ZSTD_CStream* const cStream = ZSTD_createCStream(); if (!cStream) { return ByteArray(); } const size_t initResult = ZSTD_initCStream(cStream, compressionLevel); if (ZSTD_isError(initResult)) { ZSTD_freeCStream(cStream); return ByteArray(); } size_t toRead = inputBufferSize; Array<Byte> buffer; while (const size_t read = static_cast<size_t>(reader.read(pInputBuffer.get(), toRead))) { ZSTD_inBuffer input = { pInputBuffer.get(), read, 0 }; while (input.pos < input.size) { ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 }; toRead = ZSTD_compressStream(cStream, &output, &input); if (ZSTD_isError(toRead)) { ZSTD_freeCStream(cStream); return ByteArray(); } if (toRead > inputBufferSize) { toRead = inputBufferSize; } buffer.insert(buffer.end(), pOutputBuffer.get(), pOutputBuffer.get() + output.pos); } } ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 }; const size_t remainingToFlush = ZSTD_endStream(cStream, &output); ZSTD_freeCStream(cStream); if (remainingToFlush) { return ByteArray(); } buffer.insert(buffer.end(), pOutputBuffer.get(), pOutputBuffer.get() + output.pos); return ByteArray(std::move(buffer)); }
bool CompressFileToFile(const FilePath& inputPath, const FilePath& outputPath, const int32 compressionLevel) { BinaryReader reader(inputPath); if (!reader) { return false; } const size_t inputBufferSize = ZSTD_CStreamInSize(); const auto pInputBuffer = std::make_unique<Byte[]>(inputBufferSize); const size_t outputBufferSize = ZSTD_CStreamOutSize(); const auto pOutputBuffer = std::make_unique<Byte[]>(outputBufferSize); ZSTD_CStream* const cStream = ZSTD_createCStream(); if (!cStream) { return false; } const size_t initResult = ZSTD_initCStream(cStream, compressionLevel); if (ZSTD_isError(initResult)) { ZSTD_freeCStream(cStream); return false; } size_t toRead = inputBufferSize; BinaryWriter writer(outputPath); if (!writer) { ZSTD_freeCStream(cStream); return false; } while (const size_t read = static_cast<size_t>(reader.read(pInputBuffer.get(), toRead))) { ZSTD_inBuffer input = { pInputBuffer.get(), read, 0 }; while (input.pos < input.size) { ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 }; toRead = ZSTD_compressStream(cStream, &output, &input); if (ZSTD_isError(toRead)) { writer.clear(); ZSTD_freeCStream(cStream); return false; } if (toRead > inputBufferSize) { toRead = inputBufferSize; } writer.write(pOutputBuffer.get(), output.pos); } } ZSTD_outBuffer output = { pOutputBuffer.get(), outputBufferSize, 0 }; const size_t remainingToFlush = ZSTD_endStream(cStream, &output); ZSTD_freeCStream(cStream); if (remainingToFlush) { writer.clear(); return false; } writer.write(pOutputBuffer.get(), output.pos); return true; }
size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output) { if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output); return ZSTDMT_flushStream_internal(zcs, output, 1); }