static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
fprintf(stderr, "%s : original size unknown \n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
if (dSize != rSize) {
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
ZSTD_freeDCtx(dctx);
free(rBuff);
free(cBuff);
}
ByteArray Decompress(const ByteArrayView view)
{
const uint64 originalSize = ZSTD_getDecompressedSize(view.data(), view.size());
if (originalSize == 0)
{
return ByteArray();
}
Array<Byte> outputBuffer(static_cast<size_t>(originalSize));
const size_t decompressedSize = ZSTD_decompress(outputBuffer.data(), outputBuffer.size(), view.data(), view.size());
if (originalSize != decompressedSize)
{
return ByteArray();
}
return ByteArray(std::move(outputBuffer));
}
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown \n", fname);
exit(5);
}
void* const rBuff = malloc_X(rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(
const IOBuf* data,
uint64_t uncompressedLength) {
auto zds = ZSTD_createDStream();
SCOPE_EXIT {
ZSTD_freeDStream(zds);
};
auto rc = ZSTD_initDStream(zds);
zstdThrowIfError(rc);
ZSTD_outBuffer out{};
ZSTD_inBuffer in{};
auto outputSize = ZSTD_DStreamOutSize();
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH) {
outputSize = uncompressedLength;
} else {
auto decompressedSize =
ZSTD_getDecompressedSize(data->data(), data->length());
if (decompressedSize != 0 && decompressedSize < outputSize) {
outputSize = decompressedSize;
}
}
IOBufQueue queue(IOBufQueue::cacheChainLength());
Cursor cursor(data);
for (rc = 0;;) {
if (in.pos == in.size) {
auto buffer = cursor.peekBytes();
in.src = buffer.data();
in.size = buffer.size();
in.pos = 0;
cursor.skip(in.size);
if (rc > 1 && in.size == 0) {
throw std::runtime_error(to<std::string>("ZSTD: incomplete input"));
}
}
if (out.pos == out.size) {
if (out.pos != 0) {
queue.postallocate(out.pos);
}
auto buffer = queue.preallocate(outputSize, outputSize);
out.dst = buffer.first;
out.size = buffer.second;
out.pos = 0;
outputSize = ZSTD_DStreamOutSize();
}
rc = ZSTD_decompressStream(zds, &out, &in);
zstdThrowIfError(rc);
if (rc == 0) {
break;
}
}
if (out.pos != 0) {
queue.postallocate(out.pos);
}
if (in.pos != in.size || !cursor.isAtEnd()) {
throw std::runtime_error("ZSTD: junk after end of data");
}
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
queue.chainLength() != uncompressedLength) {
throw std::runtime_error("ZSTD: invalid uncompressed length");
}
return queue.move();
}
static gint
rspamd_client_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_client_request *req =
(struct rspamd_client_request *)conn->ud;
struct rspamd_client_connection *c;
struct ucl_parser *parser;
GError *err;
const rspamd_ftok_t *tok;
c = req->conn;
if (!c->req_sent) {
c->req_sent = TRUE;
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
c->fd,
&c->timeout,
c->ev_base);
return 0;
}
else {
if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) {
err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
msg->code,
(gint)msg->status->len, msg->status->str);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
tok = rspamd_http_message_find_header (msg, "compression");
if (tok) {
/* Need to uncompress */
rspamd_ftok_t t;
t.begin = "zstd";
t.len = 4;
if (rspamd_ftok_casecmp (tok, &t) == 0) {
ZSTD_DStream *zstream;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
guchar *out;
gsize outlen, r;
zstream = ZSTD_createDStream ();
ZSTD_initDStream (zstream);
zin.pos = 0;
zin.src = msg->body_buf.begin;
zin.size = msg->body_buf.len;
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
outlen = ZSTD_DStreamOutSize ();
}
out = g_malloc (outlen);
zout.dst = out;
zout.pos = 0;
zout.size = outlen;
while (zin.pos < zin.size) {
r = ZSTD_decompressStream (zstream, &zout, &zin);
if (ZSTD_isError (r)) {
err = g_error_new (RCLIENT_ERROR, 500,
"Decompression error: %s",
ZSTD_getErrorName (r));
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
ZSTD_freeDStream (zstream);
g_free (out);
return 0;
}
if (zout.pos == zout.size) {
/* We need to extend output buffer */
zout.size = zout.size * 1.5 + 1.0;
zout.dst = g_realloc (zout.dst, zout.size);
}
}
ZSTD_freeDStream (zstream);
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, zout.dst, zout.pos)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
g_free (zout.dst);
return 0;
}
g_free (zout.dst);
}
else {
err = g_error_new (RCLIENT_ERROR, 500,
"Invalid compression method");
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
else {
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
req->cb (c, msg, c->server_name->str, ucl_parser_get_object (
parser), req->input, req->ud, NULL);
ucl_parser_free (parser);
}
return 0;
}
static int
pkg_section_maybe_uncompress(FILE *fs, struct pkg_section_hdr *hdr, void **uncompressed,
size_t *sz)
{
ZSTD_DStream *zstream;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
unsigned char *out, *in;
size_t outlen, r;
if (hdr->flags & PKG_FORMAT_FLAGS_ZSTD) {
outlen = hdr->additional;
if (outlen > UINT32_MAX) {
/* Some garbadge instead of size */
pkg_emit_error("invalid uncompressed size: %zu", outlen);
return (EPKG_FATAL);
}
/* Read input */
in = xmalloc(hdr->size);
if (fread(in, 1, hdr->size, fs) != hdr->size) {
return (EPKG_FATAL);
}
zstream = ZSTD_createDStream();
ZSTD_initDStream(zstream);
zin.pos = 0;
zin.src = in;
zin.size = hdr->size;
if (outlen == 0 &&
(outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
outlen = ZSTD_DStreamOutSize();
}
out = xmalloc(outlen);
zout.dst = out;
zout.pos = 0;
zout.size = outlen;
while (zin.pos < zin.size) {
r = ZSTD_decompressStream(zstream, &zout, &zin);
if (ZSTD_isError(r)) {
pkg_emit_error("cannot decompress data: %s",
ZSTD_getErrorName(r));
ZSTD_freeDStream(zstream);
free(out);
free(in);
return (EPKG_FATAL);
}
if (zout.pos == zout.size) {
/* We need to extend output buffer */
zout.size = zout.size * 1.5 + 1.0;
out = xrealloc(zout.dst, zout.size);
zout.dst = out;
}
}
ZSTD_freeDStream(zstream);
free(in);
*uncompressed = out;
*sz = zout.pos;
}
else {
*sz = hdr->size;
out = xmalloc(hdr->size);
if (fread(out, 1, hdr->size, fs) != hdr->size) {
free(out);
return (EPKG_FATAL);
}
*uncompressed = out;
}
return (EPKG_OK);
}
