static int encode_snappy(avro_codec_t c, void * data, int64_t len)
{
uint32_t crc;
size_t outlen = snappy_max_compressed_length(len);
if (!c->block_data) {
c->block_data = avro_malloc(outlen+4);
c->block_size = outlen+4;
} else if (c->block_size < (int64_t) (outlen+4)) {
c->block_data = avro_realloc(c->block_data, c->block_size, (outlen+4));
c->block_size = outlen+4;
}
if (!c->block_data) {
avro_set_error("Cannot allocate memory for snappy");
return 1;
}
if (snappy_compress(data, len, c->block_data, &outlen) != SNAPPY_OK)
{
avro_set_error("Error compressing block with Snappy");
return 1;
}
crc = __bswap_32(crc32(0, data, len));
memcpy(c->block_data+outlen, &crc, 4);
c->used_size = outlen+4;
return 0;
}
static VALUE
snappy_deflate(int argc, VALUE *argv, VALUE self)
{
VALUE src, dst;
size_t output_length;
snappy_status result;
rb_scan_args(argc, argv, "11", &src, &dst);
StringValue(src);
output_length = snappy_max_compressed_length(RSTRING_LEN(src));
if (NIL_P(dst)) {
dst = rb_str_new(NULL, output_length);
} else {
StringValue(dst);
rb_str_resize(dst, output_length);
}
result = snappy_compress(RSTRING_PTR(src), RSTRING_LEN(src), RSTRING_PTR(dst), &output_length);
if (result == SNAPPY_OK) {
rb_str_resize(dst, output_length);
return dst;
} else {
return snappy_raise(result);
}
}
int _tmain(int argc, _TCHAR* argv[])
{
FILE* f = NULL;
errno_t err = fopen_s(&f, "readme.txt", "rb");
fseek(f, 0, SEEK_END);
int fileSize = ftell(f);
fseek(f, 0, SEEK_SET);
char* source = new char[fileSize];
int bufferLen = fileSize + 300;
char* compressed = new char[bufferLen];
fread(source, fileSize, 1, f);
fclose(f);
size_t length = bufferLen;
snappy_status status = snappy_compress(source, fileSize, compressed, &length);
delete source;
char* uncompressed = new char[bufferLen];
size_t uncompressedLength = bufferLen;
status = snappy_uncompress(compressed, length, uncompressed, &uncompressedLength);
assert(uncompressedLength == fileSize);
delete compressed;
delete uncompressed;
return 0;
}
void ColorTable::serialize(In* in, Out* out)
{
STREAM_REGISTER_BEGIN;
size_t ctUncompressedSize = sizeof(colorTable);
size_t ctCompressedSize = 0;
std::vector<char> ctCompressed;
if(out)
{
ctCompressedSize = snappy_max_compressed_length(ctUncompressedSize);
ctCompressed.resize(ctCompressedSize);
VERIFY(snappy_compress(reinterpret_cast<char*>(&colorTable[0][0][0]), ctUncompressedSize, ctCompressed.data(), &ctCompressedSize) == SNAPPY_OK);
out->write(&ctCompressedSize, sizeof(int));
out->write(ctCompressed.data(), ctCompressedSize);
}
else
{
in->read(&ctCompressedSize, sizeof(int));
ctCompressed.resize(ctCompressedSize);
in->read(ctCompressed.data(), ctCompressedSize);
VERIFY(snappy_uncompress(ctCompressed.data(), ctCompressedSize, reinterpret_cast<char*>(&colorTable[0][0][0]), &ctUncompressedSize) == SNAPPY_OK);
ASSERT(ctUncompressedSize == sizeof(colorTable));
}
STREAM_REGISTER_FINISH;
}
void Logger::writeThread()
{
OutBinaryFile file(logFilename);
ASSERT(file.exists());
file << logFileCompressed; // Write magic byte that indicates a compressed log file
const size_t compressedSize = snappy_max_compressed_length(parameters.blockSize + 2 * sizeof(unsigned));
std::vector<char> compressedBuffer(compressedSize + sizeof(unsigned)); // Also reserve 4 bytes for header
while(writerThread.isRunning()) // Check if we are expecting more data
if(framesToWrite.wait(100)) // Wait 100 ms for new data then check again if we should quit
{
writerIdle = false;
MessageQueue& queue = *buffer[readIndex];
if(queue.getNumberOfMessages() > 0)
{
size_t size = compressedSize;
VERIFY(snappy_compress(queue.getStreamedData(), queue.getStreamedSize(),
compressedBuffer.data() + sizeof(unsigned), &size) == SNAPPY_OK);
(unsigned&) compressedBuffer[0] = (unsigned) size;
file.write(compressedBuffer.data(), size + sizeof(unsigned));
queue.clear();
}
readIndex = (readIndex + 1) % parameters.maxBufferSize;
}
else if(!writerIdle)
{
writerIdle = true;
writerIdleStart = SystemCall::getCurrentSystemTime();
}
}
int bp__compress(const char* input,
size_t input_length,
char* compressed,
size_t* compressed_length) {
int ret = snappy_compress(input, input_length, compressed, compressed_length);
return ret == SNAPPY_OK ? BP_OK : BP_ECOMP;
}
static int snappy_java_compress(FILE *infp, FILE *outfp, size_t block_size)
{
snappy_java_header_t header;
work_buffer_t wb;
size_t uncompressed_length;
int err = 1;
wb.c = NULL;
wb.uc = NULL;
if (block_size == 0) {
block_size = DEFAULT_BLOCK_SIZE;
}
/* write the file header */
memcpy(header.magic, SNAPPY_JAVA_MAGIC, SNAPPY_JAVA_MAGIC_LEN);
header.version = htonl(SNAPPY_JAVA_FILE_VERSION);
header.compatible_version = htonl(SNAPPY_JAVA_FILE_VERSION);
if (fwrite_unlocked(&header, sizeof(header), 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
/* write file body */
work_buffer_init(&wb, block_size);
while ((uncompressed_length = fread_unlocked(wb.uc, 1, wb.uclen, infp)) > 0) {
size_t compressed_length = wb.clen;
trace("read %lu bytes.\n", (unsigned long)uncompressed_length);
/* compress the block. */
snappy_compress(wb.uc, uncompressed_length, wb.c, &compressed_length);
trace("compressed_legnth is %lu.\n", (unsigned long)compressed_length);
/* write the compressed length. */
putc_unlocked((compressed_length >> 24), outfp);
putc_unlocked((compressed_length >> 16), outfp);
putc_unlocked((compressed_length >> 8), outfp);
putc_unlocked((compressed_length >> 0), outfp);
trace("write 4 bytes for compressed data length.\n");
/* write the compressed data. */
if (fwrite_unlocked(wb.c, compressed_length, 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
trace("write %ld bytes for compressed data.\n", (long)compressed_length);
}
if (!feof_unlocked(infp)) {
/* fread_unlocked() failed. */
print_error("Failed to read a file: %s\n", strerror(errno));
goto cleanup;
}
err = 0;
cleanup:
work_buffer_free(&wb);
return err;
}
int main(int ac, char **av)
{
int failed = 0, verbose = 0;
struct snappy_env env;
snappy_init_env(&env);
if (av[1] && !strcmp(av[1], "-v")) {
verbose++;
av++;
}
while (*++av) {
size_t size;
char *map = mapfile(*av, O_RDONLY, &size);
if (!map) {
if (size > 0) {
perror(*av);
failed = 1;
}
continue;
}
size_t outlen;
int err;
char *out = xmalloc(snappy_max_compressed_length(size));
char *buf2 = xmalloc(size);
err = snappy_compress(&env, map, size, out, &outlen);
if (err) {
failed = 1;
printf("compression of %s failed: %d\n", *av, err);
goto next;
}
err = snappy_uncompress(out, outlen, buf2);
if (err) {
failed = 1;
printf("uncompression of %s failed: %d\n", *av, err);
goto next;
}
if (memcmp(buf2, map, size)) {
int o = compare(buf2, map, size);
if (o >= 0) {
failed = 1;
printf("final comparision of %s failed at %d of %lu\n",
*av, o, (unsigned long)size);
}
} else {
if (verbose)
printf("%s OK!\n", *av);
}
next:
unmap_file(map, size);
free(out);
free(buf2);
}
return failed;
}
int memCompressSnappy( char *out, int outlen, char *in, int inlen ) {
size_t maxsz = snappy_max_compressed_length(inlen);
assertmsg( outlen >= maxsz, "snappy requires buffer size:%d given:%d", maxsz, outlen );
size_t osz = outlen;
snappy_status ret = snappy_compress( in, inlen, out, &osz);
if(ret == SNAPPY_OK ) return (int)osz; else assertmsg(false,"snappy_compress failed. outlen:%d inlen:%d ret:%d", outlen, inlen,ret );
return 0;
}
CChunkFileReader::Error CChunkFileReader::SaveFile(const std::string& _rFilename, int _Revision, const char *_VersionString, u8 *buffer, size_t sz) {
INFO_LOG(COMMON, "ChunkReader: Writing %s" , _rFilename.c_str());
File::IOFile pFile(_rFilename, "wb");
if (!pFile)
{
ERROR_LOG(COMMON, "ChunkReader: Error opening file for write");
return ERROR_BAD_FILE;
}
bool compress = true;
// Create header
SChunkHeader header;
header.Compress = compress ? 1 : 0;
header.Revision = _Revision;
header.ExpectedSize = (u32)sz;
header.UncompressedSize = (u32)sz;
strncpy(header.GitVersion, _VersionString, 32);
header.GitVersion[31] = '\0';
// Write to file
if (compress) {
size_t comp_len = snappy_max_compressed_length(sz);
u8 *compressed_buffer = new u8[comp_len];
snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &comp_len);
delete [] buffer;
header.ExpectedSize = (u32)comp_len;
if (!pFile.WriteArray(&header, 1))
{
ERROR_LOG(COMMON, "ChunkReader: Failed writing header");
return ERROR_BAD_FILE;
}
if (!pFile.WriteBytes(&compressed_buffer[0], comp_len)) {
ERROR_LOG(COMMON, "ChunkReader: Failed writing compressed data");
return ERROR_BAD_FILE;
} else {
INFO_LOG(COMMON, "Savestate: Compressed %i bytes into %i", (int)sz, (int)comp_len);
}
delete [] compressed_buffer;
} else {
if (!pFile.WriteArray(&header, 1))
{
ERROR_LOG(COMMON, "ChunkReader: Failed writing header");
return ERROR_BAD_FILE;
}
if (!pFile.WriteBytes(&buffer[0], sz))
{
ERROR_LOG(COMMON, "ChunkReader: Failed writing data");
return ERROR_BAD_FILE;
}
delete [] buffer;
}
INFO_LOG(COMMON, "ChunkReader: Done writing %s", _rFilename.c_str());
return ERROR_NONE;
}
static int snappy_in_java_compress(FILE *infp, FILE *outfp, size_t block_size)
{
work_buffer_t wb;
size_t uncompressed_length;
int err = 1;
wb.c = NULL;
wb.uc = NULL;
if (block_size == 0) {
block_size = DEFAULT_BLOCK_SIZE;
}
if (block_size > MAX_BLOCK_SIZE) {
print_error("Too large block size: %lu. (default: %d, max: %d)\n",
(unsigned long)block_size, DEFAULT_BLOCK_SIZE, MAX_BLOCK_SIZE);
goto cleanup;
}
if (fwrite_unlocked(&snappy_in_java_header, sizeof(snappy_in_java_header), 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
/* write file body */
work_buffer_init(&wb, block_size);
while ((uncompressed_length = fread_unlocked(wb.uc, 1, wb.uclen, infp)) > 0) {
size_t compressed_length = wb.clen;
unsigned int crc32c = masked_crc32c(wb.uc, uncompressed_length);
trace("read %lu bytes.\n", (unsigned long)uncompressed_length);
/* compress the block. */
snappy_compress(wb.uc, uncompressed_length, wb.c, &compressed_length);
trace("compressed_legnth is %lu.\n", (unsigned long)compressed_length);
if (compressed_length >= (uncompressed_length - (uncompressed_length / 8))) {
trace("write uncompressed data\n");
if (write_block(outfp, wb.uc, uncompressed_length, FALSE, crc32c) != 0) {
goto cleanup;
}
} else {
trace("write compressed data\n");
if (write_block(outfp, wb.c, compressed_length, TRUE, crc32c) != 0) {
goto cleanup;
}
}
}
if (!feof_unlocked(infp)) {
/* fread_unlocked() failed. */
print_error("Failed to read a file: %s\n", strerror(errno));
goto cleanup;
}
err = 0;
cleanup:
work_buffer_free(&wb);
return err;
}
primitiveCompressWithOffset(void)
{
char* compressed;
size_t compressedLength;
const char*input;
sqInt inputLength;
sqInt inputObj;
size_t inputOffset;
sqInt num;
char* ptr;
sqInt status;
if (!((interpreterProxy->methodArgumentCount()) == 2)) {
interpreterProxy->primitiveFail(); return;
}
inputObj = interpreterProxy->stackValue(1);
/* begin charPointerFor: */
if (!(interpreterProxy->isBytes(inputObj))) {
input = ((char*) null);
goto l1;
}
ptr = ((char*) (interpreterProxy->firstIndexableField(inputObj)));
input = ((char*) ptr);
l1: /* end charPointerFor: */;
if (!(input)) {
interpreterProxy->primitiveFail(); return;
}
/* begin stackPositiveIntegerValue: */
num = interpreterProxy->stackValue(0);
if ((interpreterProxy->isIntegerValue(num))
&& (num < 0)) {
inputOffset = ((sqInt) null);
goto l2;
}
inputOffset = ((sqInt) (interpreterProxy->positive32BitValueOf(num)));
l2: /* end stackPositiveIntegerValue: */;
inputLength = interpreterProxy->byteSizeOf(interpreterProxy->stackValue(1));
;
compressedLength = snappy_max_compressed_length(inputLength);
if (compressedLength < 1) {
interpreterProxy->primitiveFail(); return;
}
compressed = malloc(compressedLength);
if (!(compressed)) {
interpreterProxy->primitiveFail(); return;
}
status = snappy_compress(input + inputOffset, inputLength, compressed, &compressedLength);
if (!(status == 0)) {
/* begin returnErrorInfoFor: */
interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
interpreterProxy->pushInteger(-status);
return;
}
interpreterProxy->pop((interpreterProxy->methodArgumentCount()) + 1);
interpreterProxy->push(oopFromCBytessized(compressed, compressedLength));
free(compressed);
}
int SnappyImplNative::compress(u8* in, int inOffset, u8 *out, int outOffset, int len) {
size_t comp_len;
int res = snappy_compress(&this->env, (const char*) (in + inOffset), (size_t) len, (char*) (out + outOffset), &comp_len);
if (res != 0) { // not ok
return -res;
} else {
return comp_len;
}
}
static ssize_t compress_iov_to_buffer(struct trace_record *target, const struct iovec *iov, int iovcnt)
{
if (NULL == iov) {
errno = EFAULT;
return -1;
}
if (iovcnt < 1) {
return 0;
}
char *src_buf = iov[0].iov_base;
size_t input_len = iov[0].iov_len;
size_t compressed_length = 0;
struct snappy_env env;
int rc = snappy_init_env(&env);
if (rc < 0) {
goto finish;
}
if (iovcnt > 1) {
/* TODO: Snappy should be able to accept an input IOV directly. Since this support is currently broken we use a workaround instead. */
input_len = total_iovec_len(iov, iovcnt);
src_buf = malloc(input_len);
if ((NULL == src_buf) || (copy_iov_to_buffer(src_buf, iov, iovcnt) != (ssize_t) input_len)) {
rc = -errno;
goto finish;
}
}
if (input_len > 0) {
/* Don't bother compressing trailing padding chars. Those will be re-inserted by the reader. */
input_len -= trace_r_count_chr_occurrences(src_buf + 1, input_len - 1, TRACE_UNUSED_SPACE_FILL_VALUE);
rc = snappy_compress(&env, src_buf, input_len, (char *)target, &compressed_length);
}
finish:
snappy_free_env(&env);
if (src_buf != iov[0].iov_base) {
free(src_buf);
}
if (0 != rc) {
errno = -rc;
ERR("Buffer compression failed with err", errno, strerror(errno));
return (ssize_t) -1;
}
TRACE_ASSERT(((ssize_t) compressed_length > 0) || (0 == input_len));
return (ssize_t) compressed_length;
}
static hg_return_t
snappy_pull_cb(const struct hg_bulk_cb_info *hg_bulk_cb_info)
{
struct snappy_transfer_args *snappy_transfer_args =
(struct snappy_transfer_args *) hg_bulk_cb_info->arg;
hg_return_t ret = HG_SUCCESS;
void *input;
size_t input_length;
size_t source_length = HG_Bulk_get_size(snappy_transfer_args->local_input_bulk_handle);
/* Get pointer to input buffer from local handle */
HG_Bulk_access(hg_bulk_cb_info->local_handle, 0, source_length,
HG_BULK_READ_ONLY, 1, &input, &input_length, NULL);
printf("Transferred input buffer of length: %zu\n", input_length);
print_buf(20, (int *) input);
/* Allocate compressed buffer for compressing input data */
snappy_transfer_args->compressed_length = snappy_max_compressed_length(input_length);
snappy_transfer_args->compressed = malloc(snappy_transfer_args->compressed_length);
/* Compress data */
printf("Compressing buffer...\n");
snappy_transfer_args->ret = snappy_compress(input, input_length,
snappy_transfer_args->compressed, &snappy_transfer_args->compressed_length);
printf("Return value of snappy_compress is: %d\n", snappy_transfer_args->ret);
printf("Compressed buffer length is: %zu\n", snappy_transfer_args->compressed_length);
print_buf(5, (int *) snappy_transfer_args->compressed);
/* Free bulk handles */
HG_Bulk_free(snappy_transfer_args->local_input_bulk_handle);
if (snappy_validate_compressed_buffer(snappy_transfer_args->compressed,
snappy_transfer_args->compressed_length) == SNAPPY_OK) {
printf("Compressed buffer validated: compressed successfully\n");
}
/* Now set up bulk transfer for "push to origin" callback */
HG_Bulk_create(HG_Get_info(snappy_transfer_args->handle)->hg_bulk_class, 1,
&snappy_transfer_args->compressed, &snappy_transfer_args->compressed_length,
HG_BULK_WRITE_ONLY, &snappy_transfer_args->local_compressed_bulk_handle);
HG_Bulk_transfer(HG_Get_info(snappy_transfer_args->handle)->bulk_context,
snappy_push_cb, snappy_transfer_args,
HG_BULK_PUSH, HG_Get_info(snappy_transfer_args->handle)->addr,
snappy_transfer_args->snappy_compress_input.compressed_bulk_handle, 0, /* origin */
snappy_transfer_args->local_compressed_bulk_handle, 0, /* local */
snappy_transfer_args->compressed_length, HG_OP_ID_IGNORE);
return ret;
}
static int snp_compress(struct crypto_tfm *tfm, const u8 *src,
unsigned int slen, u8 *dst, unsigned int *dlen)
{
struct snappy_ctx *ctx = crypto_tfm_ctx(tfm);
size_t olen;
int err;
/* XXXX very pessimistic. check in snappy? */
if (*dlen < snappy_max_compressed_length(*dlen))
return -EINVAL;
err = snappy_compress(&ctx->env, src, slen, dst, &olen);
*dlen = olen;
return err;
}
static SquashStatus
squash_snappy_compress_buffer (SquashCodec* codec,
size_t* compressed_length,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_length)],
size_t uncompressed_length,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)],
SquashOptions* options) {
snappy_status e;
e = snappy_compress ((char*) uncompressed, uncompressed_length,
(char*) compressed, compressed_length);
return squash_snappy_status (e);
}
bool
mongoc_compress (int32_t compressor_id,
int32_t compression_level,
char *uncompressed,
size_t uncompressed_len,
char *compressed,
size_t *compressed_len)
{
TRACE ("Compressing with '%s' (%d)",
mongoc_compressor_id_to_name (compressor_id),
compressor_id);
switch (compressor_id) {
case MONGOC_COMPRESSOR_SNAPPY_ID:
#ifdef MONGOC_ENABLE_COMPRESSION_SNAPPY
/* No compression_level option for snappy */
return snappy_compress (
uncompressed, uncompressed_len, compressed, compressed_len) ==
SNAPPY_OK;
break;
#else
MONGOC_ERROR ("Client attempting to use compress with snappy, but snappy "
"compression is not compiled in");
return false;
#endif
case MONGOC_COMPRESSOR_ZLIB_ID:
#ifdef MONGOC_ENABLE_COMPRESSION_ZLIB
return compress2 ((unsigned char *) compressed,
(unsigned long *) compressed_len,
(unsigned char *) uncompressed,
uncompressed_len,
compression_level) == Z_OK;
break;
#else
MONGOC_ERROR ("Client attempting to use compress with zlib, but zlib "
"compression is not compiled in");
return false;
#endif
case MONGOC_COMPRESSOR_NOOP_ID:
memcpy (compressed, uncompressed, uncompressed_len);
*compressed_len = uncompressed_len;
return true;
default:
return false;
}
}
int ness_compress(ness_compress_method_t m,
const char *src,
uint32_t src_size,
char *dst,
uint32_t *dst_size)
{
int ret = NESS_OK;
switch (m) {
case NESS_NO_COMPRESS:
memcpy(dst + 1, src, src_size);
*dst_size = src_size + 1;
dst[0] = NESS_NO_COMPRESS;
break;
case NESS_SNAPPY_METHOD:
if (src_size == 0) {
*dst_size = 1;
} else {
size_t out_size;
int status;
struct snappy_env env;
snappy_init_env(&env);
status = snappy_compress(&env, src, src_size, dst + 1, &out_size);
snappy_free_env(&env);
if (status != 0) {
__ERROR("snappy compress error %d, src_size %d, dst_size %d",
status,
src_size,
dst_size);
ret = 0;
}
*dst_size = out_size + 1;
}
dst[0] = NESS_SNAPPY_METHOD;
break;
default:
ret = 0;
__ERROR("%s", "no compress method support!");
break;
}
return ret;
}
static int hap_compress_frame(AVCodecContext *avctx, uint8_t *dst)
{
HapContext *ctx = avctx->priv_data;
int i, final_size = 0;
for (i = 0; i < ctx->chunk_count; i++) {
HapChunk *chunk = &ctx->chunks[i];
uint8_t *chunk_src, *chunk_dst;
int ret;
if (i == 0) {
chunk->compressed_offset = 0;
} else {
chunk->compressed_offset = ctx->chunks[i-1].compressed_offset
+ ctx->chunks[i-1].compressed_size;
}
chunk->uncompressed_size = ctx->tex_size / ctx->chunk_count;
chunk->uncompressed_offset = i * chunk->uncompressed_size;
chunk->compressed_size = ctx->max_snappy;
chunk_src = ctx->tex_buf + chunk->uncompressed_offset;
chunk_dst = dst + chunk->compressed_offset;
/* Compress with snappy too, write directly on packet buffer. */
ret = snappy_compress(chunk_src, chunk->uncompressed_size,
chunk_dst, &chunk->compressed_size);
if (ret != SNAPPY_OK) {
av_log(avctx, AV_LOG_ERROR, "Snappy compress error.\n");
return AVERROR_BUG;
}
/* If there is no gain from snappy, just use the raw texture. */
if (chunk->compressed_size >= chunk->uncompressed_size) {
av_log(avctx, AV_LOG_VERBOSE,
"Snappy buffer bigger than uncompressed (%"SIZE_SPECIFIER" >= %"SIZE_SPECIFIER" bytes).\n",
chunk->compressed_size, chunk->uncompressed_size);
memcpy(chunk_dst, chunk_src, chunk->uncompressed_size);
chunk->compressor = HAP_COMP_NONE;
chunk->compressed_size = chunk->uncompressed_size;
} else {
chunk->compressor = HAP_COMP_SNAPPY;
}
final_size += chunk->compressed_size;
}
return final_size;
}
static size_t
do_snappy(struct page_data *pg)
{
struct page_data_kdump *pgkdump = pg->priv;
size_t clen;
clen = snappy_max_compressed_length(pg->len);
if (clen > pgkdump->cbufsz &&
!(clen = enlarge_cbuf(pgkdump, clen)))
return clen;
if (snappy_compress((const char*)pg->buf, pg->len,
pgkdump->cbuf, &clen) != SNAPPY_OK) {
fprintf(stderr, "snappy compression failed\n");
clen = 0;
}
return clen;
}
int main(int argc, char* argv[])
{
char arch[50];
size_t arch_sz = sizeof(arch);
snappy_status status = snappy_compress("Hello World!", 12, arch, &arch_sz);
if (status != SNAPPY_OK)
printf("compress fail");
char src[50];
size_t src_sz = sizeof(src);
status = snappy_uncompress(arch, arch_sz, src, &src_sz);
if (status != SNAPPY_OK)
return -1;
if (strncmp(src, "Hello, world!", 12) == 0)
return -1;
return 0;
}
static int
wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_len,
size_t *result_lenp, int *compression_failed)
{
snappy_status snret;
size_t snaplen;
char *snapbuf;
__UNUSED(compressor);
/*
* dst_len was computed in wt_snappy_pre_size, so we know it's big
* enough. Skip past the space we'll use to store the final count
* of compressed bytes.
*/
snaplen = dst_len - sizeof(size_t);
snapbuf = (char *)dst + sizeof(size_t);
/* snaplen is an input and an output arg. */
snret = snappy_compress((char *)src, src_len, snapbuf, &snaplen);
if (snret == SNAPPY_OK) {
/*
* On decompression, snappy requires the exact compressed byte
* count (the current value of snaplen). WiredTiger does not
* preserve that value, so save snaplen at the beginning of the
* destination buffer.
*/
if (snaplen + sizeof(size_t) < src_len) {
*(size_t *)dst = snaplen;
*result_lenp = snaplen + sizeof(size_t);
*compression_failed = 0;
} else
/* The compressor failed to produce a smaller result. */
*compression_failed = 1;
return (0);
}
return (wt_snappy_error(session, "snappy_compress", snret));
}
Datum
snappy_compress_internal(PG_FUNCTION_ARGS)
{
const char *src = PG_GETARG_POINTER(0);
size_t src_sz = PG_GETARG_INT32(1);
char *dst = PG_GETARG_POINTER(2);
size_t dst_sz = PG_GETARG_INT32(3);
size_t *dst_used = PG_GETARG_POINTER(4);
size_t compressed_length;
snappy_status retval;
compressed_length = snappy_max_compressed_length(src_sz);
Insist(dst_sz >= compressed_length);
retval = snappy_compress(src, src_sz, dst, &compressed_length);
*dst_used = compressed_length;
if (retval != SNAPPY_OK)
elog_snappy_error(retval, "snappy_compress", src_sz, dst_sz, *dst_used);
PG_RETURN_VOID();
}
static ZEND_FUNCTION(snappy_compress)
{
zval *data;
char *output;
size_t output_len;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z", &data) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"snappy_compress : expects parameter to be string.");
RETURN_FALSE;
}
output_len = snappy_max_compressed_length(Z_STRLEN_P(data));
output = (char *)emalloc(output_len);
if (!output) {
zend_error(E_WARNING, "snappy_compress : memory error");
RETURN_FALSE;
}
if (snappy_compress(Z_STRVAL_P(data), Z_STRLEN_P(data),
output, &output_len) == SNAPPY_OK) {
#if ZEND_MODULE_API_NO >= 20141001
RETVAL_STRINGL(output, output_len);
#else
RETVAL_STRINGL(output, output_len, 1);
#endif
} else {
RETVAL_FALSE;
}
efree(output);
}
unsigned int HapEncode(const void *inputBuffer, unsigned long inputBufferBytes, unsigned int textureFormat,
unsigned int compressor, void *outputBuffer, unsigned long outputBufferBytes,
unsigned long *outputBufferBytesUsed)
{
size_t maxCompressedLength;
size_t maxOutputBufferLength;
size_t headerLength;
void *compressedStart;
size_t storedLength;
unsigned int storedCompressor;
unsigned int storedFormat;
/*
Check arguments
*/
if (inputBuffer == NULL
|| inputBufferBytes == 0
|| (textureFormat != HapTextureFormat_RGB_DXT1
&& textureFormat != HapTextureFormat_RGBA_DXT5
&& textureFormat != HapTextureFormat_YCoCg_DXT5
)
|| (compressor != HapCompressorNone
&& compressor != HapCompressorSnappy
)
)
{
return HapResult_Bad_Arguments;
}
maxCompressedLength = compressor == HapCompressorSnappy ? snappy_max_compressed_length(inputBufferBytes) : inputBufferBytes;
if (maxCompressedLength < inputBufferBytes)
{
// Sanity check in case a future Snappy promises to always compress
maxCompressedLength = inputBufferBytes;
}
/*
To store frames of length greater than can be expressed in three bytes, we use an eight byte header (the last four bytes are the
frame size). We don't know the compressed size until we have performed compression, but we know the worst-case size
(the uncompressed size), so choose header-length based on that.
A simpler encoder could always use the eight-byte header variation.
*/
if (inputBufferBytes > kHapUInt24Max)
{
headerLength = 8U;
}
else
{
headerLength = 4U;
}
maxOutputBufferLength = maxCompressedLength + headerLength;
if (outputBufferBytes < maxOutputBufferLength
|| outputBuffer == NULL)
{
return HapResult_Buffer_Too_Small;
}
compressedStart = ((uint8_t *)outputBuffer) + headerLength;
if (compressor == HapCompressorSnappy)
{
snappy_status result;
storedLength = outputBufferBytes;
result = snappy_compress((const char *)inputBuffer, inputBufferBytes, (char *)compressedStart, &storedLength);
if (result != SNAPPY_OK)
{
return HapResult_Internal_Error;
}
storedCompressor = kHapCompressorSnappy;
}
else
{
// HapCompressorNone
// Setting storedLength to 0 causes the frame to be used uncompressed
storedLength = 0;
}
/*
If our "compressed" frame is no smaller than our input frame then store the input uncompressed.
*/
if (storedLength == 0 || storedLength >= inputBufferBytes)
{
memcpy(compressedStart, inputBuffer, inputBufferBytes);
storedLength = inputBufferBytes;
storedCompressor = kHapCompressorNone;
}
storedFormat = hap_texture_format_identifier_for_format_constant(textureFormat);
hap_write_section_header(outputBuffer, headerLength, storedLength, hap_4_bit_packed_byte(storedCompressor, storedFormat));
if (outputBufferBytesUsed != NULL)
{
*outputBufferBytesUsed = storedLength + headerLength;
}
return HapResult_No_Error;
}
static int framing_format_compress(FILE *infp, FILE *outfp, size_t block_size)
{
const size_t max_uncompressed_data_len = MAX_UNCOMPRESSED_DATA_LEN;
const size_t max_compressed_data_len = snappy_max_compressed_length(max_uncompressed_data_len);
size_t uncompressed_data_len;
size_t compressed_data_len;
char *uncompressed_data = malloc(max_uncompressed_data_len);
char *compressed_data = malloc(max_compressed_data_len);
int err = 1;
if (uncompressed_data == NULL || compressed_data == NULL) {
print_error("out of memory\n");
goto cleanup;
}
/* write the steam header */
fwrite_unlocked(stream_header, sizeof(stream_header), 1, outfp);
/* write file body */
while ((uncompressed_data_len = fread_unlocked(uncompressed_data, 1, max_uncompressed_data_len, infp)) > 0) {
unsigned int crc32c = masked_crc32c(uncompressed_data, uncompressed_data_len);
char type_code;
size_t write_len;
const char *write_data;
/* compress the block. */
compressed_data_len = max_compressed_data_len;
snappy_compress(uncompressed_data, uncompressed_data_len, compressed_data, &compressed_data_len);
if (compressed_data_len >= (uncompressed_data_len - (uncompressed_data_len / 8))) {
/* uncompressed data */
type_code = UNCOMPRESSED_DATA_IDENTIFIER;
write_len = uncompressed_data_len;
write_data = uncompressed_data;
} else {
/* compressed data */
type_code = COMPRESSED_DATA_IDENTIFIER;
write_len = compressed_data_len;
write_data = compressed_data;
}
/* write block type */
putc_unlocked(type_code, outfp);
/* write data length */
putc_unlocked(((write_len + 4) >> 0), outfp);
putc_unlocked(((write_len + 4) >> 8), outfp);
/* write checksum */
putc_unlocked((crc32c >> 0), outfp);
putc_unlocked((crc32c >> 8), outfp);
putc_unlocked((crc32c >> 16), outfp);
putc_unlocked((crc32c >> 24), outfp);
/* write data */
if (fwrite_unlocked(write_data, write_len, 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
}
/* check stream errors */
if (ferror_unlocked(infp)) {
print_error("Failed to read a file: %s\n", strerror(errno));
goto cleanup;
}
if (ferror_unlocked(outfp)) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
err = 0;
cleanup:
free(uncompressed_data);
free(compressed_data);
return err;
}
static int snzip_compress(FILE *infp, FILE *outfp, size_t block_size)
{
snz_header_t header;
work_buffer_t wb;
size_t uncompressed_length;
int err = 1;
int nshift;
wb.c = NULL;
wb.uc = NULL;
if (block_size == 0) {
block_size = 1ul << SNZ_DEFAULT_BLOCK_SIZE;
nshift = SNZ_DEFAULT_BLOCK_SIZE;
} else {
if (block_size > (1ul << SNZ_MAX_BLOCK_SIZE)) {
print_error("too large block size: %lu\n", block_size);
goto cleanup;
}
for (nshift = 1; nshift <= SNZ_MAX_BLOCK_SIZE; nshift++) {
if (1ul << nshift == block_size) {
break;
}
}
if (nshift == SNZ_MAX_BLOCK_SIZE) {
print_error("The block size must be power of two\n");
goto cleanup;
}
}
/* write the file header */
memcpy(header.magic, SNZ_MAGIC, SNZ_MAGIC_LEN);
header.version = SNZ_FILE_VERSION;
header.block_size = nshift;
if (fwrite_unlocked(&header, sizeof(header), 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
/* write file body */
work_buffer_init(&wb, block_size);
while ((uncompressed_length = fread_unlocked(wb.uc, 1, wb.uclen, infp)) > 0) {
size_t compressed_length = wb.clen;
trace("read %lu bytes.\n", (unsigned long)uncompressed_length);
/* compress the block. */
snappy_compress(wb.uc, uncompressed_length, wb.c, &compressed_length);
trace("compressed_legnth is %lu.\n", (unsigned long)compressed_length);
/* write the compressed length. */
if (compressed_length < (1ul << 7)) {
putc_unlocked(compressed_length, outfp);
trace("write 1 byte for compressed data length.\n");
} else if (compressed_length < (1ul << 14)) {
putc_unlocked((compressed_length >> 0) | 0x80, outfp);
putc_unlocked((compressed_length >> 7), outfp);
trace("write 2 bytes for compressed data length.\n");
} else if (compressed_length < (1ul << 21)) {
static int comment_43_compress(FILE *infp, FILE *outfp, size_t block_size)
{
const size_t max_raw_data_len = 32 * 1024; /* maximum data length */
const size_t max_compressed_data_len = snappy_max_compressed_length(max_raw_data_len); /* maximum compressed length */
size_t raw_data_len;
size_t compressed_data_len;
char *raw_data = malloc(max_raw_data_len);
char *compressed_data = malloc(max_compressed_data_len);
int err = 1;
if (raw_data == NULL || compressed_data == NULL) {
print_error("out of memory\n");
goto cleanup;
}
putc_unlocked(HEADER_TYPE_CODE, outfp);
putc_unlocked(MAGIC_LEN, outfp);
putc_unlocked(MAGIC_LEN >> 8, outfp);
fwrite_unlocked(MAGIC, MAGIC_LEN, 1, outfp);
/* write file body */
while ((raw_data_len = fread_unlocked(raw_data, 1, max_raw_data_len, infp)) > 0) {
unsigned int crc32c = masked_crc32c(raw_data, raw_data_len);
char type_code;
size_t write_len;
const char *write_data;
/* compress the block. */
compressed_data_len = max_compressed_data_len;
snappy_compress(raw_data, raw_data_len, compressed_data, &compressed_data_len);
if (compressed_data_len >= (raw_data_len - (raw_data_len / 8))) {
/* write uncompressed data */
type_code = UNCOMPRESSED_TYPE_CODE;
write_len = raw_data_len;
write_data = raw_data;
} else {
/* write compressed data */
type_code = COMPRESSED_TYPE_CODE;
write_len = compressed_data_len;
write_data = compressed_data;
}
/* block type */
putc_unlocked(type_code, outfp);
/* data length */
putc_unlocked(((write_len + 4) >> 0), outfp);
putc_unlocked(((write_len + 4) >> 8), outfp);
/* data */
putc_unlocked((crc32c >> 0), outfp);
putc_unlocked((crc32c >> 8), outfp);
putc_unlocked((crc32c >> 16), outfp);
putc_unlocked((crc32c >> 24), outfp);
if (fwrite_unlocked(write_data, write_len, 1, outfp) != 1) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
}
if (!feof_unlocked(infp)) {
/* fread_unlocked() failed. */
print_error("Failed to read a file: %s\n", strerror(errno));
goto cleanup;
}
putc_unlocked(END_OF_STREAM_TYPE_CODE, outfp);
putc_unlocked(0, outfp);
putc_unlocked(0, outfp);
if (ferror_unlocked(outfp)) {
print_error("Failed to write a file: %s\n", strerror(errno));
goto cleanup;
}
err = 0;
cleanup:
free(raw_data);
free(compressed_data);
return err;
}
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
{
return snappy_compress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
}
