コード例 #1
0
ファイル: python-lz4f.c プロジェクト: darkdragn/lz4tools
static PyObject *py_lz4f_freeCompCtx(PyObject *self, PyObject *args) {
    PyObject *py_cCtx;
    LZ4F_compressionContext_t cCtx;

    (void)self;
    if (!PyArg_ParseTuple(args, "O", &py_cCtx)) {
        return NULL;
    }

    cCtx = (LZ4F_compressionContext_t)PyCapsule_GetPointer(py_cCtx, NULL);
    LZ4F_freeCompressionContext(cCtx);

    return Py_None;
}
コード例 #2
0
ファイル: squash-lz4f.c プロジェクト: szabadka/squash
static void
squash_lz4f_stream_destroy (void* stream) {
  SquashLZ4FStream* s = (SquashLZ4FStream*) stream;

  if (((SquashStream*) stream)->stream_type == SQUASH_STREAM_COMPRESS) {
    LZ4F_freeCompressionContext(s->data.comp.ctx);

    if (s->data.comp.output_buffer != NULL)
      squash_free (s->data.comp.output_buffer);
  } else {
    LZ4F_freeDecompressionContext(s->data.decomp.ctx);
  }

  squash_stream_destroy (stream);
}
コード例 #3
0
ファイル: squash-lz4f.c プロジェクト: eevans/squash-deb
static void
squash_lz4f_stream_destroy (void* stream) {
  SquashLZ4FStream* s = (SquashLZ4FStream*) stream;
  LZ4F_errorCode_t ec;

  if (((SquashStream*) stream)->stream_type == SQUASH_STREAM_COMPRESS) {
    ec = LZ4F_freeCompressionContext(s->data.comp.ctx);

    if (s->data.comp.output_buffer != NULL)
      free (s->data.comp.output_buffer);
  } else {
    ec = LZ4F_freeDecompressionContext(s->data.decomp.ctx);
  }

  assert (!LZ4F_isError (ec));

  squash_stream_destroy (stream);
}
コード例 #4
0
ファイル: interface.c プロジェクト: bwlewis/lz4
/* lz4_compress support function
 * @param src constant input memory buffer of size size to be compressed
 * @param srz_size input memory buffer size
 * @param dest ouput memory buffer of size size in which to place results
 * @param dest_size pointer to a variable that contains the size of dest on input
 *        and is updated to contain the size of compressed data in dest on out
 * @param level compression level from 1 (low/fast) to 9 (high/slow)
 * Returns an error code, 0 for success non-zero otherwise.
 */
int
lz4_compress (void *src, size_t src_size, void *dest, size_t * dest_size,
              int level)
{
  size_t n, len = 0;
  LZ4F_errorCode_t err;
  LZ4F_preferences_t prefs;
  LZ4F_compressionContext_t ctx;
  int retval = 0;

  /* Set compression parameters */
  memset (&prefs, 0, sizeof (prefs));
  prefs.autoFlush = 1;
  prefs.compressionLevel = level;       /* 0...16 */
  prefs.frameInfo.blockMode = 0;        /* blockLinked, blockIndependent ; 0 == default */
  prefs.frameInfo.blockSizeID = 0;      /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
  prefs.frameInfo.contentChecksumFlag = 1;      /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
  prefs.frameInfo.contentSize = (long long)src_size;  /* for reference */

  /* create context */
  err = LZ4F_createCompressionContext (&ctx, LZ4F_VERSION);
  if (LZ4F_isError (err))
    return -1;

  n = LZ4F_compressFrame (dest, *dest_size, src, src_size, &prefs);
  if (LZ4F_isError (n))
    {
      retval = -2;
      goto cleanup;
    }
  /* update the compressed buffer size */
  *dest_size = n;

cleanup:
  if (ctx)
    {
      err = LZ4F_freeCompressionContext (ctx);
      if (LZ4F_isError (err))
        return -5;
    }

  return retval;
}
コード例 #5
0
ファイル: frameCompress.c プロジェクト: digideskio/lz4
static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
	LZ4F_errorCode_t r;
	LZ4F_compressionContext_t ctx;
	char *src, *buf = NULL;
	size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;

	r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
	if (LZ4F_isError(r)) {
		printf("Failed to create context: error %zu", r);
		return 1;
	}
	r = 1;

	src = malloc(BUF_SIZE);
	if (!src) {
		printf("Not enough memory");
		goto cleanup;
	}

	frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
	size =  frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
	buf = malloc(size);
	if (!buf) {
		printf("Not enough memory");
		goto cleanup;
	}

	n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
	if (LZ4F_isError(n)) {
		printf("Failed to start compression: error %zu", n);
		goto cleanup;
	}

	printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);

	for (;;) {
		k = fread(src, 1, BUF_SIZE, in);
		if (k == 0)
			break;
		count_in += k;

		n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
		if (LZ4F_isError(n)) {
			printf("Compression failed: error %zu", n);
			goto cleanup;
		}

		offset += n;
		count_out += n;
		if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
			printf("Writing %zu bytes\n", offset);

			k = fwrite(buf, 1, offset, out);
			if (k < offset) {
				if (ferror(out))
					printf("Write failed");
				else
					printf("Short write");
				goto cleanup;
			}

			offset = 0;
		}
	}

	n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
	if (LZ4F_isError(n)) {
		printf("Failed to end compression: error %zu", n);
		goto cleanup;
	}

	offset += n;
	count_out += n;
	printf("Writing %zu bytes\n", offset);

	k = fwrite(buf, 1, offset, out);
	if (k < offset) {
		if (ferror(out))
			printf("Write failed");
		else
			printf("Short write");
		goto cleanup;
	}

	*size_in = count_in;
	*size_out = count_out;
	r = 0;
 cleanup:
	if (ctx)
		LZ4F_freeCompressionContext(ctx);
	free(src);
	free(buf);
	return r;
}
コード例 #6
0
ファイル: sgtest.c プロジェクト: infidob/lz4
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
    unsigned testResult = 0;
    unsigned testNb = 0;
    void* srcBuffer = NULL;
    void* compressedBuffer = NULL;
    void* decodedBuffer = NULL;
    U32 coreRand = seed;
    LZ4F_decompressionContext_t dCtx = NULL;
    LZ4F_compressionContext_t cCtx = NULL;
    size_t result;
    const U32 startTime = FUZ_GetMilliStart();
    XXH64_state_t xxh64;
#   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                            DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }

    // backup all allocated addresses, from which we will later select buffers
    const size_t max_buf_size = 131 KB;
    size_t num_buf_size_distribution_deviations = 0;

    LZ4SG_in_t  sg_in_buf_potential [2*MAX_SG_BUFFERS];
    LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS];

    LZ4SG_in_t  sg_cin [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
    LZ4SG_in_t  sg_din [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
    size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len;
    const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout));

    unsigned int i;
    for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
        sg_in_buf_potential [i].sg_base = malloc(max_buf_size);
        sg_in_buf_potential [i].sg_len  = max_buf_size;
        sg_out_buf_potential[i].sg_base = malloc(max_buf_size);
        sg_out_buf_potential[i].sg_len  = max_buf_size;
    }

    /* Init */
    duration *= 1000;

    /* Create buffers */
    result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
    CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
    result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
    CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
    srcBuffer = malloc(srcDataLength);
    CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
    const size_t compressedBufferLength = maxDstSize;
    compressedBuffer = malloc(compressedBufferLength);
    CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
    decodedBuffer = calloc(1, srcDataLength);   /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
    CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
    FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);

    /* jump to requested testNb */
    for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand);   // sync randomizer

    /* main fuzzer test loop */
    for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
    {
        U32 randState = coreRand ^ prime1;
        (void)FUZ_rand(&coreRand);   /* update seed */

        srand48(FUZ_rand(&randState));

        DISPLAYUPDATE(2, "\r%5u   ", testNb);

        const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB;
        unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1;
        const size_t min_src_size = 20;
        const size_t min_first_dest_buf_size = 21;
        const size_t min_src_buf_size = 1;
        const size_t min_dst_buf_size = 10;
        size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size;
        size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
        size_t cSize;
        size_t dstSize;
        size_t dstSizeBound;
        U64 crcOrig, crcDecoded;

        unsigned int test_selection = FUZ_rand(&randState);
        //TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress
        int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0;

        if (!lz4f_compress_compatibility_test)
        {
            // SGL compress
            unsigned int buffer_selection = FUZ_rand(&randState);

            if ((buffer_selection & 0xF) == 1)
            {
                // SG compress single source and single target buffers
                sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
                sg_cin[0].sg_len  = srcSize;
                sg_cin_len = 1;
                sg_cout[0].sg_base = compressedBuffer;
                sg_cout[0].sg_len  = compressedBufferLength;
                sg_cout_len = 1;
                dstSizeBound = dstSize = compressedBufferLength;
            }
            else
            {
                // SG compress random number and size source and target buffers
                sg_cin_len  = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
                sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);

                // single source buffer
                if (1 == sg_cin_len) {
                    sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
                    sg_cin[0].sg_len  = srcSize;

                    DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize);
                }
                else {
                    // multiple source buffers
                    if (srcSize > sg_cin_len*max_buf_size/2) {
                        srcSize = sg_cin_len*max_buf_size/2;
                        num_buf_size_distribution_deviations++;
                        DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations);
                    }

                    size_t exact_src_size = 0;
                    unsigned int buf_size_mean = srcSize / sg_cin_len;
                    for (i = 0; i < sg_cin_len; i++) {
                        size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size);
                        DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size);

                        if (srcStart+exact_src_size+buf_size > srcDataLength) {
                            buf_size = srcDataLength-(srcStart+exact_src_size);
                        }
                        sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base;
                        sg_cin[i].sg_len  = buf_size;
                        memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size);
                        exact_src_size += buf_size;
                        if (srcStart+exact_src_size == srcDataLength) {
                            num_buf_size_distribution_deviations++;
                            sg_cin_len = i+1;
                            DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len);
                        }
                    }
                    srcSize = exact_src_size;
                }

                // we can now derive the required limit for output
                dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len);

                // single target buffer
                if (1 == sg_cout_len) {
                    sg_cout[0].sg_base = compressedBuffer;
                    sg_cout[0].sg_len  = compressedBufferLength;
                }
                else {
                    // multiple target buffers
                    int finalBufferTruncated = 0;
                    dstSize = 0;
                    unsigned int buf_size_mean = dstSizeBound / sg_cout_len;
                    for (i = 0; i < sg_cout_len; i++) {
                        const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size;
                        size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size);
                        DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size);

                        if (dstSize+buf_size > dstSizeBound) {
                            buf_size = dstSizeBound-dstSize;
                            finalBufferTruncated = 1;
                        }
                        dstSize += buf_size;

                        sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base;
                        sg_cout[i].sg_len  = buf_size;
                        if (finalBufferTruncated) {
                            num_buf_size_distribution_deviations++;

                            if (buf_size < min_buf_size) {
                                // merge truncated with previous?
                                if (i > 0) {
                                    sg_cout[i-1].sg_len += buf_size;
                                    if (sg_cout[i-1].sg_len > max_buf_size) {
                                        // skip, too much hassle
                                        DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
                                        sg_cout_len = 0; break;
                                    }
                                }
                                else {
                                    // can this happen?
                                    DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
                                    sg_cout_len = 0; break;
                                }
                                sg_cout_len = i;
                            }
                            else {
                                sg_cout_len = i+1;
                            }
                            DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n",
                                    (int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations);
                        }
                    }

                    // skip/abort condition
                    if (0 == sg_cout_len) continue;
                }

                if ((buffer_selection & 0xF) == 0) {
                    //TODO: select a random input and output buffer and split it in two,
                    // feeding consecutive addresses as consecutive entries in SGL

                }
            }

            crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);

            size_t sourceSizeOut = srcSize;
            result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
            if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) {
                // forgive compression failure when output total size is lower than bound
                num_buf_size_distribution_deviations++;
                DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n",
                        (int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations);
                continue;
            }

            CHECK(result <= 0, "Compression failed (error %i)", (int)result);
            CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize);
            cSize = result;
        }
        else
        {
            // LZ4F compression - use it in order to verify SGL decompress compatibility with it
            DISPLAYUPDATE(4, "INFO: LZ4F compression\n");

// alternative
//            size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr);
//            DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n",
//                    srcSize, dstMaxSize);
//            cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr);
//            CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));

            crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);

            unsigned BSId   = 4 + (FUZ_rand(&randState) & 3);
            unsigned BMId   = FUZ_rand(&randState) & 1;
            unsigned CCflag = FUZ_rand(&randState) & 1;
            unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
            U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
            LZ4F_preferences_t prefs;
            LZ4F_compressOptions_t cOptions;

            LZ4F_preferences_t* prefsPtr = &prefs;
            memset(&prefs, 0, sizeof(prefs));
            memset(&cOptions, 0, sizeof(cOptions));
            prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
            prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
            prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
            prefs.frameInfo.contentSize = frameContentSize;
            prefs.autoFlush = autoflush;
            prefs.compressionLevel = FUZ_rand(&randState) % 5;
            if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;

            const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
            const BYTE* const iend = ip + srcSize;
            BYTE* op = (BYTE*)compressedBuffer;
            BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
            unsigned maxBits = FUZ_highbit((U32)srcSize);
            result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
            CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
            op += result;
            while (ip < iend)
            {
                unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
                size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
                size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
                unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
                if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);

                DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n",
                        (int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush);
                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                op += result;
                ip += iSize;

                if (forceFlush)
                {
                    result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
                    CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                    op += result;
                }
            }
            result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
            CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
            op += result;
            cSize = op-(BYTE*)compressedBuffer;
        }

        //DECOMPRESS
        test_selection = FUZ_rand(&randState);

        if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0))
        {
            //TODO: SGL decompress with random buffer sizes

            // SGL decompress with same buffer sizes used for compression
            // prepare din with cout's data
            sg_din_len  = sg_cout_len;
            for (i = 0; i < sg_din_len; i++) {
                sg_din[i].sg_len  = sg_cout[i].sg_len;
                if (sg_cout[i].sg_len <= max_buf_size) {
                    // enough room to copy - do it
                    sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base;
                    if (sg_din[i].sg_base != sg_cout[i].sg_base) {
                        memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len);
                    }
                }
                else {
                    // this is probably single output buffer - skip copy, use directly
                    sg_din[i].sg_base = sg_cout[i].sg_base;
                }
            }
            // prepare dout to receive decompressed data
            sg_dout_len = sg_cin_len;
            for (i = 0; i < sg_dout_len; i++) {
                sg_dout[i].sg_len  = sg_cin[i].sg_len;
                if (sg_cin[i].sg_len <= max_buf_size) {
                    // enough room to decompress into independent buffer
                    sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base;
                }
                else {
                    // this is probably single input buffer, use an external output buffer
                    sg_dout[i].sg_base = decodedBuffer;
                }
            }

            size_t sourceSizeOut = cSize;
            size_t maxOutputSize = srcSize;
            int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize);
            CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result);
            CHECK(decomp_result != (int)srcSize, "SG decompression stopped at  %i", (int)decomp_result);

            // verify result checksum
            size_t total_checked = 0;
            XXH64_reset(&xxh64, 1);
            for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
                size_t cur_size = sg_dout[i].sg_len;
                size_t rem = decomp_result - total_checked;
                if (rem < cur_size) cur_size = rem;
                total_checked += cur_size;

                XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size);
            }
            crcDecoded = XXH64_digest(&xxh64);
            if (crcDecoded != crcOrig) {
                DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n",
                        (int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig);
                // locate error if any
                total_checked = 0;
                for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
                    size_t cur_size = sg_dout[i].sg_len;
                    size_t rem = decomp_result - total_checked;
                    if (rem < cur_size) cur_size = rem;
                    total_checked += cur_size;

                    U64 crc_in  = XXH64(sg_cin [i].sg_base, cur_size, 1);
                    U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1);
                    if (crc_in != crc_out) {
                        locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size);
                        break;
                    }
                }
                DISPLAYLEVEL(1, "checked %i out of %i\n",
                        (int)total_checked, decomp_result);
            }
            CHECK(crcDecoded != crcOrig, "Decompression corruption");
        }
        else
        {
            // prepare compressedBuffer from SGL
            size_t total_copied = 0;
            for (i = 0; i < sg_cout_len; i++) {
                size_t buf_size_bytes = cSize - total_copied;
                if (buf_size_bytes == 0) break;
                if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len;
                if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) {
                    memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes);
                }
                total_copied += buf_size_bytes;
            }

            LZ4F_decompressOptions_t dOptions;
            memset(&dOptions, 0, sizeof(dOptions));

            const BYTE* ip = (const BYTE*)compressedBuffer;
            const BYTE* const iend = ip + cSize;
            BYTE* op = (BYTE*)decodedBuffer;
            BYTE* const oend = op + srcDataLength;
            size_t totalOut = 0;
            unsigned maxBits = FUZ_highbit((U32)cSize);
            XXH64_reset(&xxh64, 1);
            if (maxBits < 3) maxBits = 3;
            while (ip < iend)
            {
                unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
                unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
                size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
                size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
                if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                if (oSize > (size_t)(oend-op)) oSize = oend-op;
                dOptions.stableDst = FUZ_rand(&randState) & 1;
                result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
                if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
                    locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
                CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)",
                        (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer));
                XXH64_update(&xxh64, op, (U32)oSize);
                totalOut += oSize;
                op += oSize;
                ip += iSize;
            }
            CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
            if (totalOut)   /* otherwise, it's a skippable frame */
            {
                crcDecoded = XXH64_digest(&xxh64);
                if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
                CHECK(crcDecoded != crcOrig, "Decompression corruption");
            }
        }
    }

    DISPLAYLEVEL(2, "\rAll tests completed   \n");

_end:
    LZ4F_freeDecompressionContext(dCtx);
    LZ4F_freeCompressionContext(cCtx);
    free(srcBuffer);
    free(compressedBuffer);
    free(decodedBuffer);
    for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
        free((void *)(sg_in_buf_potential [i].sg_base));
        free(         sg_out_buf_potential[i].sg_base);
    }

    if (num_buf_size_distribution_deviations > 0) {
        DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations);
    }

    if (pause)
    {
        DISPLAY("press enter to finish \n");
        (void)getchar();
    }
    return testResult;

_output_error:
    testResult = 1;
    goto _end;

    // unreachable
    return -1;
#undef CHECK
}
コード例 #7
0
ファイル: sgtest.c プロジェクト: infidob/lz4
int basicTests(U32 seed, double compressibility)
{
    int testResult = 0;
    void* CNBuffer;
    void* compressedBuffer;
    void* decodedBuffer;
    U32 randState = seed;
    int cSize, testSize;
    LZ4F_preferences_t prefs;
    LZ4F_compressionContext_t cctx = NULL;
    U64 crcOrig;
    LZ4SG_in_t  sg_cin [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
    LZ4SG_in_t  sg_din [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
    size_t sg_in_len, sg_out_len;
    size_t maxDstSize = LZ4_SG_compressBound(COMPRESSIBLE_NOISE_LENGTH, NELEMS(sg_cin), NELEMS(sg_cout));
    size_t sourceSizeOut;

    /* Create compressible test buffer */
    memset(&prefs, 0, sizeof(prefs));
    CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
    compressedBuffer = malloc(maxDstSize);
    decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH);
    FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
    crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);

    /* Trivial tests : one input and one output buffers */
    testSize = COMPRESSIBLE_NOISE_LENGTH;
    DISPLAYLEVEL(3, "One input and one output buffers : \n");
    sg_cin[0].sg_base = CNBuffer;
    sg_cin[0].sg_len  = COMPRESSIBLE_NOISE_LENGTH;
    sg_in_len        = 1;
    sg_cout[0].sg_base = compressedBuffer;
    sg_cout[0].sg_len  = maxDstSize;
    sg_out_len        = 1;
    sourceSizeOut = testSize;
    cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
    DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
    UT_VERIFY(cSize > 0, goto _output_error);

    DISPLAYLEVEL(3, "Decompress test various valid sg_out and maxDstSize combinations \n");
    {
        U64 crcDest;
        // sg_out.sg_len == maxDstSize == originalSize
        sg_din[0].sg_base = compressedBuffer;
        sg_din[0].sg_len  = cSize;
        sg_dout[0].sg_base = decodedBuffer;
        sg_dout[0].sg_len  = COMPRESSIBLE_NOISE_LENGTH;
        sourceSizeOut = cSize;
        maxDstSize = testSize;
        memset(decodedBuffer, 0, testSize);
        testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
        crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
        DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
        UT_VERIFY(testResult >  0       , goto _output_error);
        UT_VERIFY(testResult == testSize, goto _output_error);
        UT_VERIFY(   crcDest == crcOrig , goto _output_error);

        // maxDstSize > originalSize
        maxDstSize = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH;
        sourceSizeOut = cSize;
        memset(decodedBuffer, 0, testSize);
        testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
        crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
        DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
        UT_VERIFY(testResult >  0       , goto _output_error);
        UT_VERIFY(testResult == testSize, goto _output_error);
        UT_VERIFY(   crcDest == crcOrig , goto _output_error);

        // sg_out.sg_len > originalSize
        maxDstSize = testSize;
        sg_dout[0].sg_len  = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH;
        sourceSizeOut = cSize;
        memset(decodedBuffer, 0, testSize);
        testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
        crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
        DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
        UT_VERIFY(testResult >  0       , goto _output_error);
        UT_VERIFY(testResult == testSize, goto _output_error);
        UT_VERIFY(   crcDest == crcOrig , goto _output_error);
    }

    DISPLAYLEVEL(3, "Frame Decompression test (%08x): \n", (unsigned int)crcOrig);
    {
        int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize);
        UT_VERIFY(0 == lz4f_result, goto _output_error);
    }

    // basic SGL test
    {
        // prepare SGL input
        const size_t num_data_buffers = 16;
        const size_t buf_size_bytes = 4 KB;
        unsigned int i;
        for (i = 0; i < 1+num_data_buffers; i++) {
            sg_cin [i].sg_base = malloc(buf_size_bytes);
            sg_cin [i].sg_len  = buf_size_bytes;
            sg_cout[i].sg_base = malloc(buf_size_bytes);
            sg_cout[i].sg_len  = buf_size_bytes;
            sg_din [i].sg_base = malloc(buf_size_bytes);
            sg_din [i].sg_len  = buf_size_bytes;
            sg_dout[i].sg_base = malloc(buf_size_bytes);
            sg_dout[i].sg_len  = buf_size_bytes;
        }
        for (i = 0; i < num_data_buffers; i++) {
            memcpy((void *)sg_cin [i].sg_base, ((const char *)CNBuffer)+(i*buf_size_bytes), buf_size_bytes);
        }
        sg_in_len  = num_data_buffers;
        sg_out_len = 1+num_data_buffers;
        testSize = num_data_buffers * buf_size_bytes;
        maxDstSize = (1+num_data_buffers) * buf_size_bytes;
        crcOrig = XXH64(CNBuffer, testSize, 1);

        DISPLAYLEVEL(3, "Compress 16 4KB buffers into 17 4KB buffers : \n");
        sourceSizeOut = testSize;
        cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
        DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
        UT_VERIFY(cSize > 0, goto _output_error);

        DISPLAYLEVEL(3, "Decompress 17 4KB buffers into 16 4KB buffers : \n");
        for (i = 0; i < 1+num_data_buffers; i++) {
            memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, buf_size_bytes);
        }
        sourceSizeOut = cSize;
        maxDstSize = testSize;
        sg_in_len  = 1+num_data_buffers;
        sg_out_len = num_data_buffers;
        memset(decodedBuffer, 0, testSize);
        testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
        DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
        for (i = 0; i < num_data_buffers; i++) {
            memcpy(((char *)decodedBuffer)+(i*buf_size_bytes), sg_dout[i].sg_base, buf_size_bytes);
        }
        UT_VERIFY(testResult >  0       , goto _output_error);
        UT_VERIFY(testResult == testSize, goto _output_error);

        U64 crcDest;
        crcDest = XXH64(decodedBuffer, testSize, 1);
        UT_VERIFY(   crcDest == crcOrig , goto _output_error);

        DISPLAYLEVEL(3, "verify frame decompress on concatenated buffer: \n");
        for (i = 0; i < 1+num_data_buffers; i++) {
            memcpy(((char *)compressedBuffer)+(i*buf_size_bytes), sg_cout[i].sg_base, buf_size_bytes);
        }
        int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize);
        UT_VERIFY(0 == lz4f_result, goto _output_error);

        // release SGL
        for (i = 0; i < 1+num_data_buffers; i++) {
            free((void *)sg_cin [i].sg_base); sg_cin [i].sg_base = NULL; sg_cin [i].sg_len = 0;
            free(sg_cout[i].sg_base); sg_cout[i].sg_base = NULL; sg_cout[i].sg_len = 0;
            free((void *)sg_din [i].sg_base); sg_din [i].sg_base = NULL; sg_din [i].sg_len = 0;
            free(sg_dout[i].sg_base); sg_dout[i].sg_base = NULL; sg_dout[i].sg_len = 0;
        }
    }

    DISPLAY("Basic tests completed \n");
    testResult = 0;
_end:
    free(CNBuffer);
    free(compressedBuffer);
    free(decodedBuffer);
    LZ4F_freeCompressionContext(cctx); cctx = NULL;
    return testResult;

_output_error:
    testResult = 1;
    DISPLAY("Error detected ! \n");
    locateBuffDiff(CNBuffer, decodedBuffer, testSize);
    goto _end;

    // unreachable
    return -1;
}
コード例 #8
0
void Message::compress(Message::Compression type, int level) {
	if (isCompressed())
		return;

	switch (type) {
#ifdef BUILD_WITH_COMPRESSION_MINIZ
	case COMPRESS_MINIZ: {
		mz_ulong compressedSize = mz_compressBound(_size);
		int cmp_status;
		uint8_t *pCmp;

		pCmp = (mz_uint8 *)malloc((size_t)compressedSize);
		level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_MINIZ);

		// last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION
		cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, level);
		if (cmp_status != Z_OK) {
			// error
			free(pCmp);
		}

		_data = SharedPtr<char>((char*)pCmp);
		_meta["um.compressed"] = toStr(_size) + ":miniz";
		_size = compressedSize;
	}
	return;
#endif
#ifdef BUILD_WITH_COMPRESSION_FASTLZ
	case COMPRESS_FASTLZ: {
		// The minimum input buffer size is 16.
		if (_size < 16)
			return;

		// The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes.
		int compressedSize = _size + (double)_size * 0.06;
		if (compressedSize < 66)
			compressedSize = 66;

		char* compressedData = (char*)malloc(compressedSize);
		compressedSize = fastlz_compress(_data.get(), _size, compressedData);

		// If the input is not compressible, the return value might be larger than length
		if (compressedSize > _size) {
			free(compressedData);
			return;
		}

		//	std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl;

		_data = SharedPtr<char>((char*)compressedData);
		_meta["um.compressed"] = toStr(_size) + ":fastlz";
		_size = compressedSize;
	}
	return;
#endif
#ifdef BUILD_WITH_COMPRESSION_LZ4
	case COMPRESS_LZ4: {
		level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_LZ4);

#ifdef LZ4_FRAME
		LZ4F_preferences_t lz4_preferences = {
			{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } },
			level,   /* compression level */
			0,   /* autoflush */
			{ 0, 0, 0, 0 },  /* reserved, must be set to 0 */
		};

		LZ4F_errorCode_t r;
		LZ4F_compressionContext_t ctx;

		r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
		if (LZ4F_isError(r)) {
			//        printf("Failed to create context: error %zu", r);
			return;
		}

#define LZ4_HEADER_SIZE 19
#define LZ4_FOOTER_SIZE 4
		size_t n, offset = 0;


		size_t frameSize = LZ4F_compressBound(_size, &lz4_preferences);
		size_t compressedSize = frameSize + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
		char* compressedData = (char*)malloc(compressedSize);

		n = LZ4F_compressBegin(ctx, compressedData, compressedSize, &lz4_preferences);
		if (LZ4F_isError(n)) {
			//        printf("Failed to start compression: error %zu", n);
			LZ4F_freeCompressionContext(ctx);
			free(compressedData);
			return;
		}
		offset += n;

		n = LZ4F_compressUpdate(ctx, compressedData + offset, compressedSize - offset, _data.get(), _size, NULL);
		if (LZ4F_isError(n)) {
			//        printf("Compression failed: error %zu", n);
			LZ4F_freeCompressionContext(ctx);
			free(compressedData);
			return;
		}
		offset += n;

		n = LZ4F_compressEnd(ctx, compressedData + offset, compressedSize - offset, NULL);
		if (LZ4F_isError(n)) {
			//        printf("Failed to end compression: error %zu", n);
			LZ4F_freeCompressionContext(ctx);
			free(compressedData);
			return;
		}
		offset += n;

		_data = SharedPtr<char>((char*)compressedData);
		_meta["um.compressed"] = toStr(_size) + ":lz4";
		_size = offset;

		LZ4F_freeCompressionContext(ctx);
#else
		size_t compressedSize = LZ4_compressBound(_size);
		char* compressedData = (char*)malloc(compressedSize);
		int actualSize = 0;

		actualSize = LZ4_compress_fast(_data.get(), compressedData, _size, compressedSize, level);
		if (actualSize == 0) {
			free(compressedData);
			return;
		}

		_data = SharedPtr<char>((char*)compressedData);
		_meta["um.compressed"] = toStr(_size) + ":lz4";
		_size = actualSize;
#endif

	}
	return;
#endif
	default:
		break;
	}
}