/* Decompress and return a chunk that is part of a super-chunk. */
int blosc2_decompress_chunk(blosc2_sheader* sheader, int64_t nchunk,
void* dest, int nbytes) {
int64_t nchunks = sheader->nchunks;
void* src;
int chunksize;
int nbytes_;
uint8_t* filters = decode_filters(sheader->filters);
if (nchunk >= nchunks) {
printf("specified nchunk ('%ld') exceeds the number of chunks "
"('%ld') in super-chunk\n", (long)nchunk, (long)nchunks);
return -10;
}
/* Grab the address of the chunk */
src = sheader->data[nchunk];
/* Create a buffer for destination */
nbytes_ = *(int32_t*)((uint8_t*)src + 4);
if (nbytes < nbytes_) {
printf("Buffer size is too small for the decompressed buffer ('%d' bytes, but '%d' are needed)\n", nbytes, nbytes_);
return -11;
}
/* Put the super-chunk address in the global context for Blosc1 */
blosc_set_schunk(sheader);
/* And decompress the chunk */
chunksize = blosc_decompress(src, dest, (size_t)nbytes);
free(filters);
return chunksize;
}
/**
* \brief Uncompress compressed data \p m_cdata to \p data m_rdata.
*
* \p m_rdata must point to writable storage space and
* \p m_rdata_size must specify the legal space.
*
* \returns true on success
*/
bool bloscDecompress()
{
assert( m_rdata && m_cdata );
size_t blosc_nbytes, blosc_cbytes, blosc_blocksize;
// calculate necessary buffer sizes
blosc_cbuffer_sizes( m_cdata, &blosc_nbytes, &blosc_cbytes, &blosc_blocksize );
// uncompressed data must fit into
if( blosc_nbytes != m_rdata_size )
{
m_err.set( MSG_ERRCOMPRESSION );
return false;
}
// decompress directly into items memory space
if( blosc_decompress( m_cdata, m_rdata, m_rdata_size ) <= 0 )
{
m_err.set( MSG_ERRCOMPRESSION );
return false;
}
return true;
}
/* Check compressing + decompressing */
static const char *test_compress_decompress(void) {
const char* compressor;
/* Activate the BLOSC_COMPRESSOR variable */
setenv("BLOSC_COMPRESSOR", "lz4", 0);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Reset envvar */
unsetenv("BLOSC_COMPRESSOR");
return 0;
}
int main(int argc, char **argv) {
printf("STARTING TESTS for %s", argv[0]);
blosc_set_nthreads(1);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest = malloc(size);
dest2 = malloc(size);
memset(src, 0, size);
memcpy(srccpy, src, size);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size);
/* Get a decompressed buffer */
nbytes = blosc_decompress(dest, dest2, size);
/* Run all the suite */
const char *result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED");
}
printf("\tTests run: %d\n", tests_run);
free(src); free(srccpy); free(dest); free(dest2);
return result != 0;
}
int main() {
static float data[SIZE];
static float data_out[SIZE];
static float data_dest[SIZE];
int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float);
int dsize = SIZE * sizeof(float), csize;
int nthreads, pnthreads, i;
for (i = 0; i < SIZE; i++) {
data[i] = i;
}
/* Register the filter with the library */
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Tell Blosc to use some number of threads */
for (nthreads = 1; nthreads <= 4; nthreads++) {
pnthreads = blosc_set_nthreads(nthreads);
printf("Using %d threads (previously using %d)\n", nthreads, pnthreads);
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize);
/* Decompress */
dsize = blosc_decompress(data_out, data_dest, dsize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
for (i = 0; i < SIZE; i++) {
if (data[i] != data_dest[i]) {
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
}
/* After using it, destroy the Blosc environment */
blosc_destroy();
return 0;
}
int main(){
static float data[SIZE];
static float data_out[SIZE];
static float data_dest[SIZE];
int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float);
int dsize = SIZE*sizeof(float), csize;
int i;
for(i=0; i<SIZE; i++){
data[i] = i;
}
/* Register the filter with the library */
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
if (csize == 0) {
printf("Buffer is uncompressible. Giving up.\n");
return 1;
}
else if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize);
/* Decompress */
dsize = blosc_decompress(data_out, data_dest, dsize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
printf("Decompression succesful!\n");
/* After using it, destroy the Blosc environment */
blosc_destroy();
for(i=0;i<SIZE;i++){
if(data[i] != data_dest[i]) {
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
return 0;
}
/* Check maxout with maxout > size (memcpy version) */
static const char *test_maxout_great_memcpy(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
return 0;
}
/* Check for decompressing into a buffer larger than necessary (v2) */
static char *test_too_long_dest2() {
int cbytes1;
int cbytes2;
int srclen = 3069;
cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 <= srclen + BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, srclen + 1021);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen);
return 0;
}
/* Check for compressing an empty buffer */
static char *test_empty_buffer() {
int cbytes1;
int cbytes2;
cbytes1 = blosc_compress(1, 1, 1, 0, src, dest, BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 == BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, 0);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == 0);
return 0;
}
/* Check maxout with maxout > size */
static char* test_maxout_great() {
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + 17);
mu_assert("ERROR: cbytes is not 0", cbytes == size + 16);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
return 0;
}
/* Check for compressing a very small buffer */
static char *test_small_buffer() {
int cbytes1;
int cbytes2;
int srclen;
for (srclen = 1; srclen < BLOSC_MAX_OVERHEAD; srclen++) {
cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 == srclen + BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, srclen);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen);
}
return 0;
}
/* Decompress nbytes from input into output.
*
* Returns 1 on success and 0 on failure with a Python exception set.
*
* */
static int
decompress_helper(void * input, size_t nbytes, void * output)
{
int err;
/* Do the decompression */
err = blosc_decompress(input, output, nbytes);
if (err < 0 || err != (int)nbytes) {
blosc_error(err, "while decompressing data");
return 0;
}
return 1;
}
/* Decompress nbytes from input into output.
*
* Returns 1 on success and 0 on failure with a Python exception set.
*
* */
static int
decompress_helper(void * input, size_t nbytes, void * output)
{
int err, nthreads;
PyThreadState *_save = NULL;
/* Do the decompression */
// int blosc_decompress_ctx(const void *src, void *dest, size_t destsize,
// int numinternalthreads)
if( RELEASEGIL )
{
_save = PyEval_SaveThread();
nthreads = blosc_get_nthreads();
err = blosc_decompress_ctx(input, output, nbytes, nthreads);
PyEval_RestoreThread(_save);
_save = NULL;
}
else
{ // Run while holding the GIL
err = blosc_decompress(input, output, nbytes);
}
if (err < 0) {
blosc_error(err, "while decompressing data");
return 0;
}
else if (err != (int)nbytes) {
PyErr_Format(BloscError,
"expected %d bytes of decompressed data, got %d",
(int) nbytes,
err);
return 0;
}
return 1;
}
/* Decompress and return a chunk that is part of a *packed* super-chunk. */
int blosc2_packed_decompress_chunk(void* packed, int nchunk, void** dest) {
int64_t nchunks = *(int64_t*)((uint8_t*)packed + 16);
uint8_t* filters = decode_filters(*(uint16_t*)((uint8_t*)packed + 8));
uint8_t* filters_chunk = (uint8_t*)packed + *(uint64_t*)((uint8_t*)packed + 40);
int64_t* data = (int64_t*)((uint8_t*)packed + *(int64_t*)((uint8_t*)packed + 72));
void* src;
int chunksize;
int32_t nbytes;
if (nchunk >= nchunks) {
return -10;
}
/* Grab the address of the chunk */
src = (uint8_t*)packed + data[nchunk];
/* Create a buffer for destination */
nbytes = *(int32_t*)((uint8_t*)src + 4);
*dest = malloc((size_t)nbytes);
/* And decompress it */
chunksize = blosc_decompress(src, *dest, (size_t)nbytes);
if (chunksize < 0) {
return chunksize;
}
if (chunksize != nbytes) {
return -11;
}
/* Apply filters after de-compress */
if (filters[0] == BLOSC_DELTA) {
delta_decoder8(filters_chunk, 0, nbytes, *dest);
}
free(filters);
return chunksize;
}
void do_bench(int nthreads, int size, int elsize, int rshift, FILE * ofile) {
void *src, *srccpy;
void **dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j;
struct timeval last, current;
float tmemcpy, tshuf, tunshuf;
int clevel, doshuffle=1;
unsigned char *orig, *round;
blosc_set_nthreads(nthreads);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest2 = malloc(size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
dest[j] = malloc(size+BLOSC_MAX_OVERHEAD);
}
/* Warm destination memory (memcpy() will go a bit faster later on) */
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
fprintf(ofile, "--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
gettimeofday(¤t, NULL);
tshuf = get_usec_chunk(last, current);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, size/(tshuf*MB/1e6));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
gettimeofday(¤t, NULL);
tunshuf = get_usec_chunk(last, current);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, nbytes/(tunshuf*MB/1e6));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
free(src); free(srccpy); free(dest2);
for (i = 0; i < nchunks; i++) {
free(dest[i]);
}
}
void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize,
int rshift, FILE * ofile) {
void *src, *srccpy;
void *dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j, retcode;
unsigned char *orig, *round;
blosc_timestamp_t last, current;
double tmemcpy, tshuf, tunshuf;
int clevel, doshuffle;
if (strcmp(shuffle, "shuffle") == 0) {
doshuffle = BLOSC_SHUFFLE;
}
else if (strcmp(shuffle, "bitshuffle") == 0) {
doshuffle = BLOSC_BITSHUFFLE;
}
else if (strcmp(shuffle, "noshuffle") == 0) {
doshuffle = BLOSC_NOSHUFFLE;
}
blosc_set_nthreads(nthreads);
if(blosc_set_compressor(compressor) < 0){
printf("Compiled w/o support for compressor: '%s', so sorry.\n",
compressor);
exit(1);
}
/* Initialize buffers */
srccpy = malloc(size);
retcode = posix_memalign( (void **)(&src), 32, size);
retcode = posix_memalign( (void **)(&dest2), 32, size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD);
}
fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
blosc_set_timestamp(¤t);
tshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, (size * 1e6) / (tshuf*MB));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
blosc_set_timestamp(¤t);
tunshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, (nbytes * 1e6) / (tunshuf*MB));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip.
Byte-by-byte comparison is slow, so use 'memcmp' to check whether the
roundtripped data is correct. If not, fall back to the slow path to
print diagnostic messages. */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
if (memcmp(orig, round, size) != 0)
{
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
}
else { i = size; }
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
aligned_free(src); free(srccpy); aligned_free(dest2);
for (i = 0; i < nchunks; i++) {
aligned_free(dest[i]);
}
}
// decompress a buffer
size_t
BloscWrapper::decompress(void* src, void* dst, size_t dstsize) {
return blosc_decompress(src, dst, dstsize);
}
