static PyObject * compress_helper(void * input, size_t nbytes, size_t typesize, int clevel, int shuffle, char *cname){ int cbytes; PyObject *output = NULL; /* Alloc memory for compression */ if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD))) return NULL; /* Select compressor */ if (blosc_set_compressor(cname) < 0) { /* The compressor is not available (should never happen here) */ blosc_error(-1, "compressor not available"); return NULL; } /* Compress */ cbytes = blosc_compress(clevel, shuffle, typesize, nbytes, input, PyBytes_AS_STRING(output), nbytes+BLOSC_MAX_OVERHEAD); if (cbytes < 0) { blosc_error(cbytes, "while compressing data"); Py_XDECREF(output); return NULL; } /* Attempt to resize, if it's much smaller, a copy is required. */ if (_PyBytes_Resize(&output, cbytes) < 0){ /* the memory exception will have been set, hopefully */ return NULL; } return output; }
int main(int argc, char **argv) { printf("STARTING TESTS for %s", argv[0]); blosc_set_nthreads(1); /* Initialize buffers */ src = malloc(size); srccpy = malloc(size); dest = malloc(size); dest2 = malloc(size); memset(src, 0, size); memcpy(srccpy, src, size); /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size); /* Get a decompressed buffer */ nbytes = blosc_decompress(dest, dest2, size); /* Run all the suite */ const char *result = all_tests(); if (result != 0) { printf(" (%s)\n", result); } else { printf(" ALL TESTS PASSED"); } printf("\tTests run: %d\n", tests_run); free(src); free(srccpy); free(dest); free(dest2); return result != 0; }
/** * \brief Allocates memory for compressed data and use it to store results (lossless data compression) * * \returns true on success */ bool bloscCompress() { assert( m_rdata && !m_cdata ); // BLOSC grants for that compressed data never // exceeds original size + BLOSC_MAX_OVERHEAD m_cdata_size = m_rdata_size + BLOSC_MAX_OVERHEAD; m_cdata = m_Allocator( m_cdata_size ); if( NULL == m_cdata ) { m_err.set( MSG_ERRMEMORY ); return false; } /* compress raw data (rdata) and store it in cdata */ m_cdata_size = blosc_compress( /*clevel*/ m_iCompressionLevel, /*doshuffle*/ BLOSC_DOSHUFFLE, /*typesize*/ m_rdata_element_size, /*nbytes*/ m_rdata_size, /*src*/ m_rdata, /*dest*/ m_cdata, /*destsize*/ m_cdata_size ); return NULL != m_cdata; }
/* Set a delta reference for the super-chunk */ int blosc2_set_delta_ref(blosc2_sheader* sheader, size_t nbytes, void* ref) { int cbytes; void* filters_chunk; uint8_t* dec_filters = decode_filters(sheader->filters); if (dec_filters[0] == BLOSC_DELTA) { if (sheader->filters_chunk != NULL) { sheader->cbytes -= *(uint32_t*)(sheader->filters_chunk + 4); free(sheader->filters_chunk); } } else { printf("You cannot set a delta reference if delta filter is not set\n"); return(-1); } free(dec_filters); filters_chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD); cbytes = blosc_compress(0, 0, 1, nbytes, ref, filters_chunk, nbytes + BLOSC_MAX_OVERHEAD); if (cbytes < 0) { free(filters_chunk); return cbytes; } sheader->filters_chunk = filters_chunk; sheader->cbytes += cbytes; return cbytes; }
/* Check splitmode with an environment variable */ static char *test_splitmode_envvar() { int cbytes2; /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); /* Deactivate the split */ setenv("BLOSC_SPLITMODE", "NEVER", 0); cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: BLOSC_SPLITMODE envvar does not work correctly", cbytes2 > cbytes); return 0; }
/** Test the blosc_getitem function. */ static int test_getitem(size_t type_size, size_t num_elements, size_t buffer_alignment, int compression_level, int do_shuffle) { size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* intermediate = blosc_test_malloc(buffer_alignment, buffer_size + BLOSC_MAX_OVERHEAD); void* result = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Compress the input data, then use blosc_getitem to extract (decompress) a range of elements into a new buffer. */ blosc_compress(compression_level, do_shuffle, type_size, buffer_size, original, intermediate, buffer_size + BLOSC_MAX_OVERHEAD); blosc_getitem(intermediate, 0, num_elements, result); /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, result, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(intermediate); blosc_test_free(result); return exit_code; }
/* Check compressing + decompressing */ static const char *test_compress_decompress(void) { const char* compressor; /* Activate the BLOSC_COMPRESSOR variable */ setenv("BLOSC_COMPRESSOR", "lz4", 0); compressor = blosc_get_compressor(); mu_assert("ERROR: get_compressor incorrect", strcmp(compressor, "lz4") == 0); /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); compressor = blosc_get_compressor(); mu_assert("ERROR: get_compressor incorrect", strcmp(compressor, "lz4") == 0); /* Decompress the buffer */ nbytes = blosc_decompress(dest, dest2, size); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); compressor = blosc_get_compressor(); mu_assert("ERROR: get_compressor incorrect", strcmp(compressor, "lz4") == 0); /* Reset envvar */ unsetenv("BLOSC_COMPRESSOR"); return 0; }
/* Check maxout with maxout < size (memcpy version) */ static const char *test_maxout_less_memcpy(void) { /* Get a compressed buffer */ cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1); mu_assert("ERROR: cbytes is not 0", cbytes == 0); return 0; }
/* Check maxout with maxout < BLOSC_MAX_OVERHEAD */ static const char *test_max_overhead(void) { blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, 0); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); return 0; }
/* Check splitmode */ static char *test_splitmode() { int cbytes2; /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); /* Deactivate the split */ blosc_set_splitmode(BLOSC_NEVER_SPLIT); cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: blosc_set_splitmode does not work correctly", cbytes2 > cbytes); /* Reset the splitmode */ blosc_set_splitmode(BLOSC_FORWARD_COMPAT_SPLIT); return 0; }
/* Check maxout with maxout < size */ static char* test_maxout_less() { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + 15); mu_assert("ERROR: cbytes is not 0", cbytes == 0); return 0; }
/* Check just compressing */ static const char *test_compress(void) { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); return 0; }
/* Check compression level */ static const char *test_clevel(void) { int cbytes2; /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); /* Activate the BLOSC_CLEVEL variable */ setenv("BLOSC_CLEVEL", "9", 0); cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: BLOSC_CLEVEL does not work correctly", cbytes2 < cbytes); /* Reset envvar */ unsetenv("BLOSC_CLEVEL"); return 0; }
int main() { static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float); int dsize = SIZE * sizeof(float), csize; int nthreads, pnthreads, i; for (i = 0; i < SIZE; i++) { data[i] = i; } /* Register the filter with the library */ printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Tell Blosc to use some number of threads */ for (nthreads = 1; nthreads <= 4; nthreads++) { pnthreads = blosc_set_nthreads(nthreads); printf("Using %d threads (previously using %d)\n", nthreads, pnthreads); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize); if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } for (i = 0; i < SIZE; i++) { if (data[i] != data_dest[i]) { printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); } /* After using it, destroy the Blosc environment */ blosc_destroy(); return 0; }
/* Check maxout with maxout > size (memcpy version) */ static const char *test_maxout_great_memcpy(void) { /* Get a compressed buffer */ cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1); mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD); /* Decompress the buffer */ nbytes = blosc_decompress(dest, dest2, size); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); return 0; }
int main(){ static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int i; for(i=0; i<SIZE; i++){ data[i] = i; } /* Register the filter with the library */ printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize); if (csize == 0) { printf("Buffer is uncompressible. Giving up.\n"); return 1; } else if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); /* After using it, destroy the Blosc environment */ blosc_destroy(); for(i=0;i<SIZE;i++){ if(data[i] != data_dest[i]) { printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); return 0; }
/* Check bitshuffle */ static const char *test_bitshuffle(void) { int cbytes2; /* Get a compressed buffer */ blosc_set_compressor("blosclz"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not 0", cbytes < size); /* Activate the BLOSC_BITSHUFFLE variable */ setenv("BLOSC_SHUFFLE", "BITSHUFFLE", 0); cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: BLOSC_SHUFFLE=BITSHUFFLE does not work correctly", cbytes2 < cbytes * 1.5); /* Reset env var */ unsetenv("BLOSC_SHUFFLE"); return 0; }
/* Check maxout with maxout > size */ static char* test_maxout_great() { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + 17); mu_assert("ERROR: cbytes is not 0", cbytes == size + 16); /* Decompress the buffer */ nbytes = blosc_decompress(dest, dest2, size); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); return 0; }
/* Check for compressing an empty buffer */ static char *test_empty_buffer() { int cbytes1; int cbytes2; cbytes1 = blosc_compress(1, 1, 1, 0, src, dest, BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes1 == BLOSC_MAX_OVERHEAD); cbytes2 = blosc_decompress(dest, src, 0); mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == 0); return 0; }
/* Check for decompressing into a buffer larger than necessary (v2) */ static char *test_too_long_dest2() { int cbytes1; int cbytes2; int srclen = 3069; cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes1 <= srclen + BLOSC_MAX_OVERHEAD); cbytes2 = blosc_decompress(dest, src, srclen + 1021); mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen); return 0; }
/* Check for compressing a very small buffer */ static char *test_small_buffer() { int cbytes1; int cbytes2; int srclen; for (srclen = 1; srclen < BLOSC_MAX_OVERHEAD; srclen++) { cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes1 == srclen + BLOSC_MAX_OVERHEAD); cbytes2 = blosc_decompress(dest, src, srclen); mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen); } return 0; }
/* Append a data buffer to a *packed* super-chunk. */ void* blosc2_packed_append_buffer(void* packed, size_t typesize, size_t nbytes, void* src) { int cname = *(int16_t*)((uint8_t*)packed + 4); int clevel = *(int16_t*)((uint8_t*)packed + 6); void* filters_chunk = (uint8_t*)packed + *(uint64_t*)((uint8_t*)packed + 40); uint8_t* filters = decode_filters(*(uint16_t*)((uint8_t*)packed + 8)); int cbytes; void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD); void* dest = malloc(nbytes); char* compname; int doshuffle; void* new_packed; /* Apply filters prior to compress */ if (filters[0] == BLOSC_DELTA) { doshuffle = filters[1]; if (filters_chunk == NULL) { /* For packed super-buffers, the filters schunk should exist */ return NULL; } delta_encoder8(filters_chunk, 0, (int)nbytes, src, dest); /* memcpy(dest, src, nbytes); */ src = dest; } else { doshuffle = filters[0]; } /* Compress the src buffer using super-chunk defaults */ blosc_compcode_to_compname(cname, &compname); blosc_set_compressor(compname); cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk, nbytes + BLOSC_MAX_OVERHEAD); if (cbytes < 0) { free(chunk); free(dest); free(filters); return NULL; } /* We don't need dest and filters anymore */ free(dest); free(filters); /* Append the chunk and free it */ new_packed = packed_append_chunk(packed, chunk); free(chunk); return new_packed; }
// compress a buffer size_t BloscWrapper::compress(void* src, size_t srcsize, void* dst, size_t dstsize, int clevel, bool doshuffle, size_t typesize) { // convert doshuffle to int int doshuffle_int = 0; if (doshuffle) { doshuffle_int = 1; } // perform compression return blosc_compress(clevel, doshuffle_int, typesize, srcsize, src, dst, dstsize); }
/* Append a data buffer to a super-chunk. */ size_t blosc2_append_buffer(blosc2_sheader* sheader, size_t typesize, size_t nbytes, void* src) { int cbytes; void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD); uint8_t* dec_filters = decode_filters(sheader->filters); int clevel = sheader->clevel; char* compname; int doshuffle, ret; /* Apply filters prior to compress */ if (dec_filters[0] == BLOSC_DELTA) { doshuffle = dec_filters[1]; if (sheader->filters_chunk == NULL) { ret = blosc2_set_delta_ref(sheader, nbytes, src); if (ret < 0) { return((size_t)ret); } } } else { doshuffle = dec_filters[0]; } free(dec_filters); /* Compress the src buffer using super-chunk defaults */ blosc_compcode_to_compname(sheader->compressor, &compname); blosc_set_compressor(compname); blosc_set_schunk(sheader); cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk, nbytes + BLOSC_MAX_OVERHEAD); if (cbytes < 0) { free(chunk); return cbytes; } /* Append the chunk */ return append_chunk(sheader, chunk); }
/* Check compressor */ static const char *test_compressor(void) { const char* compressor; /* Before any blosc_compress() the compressor must be blosclz */ compressor = blosc_get_compressor(); mu_assert("ERROR: get_compressor (compress, before) incorrect", strcmp(compressor, "blosclz") == 0); /* Activate the BLOSC_COMPRESSOR variable */ setenv("BLOSC_COMPRESSOR", "lz4", 0); /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes < size); compressor = blosc_get_compressor(); mu_assert("ERROR: get_compressor (compress, after) incorrect", strcmp(compressor, "lz4") == 0); /* Reset envvar */ unsetenv("BLOSC_COMPRESSOR"); return 0; }
void do_bench(int nthreads, int size, int elsize, int rshift, FILE * ofile) { void *src, *srccpy; void **dest[NCHUNKS], *dest2; int nbytes = 0, cbytes = 0; int i, j; struct timeval last, current; float tmemcpy, tshuf, tunshuf; int clevel, doshuffle=1; unsigned char *orig, *round; blosc_set_nthreads(nthreads); /* Initialize buffers */ src = malloc(size); srccpy = malloc(size); dest2 = malloc(size); /* zero src to initialize byte on it, and not only multiples of 4 */ memset(src, 0, size); init_buffer(src, size, rshift); memcpy(srccpy, src, size); for (j = 0; j < nchunks; j++) { dest[j] = malloc(size+BLOSC_MAX_OVERHEAD); } /* Warm destination memory (memcpy() will go a bit faster later on) */ for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } fprintf(ofile, "--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift); fprintf(ofile, "********************** Run info ******************************\n"); fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift); fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize); fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB); fprintf(ofile, "Number of threads: %d\n", nthreads); fprintf(ofile, "********************** Running benchmarks *********************\n"); gettimeofday(&last, NULL); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } gettimeofday(¤t, NULL); tmemcpy = get_usec_chunk(last, current); fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, size/(tmemcpy*MB/1e6)); gettimeofday(&last, NULL); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest2, dest[j], size); } } gettimeofday(¤t, NULL); tmemcpy = get_usec_chunk(last, current); fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, size/(tmemcpy*MB/1e6)); for (clevel=0; clevel<10; clevel++) { fprintf(ofile, "Compression level: %d\n", clevel); gettimeofday(&last, NULL); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, dest[j], size+BLOSC_MAX_OVERHEAD); } } gettimeofday(¤t, NULL); tshuf = get_usec_chunk(last, current); fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ", tshuf, size/(tshuf*MB/1e6)); fprintf(ofile, "Final bytes: %d ", cbytes); if (cbytes > 0) { fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes); } fprintf(ofile, "\n"); /* Compressor was unable to compress. Copy the buffer manually. */ if (cbytes == 0) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } gettimeofday(&last, NULL); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { if (cbytes == 0) { memcpy(dest2, dest[j], size); nbytes = size; } else { nbytes = blosc_decompress(dest[j], dest2, size); } } } gettimeofday(¤t, NULL); tunshuf = get_usec_chunk(last, current); fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ", tunshuf, nbytes/(tunshuf*MB/1e6)); if (nbytes < 0) { fprintf(ofile, "FAILED. Error code: %d\n", nbytes); } /* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */ /* Check if data has had a good roundtrip */ orig = (unsigned char *)srccpy; round = (unsigned char *)dest2; for(i = 0; i<size; ++i){ if (orig[i] != round[i]) { fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n", (int)i); fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]); break; } } if (i == size) fprintf(ofile, "OK\n"); } /* End clevel loop */ /* To compute the totalsize, we should take into account the 10 compression levels */ totalsize += (size * nchunks * niter * 10.); free(src); free(srccpy); free(dest2); for (i = 0; i < nchunks; i++) { free(dest[i]); } }
void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize, int rshift, FILE * ofile) { void *src, *srccpy; void *dest[NCHUNKS], *dest2; int nbytes = 0, cbytes = 0; int i, j, retcode; unsigned char *orig, *round; blosc_timestamp_t last, current; double tmemcpy, tshuf, tunshuf; int clevel, doshuffle; if (strcmp(shuffle, "shuffle") == 0) { doshuffle = BLOSC_SHUFFLE; } else if (strcmp(shuffle, "bitshuffle") == 0) { doshuffle = BLOSC_BITSHUFFLE; } else if (strcmp(shuffle, "noshuffle") == 0) { doshuffle = BLOSC_NOSHUFFLE; } blosc_set_nthreads(nthreads); if(blosc_set_compressor(compressor) < 0){ printf("Compiled w/o support for compressor: '%s', so sorry.\n", compressor); exit(1); } /* Initialize buffers */ srccpy = malloc(size); retcode = posix_memalign( (void **)(&src), 32, size); retcode = posix_memalign( (void **)(&dest2), 32, size); /* zero src to initialize byte on it, and not only multiples of 4 */ memset(src, 0, size); init_buffer(src, size, rshift); memcpy(srccpy, src, size); for (j = 0; j < nchunks; j++) { retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD); } fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle); fprintf(ofile, "********************** Run info ******************************\n"); fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift); fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize); fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB); fprintf(ofile, "Number of threads: %d\n", nthreads); fprintf(ofile, "********************** Running benchmarks *********************\n"); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } blosc_set_timestamp(¤t); tmemcpy = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, (size * 1e6) / (tmemcpy*MB)); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest2, dest[j], size); } } blosc_set_timestamp(¤t); tmemcpy = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, (size * 1e6) / (tmemcpy*MB)); for (clevel=0; clevel<10; clevel++) { fprintf(ofile, "Compression level: %d\n", clevel); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, dest[j], size+BLOSC_MAX_OVERHEAD); } } blosc_set_timestamp(¤t); tshuf = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ", tshuf, (size * 1e6) / (tshuf*MB)); fprintf(ofile, "Final bytes: %d ", cbytes); if (cbytes > 0) { fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes); } fprintf(ofile, "\n"); /* Compressor was unable to compress. Copy the buffer manually. */ if (cbytes == 0) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { if (cbytes == 0) { memcpy(dest2, dest[j], size); nbytes = size; } else { nbytes = blosc_decompress(dest[j], dest2, size); } } } blosc_set_timestamp(¤t); tunshuf = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ", tunshuf, (nbytes * 1e6) / (tunshuf*MB)); if (nbytes < 0) { fprintf(ofile, "FAILED. Error code: %d\n", nbytes); } /* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */ /* Check if data has had a good roundtrip. Byte-by-byte comparison is slow, so use 'memcmp' to check whether the roundtripped data is correct. If not, fall back to the slow path to print diagnostic messages. */ orig = (unsigned char *)srccpy; round = (unsigned char *)dest2; if (memcmp(orig, round, size) != 0) { for(i = 0; i<size; ++i){ if (orig[i] != round[i]) { fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n", (int)i); fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]); break; } } } else { i = size; } if (i == size) fprintf(ofile, "OK\n"); } /* End clevel loop */ /* To compute the totalsize, we should take into account the 10 compression levels */ totalsize += (size * nchunks * niter * 10.); aligned_free(src); free(srccpy); aligned_free(dest2); for (i = 0; i < nchunks; i++) { aligned_free(dest[i]); } }
int main() { static float data[SIZE]; static float data_dest[SIZE]; int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float); int dsize, csize; blosc2_sparams sparams = BLOSC_SPARAMS_DEFAULTS; blosc2_sheader* sheader; int i, nchunks; for (i = 0; i < SIZE; i++) { data[i] = i; } printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, BLOSC_SHUFFLE, sizeof(float), isize, data, data_dest, osize); if (csize == 0) { printf("Buffer is uncompressible. Giving up.\n"); return 1; } else if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize); /* Create a super-chunk container */ sparams.filters[0] = BLOSC_DELTA; sparams.filters[1] = BLOSC_SHUFFLE; sheader = blosc2_new_schunk(&sparams); /* Now append a couple of chunks */ nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data); assert(nchunks == 1); nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data); assert(nchunks == 2); /* Retrieve and decompress the chunks (0-based count) */ dsize = blosc2_decompress_chunk(sheader, 0, (void*)data_dest, isize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } dsize = blosc2_decompress_chunk(sheader, 1, (void*)data_dest, isize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); for (i = 0; i < SIZE; i++) { if (data[i] != data_dest[i]) { printf("i, values: %d, %f, %f\n", i, data[i], data_dest[i]); printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); /* Free resources */ /* Destroy the super-chunk */ blosc2_destroy_schunk(sheader); /* Destroy the Blosc environment */ blosc_destroy(); return 0; }
static PyObject * compress_helper(void * input, size_t nbytes, size_t typesize, int clevel, int shuffle, char *cname){ int cbytes, blocksize, nthreads; PyObject *output; char *output_ptr; PyThreadState *_save = NULL; /* Alloc memory for compression */ if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD))) return NULL; /* Select compressor */ if (blosc_set_compressor(cname) < 0) { /* The compressor is not available (should never happen here) */ blosc_error(-1, "compressor not available"); Py_DECREF(output); return NULL; } /* Compress */ // This macro probably doesn't require the Python interpreter but let's leave it outside for safety output_ptr = PyBytes_AS_STRING(output); if( RELEASEGIL ) { // Run with GIL released, tiny overhead penalty from this (although it // may be significant for smaller chunks.) _save = PyEval_SaveThread(); blocksize = blosc_get_blocksize(); // if blocksize==0, blosc_compress_ctx will try to auto-optimize it. nthreads = blosc_get_nthreads(); cbytes = blosc_compress_ctx(clevel, shuffle, typesize, nbytes, input, output_ptr, nbytes+BLOSC_MAX_OVERHEAD, cname, blocksize, nthreads); PyEval_RestoreThread(_save); _save = NULL; } else { // Hold onto the Python GIL while compressing cbytes = blosc_compress(clevel, shuffle, typesize, nbytes, input, output_ptr, nbytes+BLOSC_MAX_OVERHEAD); } if (cbytes < 0) { blosc_error(cbytes, "while compressing data"); Py_DECREF(output); return NULL; } /* Attempt to resize, if it's much smaller, a copy is required. */ if (_PyBytes_Resize(&output, cbytes) < 0){ /* the memory exception will have been set, hopefully */ Py_DECREF(output); return NULL; } return output; }