static PyObject *
compress_helper(void * input, size_t nbytes, size_t typesize,
int clevel, int shuffle, char *cname){
int cbytes;
PyObject *output = NULL;
/* Alloc memory for compression */
if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD)))
return NULL;
/* Select compressor */
if (blosc_set_compressor(cname) < 0) {
/* The compressor is not available (should never happen here) */
blosc_error(-1, "compressor not available");
return NULL;
}
/* Compress */
cbytes = blosc_compress(clevel, shuffle, typesize, nbytes,
input, PyBytes_AS_STRING(output),
nbytes+BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
blosc_error(cbytes, "while compressing data");
Py_XDECREF(output);
return NULL;
}
/* Attempt to resize, if it's much smaller, a copy is required. */
if (_PyBytes_Resize(&output, cbytes) < 0){
/* the memory exception will have been set, hopefully */
return NULL;
}
return output;
}
int main(int argc, char **argv) {
printf("STARTING TESTS for %s", argv[0]);
blosc_set_nthreads(1);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest = malloc(size);
dest2 = malloc(size);
memset(src, 0, size);
memcpy(srccpy, src, size);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size);
/* Get a decompressed buffer */
nbytes = blosc_decompress(dest, dest2, size);
/* Run all the suite */
const char *result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED");
}
printf("\tTests run: %d\n", tests_run);
free(src); free(srccpy); free(dest); free(dest2);
return result != 0;
}
/**
* \brief Allocates memory for compressed data and use it to store results (lossless data compression)
*
* \returns true on success
*/
bool bloscCompress()
{
assert( m_rdata && !m_cdata );
// BLOSC grants for that compressed data never
// exceeds original size + BLOSC_MAX_OVERHEAD
m_cdata_size = m_rdata_size + BLOSC_MAX_OVERHEAD;
m_cdata = m_Allocator( m_cdata_size );
if( NULL == m_cdata )
{
m_err.set( MSG_ERRMEMORY );
return false;
}
/* compress raw data (rdata) and store it in cdata */
m_cdata_size = blosc_compress(
/*clevel*/ m_iCompressionLevel,
/*doshuffle*/ BLOSC_DOSHUFFLE,
/*typesize*/ m_rdata_element_size,
/*nbytes*/ m_rdata_size,
/*src*/ m_rdata,
/*dest*/ m_cdata,
/*destsize*/ m_cdata_size );
return NULL != m_cdata;
}
/* Set a delta reference for the super-chunk */
int blosc2_set_delta_ref(blosc2_sheader* sheader, size_t nbytes,
void* ref) {
int cbytes;
void* filters_chunk;
uint8_t* dec_filters = decode_filters(sheader->filters);
if (dec_filters[0] == BLOSC_DELTA) {
if (sheader->filters_chunk != NULL) {
sheader->cbytes -= *(uint32_t*)(sheader->filters_chunk + 4);
free(sheader->filters_chunk);
}
}
else {
printf("You cannot set a delta reference if delta filter is not set\n");
return(-1);
}
free(dec_filters);
filters_chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
cbytes = blosc_compress(0, 0, 1, nbytes, ref, filters_chunk,
nbytes + BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
free(filters_chunk);
return cbytes;
}
sheader->filters_chunk = filters_chunk;
sheader->cbytes += cbytes;
return cbytes;
}
/* Check splitmode with an environment variable */
static char *test_splitmode_envvar() {
int cbytes2;
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
/* Deactivate the split */
setenv("BLOSC_SPLITMODE", "NEVER", 0);
cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: BLOSC_SPLITMODE envvar does not work correctly", cbytes2 > cbytes);
return 0;
}
/** Test the blosc_getitem function. */
static int test_getitem(size_t type_size, size_t num_elements,
size_t buffer_alignment, int compression_level, int do_shuffle) {
size_t buffer_size = type_size * num_elements;
/* Allocate memory for the test. */
void* original = blosc_test_malloc(buffer_alignment, buffer_size);
void* intermediate = blosc_test_malloc(buffer_alignment, buffer_size + BLOSC_MAX_OVERHEAD);
void* result = blosc_test_malloc(buffer_alignment, buffer_size);
/* Fill the input data buffer with random values. */
blosc_test_fill_random(original, buffer_size);
/* Compress the input data, then use blosc_getitem to extract (decompress)
a range of elements into a new buffer. */
blosc_compress(compression_level, do_shuffle, type_size, buffer_size,
original, intermediate, buffer_size + BLOSC_MAX_OVERHEAD);
blosc_getitem(intermediate, 0, num_elements, result);
/* The round-tripped data matches the original data when the
result of memcmp is 0. */
int exit_code = memcmp(original, result, buffer_size) ?
EXIT_FAILURE : EXIT_SUCCESS;
/* Free allocated memory. */
blosc_test_free(original);
blosc_test_free(intermediate);
blosc_test_free(result);
return exit_code;
}
/* Check compressing + decompressing */
static const char *test_compress_decompress(void) {
const char* compressor;
/* Activate the BLOSC_COMPRESSOR variable */
setenv("BLOSC_COMPRESSOR", "lz4", 0);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor incorrect",
strcmp(compressor, "lz4") == 0);
/* Reset envvar */
unsetenv("BLOSC_COMPRESSOR");
return 0;
}
/* Check maxout with maxout < size (memcpy version) */
static const char *test_maxout_less_memcpy(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not 0", cbytes == 0);
return 0;
}
/* Check maxout with maxout < BLOSC_MAX_OVERHEAD */
static const char *test_max_overhead(void) {
blosc_init();
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1);
mu_assert("ERROR: cbytes is not correct", cbytes < 0);
blosc_destroy();
blosc_init();
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2);
mu_assert("ERROR: cbytes is not correct", cbytes < 0);
blosc_destroy();
blosc_init();
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, 0);
mu_assert("ERROR: cbytes is not correct", cbytes < 0);
blosc_destroy();
return 0;
}
/* Check splitmode */
static char *test_splitmode() {
int cbytes2;
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
/* Deactivate the split */
blosc_set_splitmode(BLOSC_NEVER_SPLIT);
cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: blosc_set_splitmode does not work correctly", cbytes2 > cbytes);
/* Reset the splitmode */
blosc_set_splitmode(BLOSC_FORWARD_COMPAT_SPLIT);
return 0;
}
/* Check maxout with maxout < size */
static char* test_maxout_less() {
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + 15);
mu_assert("ERROR: cbytes is not 0", cbytes == 0);
return 0;
}
/* Check just compressing */
static const char *test_compress(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
return 0;
}
/* Check compression level */
static const char *test_clevel(void) {
int cbytes2;
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
/* Activate the BLOSC_CLEVEL variable */
setenv("BLOSC_CLEVEL", "9", 0);
cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: BLOSC_CLEVEL does not work correctly", cbytes2 < cbytes);
/* Reset envvar */
unsetenv("BLOSC_CLEVEL");
return 0;
}
int main() {
static float data[SIZE];
static float data_out[SIZE];
static float data_dest[SIZE];
int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float);
int dsize = SIZE * sizeof(float), csize;
int nthreads, pnthreads, i;
for (i = 0; i < SIZE; i++) {
data[i] = i;
}
/* Register the filter with the library */
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Tell Blosc to use some number of threads */
for (nthreads = 1; nthreads <= 4; nthreads++) {
pnthreads = blosc_set_nthreads(nthreads);
printf("Using %d threads (previously using %d)\n", nthreads, pnthreads);
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize);
/* Decompress */
dsize = blosc_decompress(data_out, data_dest, dsize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
for (i = 0; i < SIZE; i++) {
if (data[i] != data_dest[i]) {
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
}
/* After using it, destroy the Blosc environment */
blosc_destroy();
return 0;
}
/* Check maxout with maxout > size (memcpy version) */
static const char *test_maxout_great_memcpy(void) {
/* Get a compressed buffer */
cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
return 0;
}
int main(){
static float data[SIZE];
static float data_out[SIZE];
static float data_dest[SIZE];
int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float);
int dsize = SIZE*sizeof(float), csize;
int i;
for(i=0; i<SIZE; i++){
data[i] = i;
}
/* Register the filter with the library */
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
if (csize == 0) {
printf("Buffer is uncompressible. Giving up.\n");
return 1;
}
else if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize);
/* Decompress */
dsize = blosc_decompress(data_out, data_dest, dsize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
printf("Decompression succesful!\n");
/* After using it, destroy the Blosc environment */
blosc_destroy();
for(i=0;i<SIZE;i++){
if(data[i] != data_dest[i]) {
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
return 0;
}
/* Check bitshuffle */
static const char *test_bitshuffle(void) {
int cbytes2;
/* Get a compressed buffer */
blosc_set_compressor("blosclz"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not 0", cbytes < size);
/* Activate the BLOSC_BITSHUFFLE variable */
setenv("BLOSC_SHUFFLE", "BITSHUFFLE", 0);
cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: BLOSC_SHUFFLE=BITSHUFFLE does not work correctly",
cbytes2 < cbytes * 1.5);
/* Reset env var */
unsetenv("BLOSC_SHUFFLE");
return 0;
}
/* Check maxout with maxout > size */
static char* test_maxout_great() {
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + 17);
mu_assert("ERROR: cbytes is not 0", cbytes == size + 16);
/* Decompress the buffer */
nbytes = blosc_decompress(dest, dest2, size);
mu_assert("ERROR: nbytes incorrect(1)", nbytes == size);
return 0;
}
/* Check for compressing an empty buffer */
static char *test_empty_buffer() {
int cbytes1;
int cbytes2;
cbytes1 = blosc_compress(1, 1, 1, 0, src, dest, BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 == BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, 0);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == 0);
return 0;
}
/* Check for decompressing into a buffer larger than necessary (v2) */
static char *test_too_long_dest2() {
int cbytes1;
int cbytes2;
int srclen = 3069;
cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 <= srclen + BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, srclen + 1021);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen);
return 0;
}
/* Check for compressing a very small buffer */
static char *test_small_buffer() {
int cbytes1;
int cbytes2;
int srclen;
for (srclen = 1; srclen < BLOSC_MAX_OVERHEAD; srclen++) {
cbytes1 = blosc_compress(1, 1, typesize, srclen, src, dest, srclen + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes1 == srclen + BLOSC_MAX_OVERHEAD);
cbytes2 = blosc_decompress(dest, src, srclen);
mu_assert("ERROR: decompressed bytes is not correct", cbytes2 == srclen);
}
return 0;
}
/* Append a data buffer to a *packed* super-chunk. */
void* blosc2_packed_append_buffer(void* packed, size_t typesize, size_t nbytes, void* src) {
int cname = *(int16_t*)((uint8_t*)packed + 4);
int clevel = *(int16_t*)((uint8_t*)packed + 6);
void* filters_chunk = (uint8_t*)packed + *(uint64_t*)((uint8_t*)packed + 40);
uint8_t* filters = decode_filters(*(uint16_t*)((uint8_t*)packed + 8));
int cbytes;
void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
void* dest = malloc(nbytes);
char* compname;
int doshuffle;
void* new_packed;
/* Apply filters prior to compress */
if (filters[0] == BLOSC_DELTA) {
doshuffle = filters[1];
if (filters_chunk == NULL) {
/* For packed super-buffers, the filters schunk should exist */
return NULL;
}
delta_encoder8(filters_chunk, 0, (int)nbytes, src, dest);
/* memcpy(dest, src, nbytes); */
src = dest;
}
else {
doshuffle = filters[0];
}
/* Compress the src buffer using super-chunk defaults */
blosc_compcode_to_compname(cname, &compname);
blosc_set_compressor(compname);
cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk,
nbytes + BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
free(chunk);
free(dest);
free(filters);
return NULL;
}
/* We don't need dest and filters anymore */
free(dest);
free(filters);
/* Append the chunk and free it */
new_packed = packed_append_chunk(packed, chunk);
free(chunk);
return new_packed;
}
// compress a buffer
size_t
BloscWrapper::compress(void* src,
size_t srcsize,
void* dst,
size_t dstsize,
int clevel,
bool doshuffle,
size_t typesize) {
// convert doshuffle to int
int doshuffle_int = 0;
if (doshuffle) {
doshuffle_int = 1;
}
// perform compression
return blosc_compress(clevel, doshuffle_int, typesize, srcsize, src, dst, dstsize);
}
/* Append a data buffer to a super-chunk. */
size_t blosc2_append_buffer(blosc2_sheader* sheader, size_t typesize,
size_t nbytes, void* src) {
int cbytes;
void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
uint8_t* dec_filters = decode_filters(sheader->filters);
int clevel = sheader->clevel;
char* compname;
int doshuffle, ret;
/* Apply filters prior to compress */
if (dec_filters[0] == BLOSC_DELTA) {
doshuffle = dec_filters[1];
if (sheader->filters_chunk == NULL) {
ret = blosc2_set_delta_ref(sheader, nbytes, src);
if (ret < 0) {
return((size_t)ret);
}
}
}
else {
doshuffle = dec_filters[0];
}
free(dec_filters);
/* Compress the src buffer using super-chunk defaults */
blosc_compcode_to_compname(sheader->compressor, &compname);
blosc_set_compressor(compname);
blosc_set_schunk(sheader);
cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk,
nbytes + BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
free(chunk);
return cbytes;
}
/* Append the chunk */
return append_chunk(sheader, chunk);
}
/* Check compressor */
static const char *test_compressor(void) {
const char* compressor;
/* Before any blosc_compress() the compressor must be blosclz */
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor (compress, before) incorrect",
strcmp(compressor, "blosclz") == 0);
/* Activate the BLOSC_COMPRESSOR variable */
setenv("BLOSC_COMPRESSOR", "lz4", 0);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not correct", cbytes < size);
compressor = blosc_get_compressor();
mu_assert("ERROR: get_compressor (compress, after) incorrect",
strcmp(compressor, "lz4") == 0);
/* Reset envvar */
unsetenv("BLOSC_COMPRESSOR");
return 0;
}
void do_bench(int nthreads, int size, int elsize, int rshift, FILE * ofile) {
void *src, *srccpy;
void **dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j;
struct timeval last, current;
float tmemcpy, tshuf, tunshuf;
int clevel, doshuffle=1;
unsigned char *orig, *round;
blosc_set_nthreads(nthreads);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest2 = malloc(size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
dest[j] = malloc(size+BLOSC_MAX_OVERHEAD);
}
/* Warm destination memory (memcpy() will go a bit faster later on) */
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
fprintf(ofile, "--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
gettimeofday(¤t, NULL);
tshuf = get_usec_chunk(last, current);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, size/(tshuf*MB/1e6));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
gettimeofday(¤t, NULL);
tunshuf = get_usec_chunk(last, current);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, nbytes/(tunshuf*MB/1e6));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
free(src); free(srccpy); free(dest2);
for (i = 0; i < nchunks; i++) {
free(dest[i]);
}
}
void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize,
int rshift, FILE * ofile) {
void *src, *srccpy;
void *dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j, retcode;
unsigned char *orig, *round;
blosc_timestamp_t last, current;
double tmemcpy, tshuf, tunshuf;
int clevel, doshuffle;
if (strcmp(shuffle, "shuffle") == 0) {
doshuffle = BLOSC_SHUFFLE;
}
else if (strcmp(shuffle, "bitshuffle") == 0) {
doshuffle = BLOSC_BITSHUFFLE;
}
else if (strcmp(shuffle, "noshuffle") == 0) {
doshuffle = BLOSC_NOSHUFFLE;
}
blosc_set_nthreads(nthreads);
if(blosc_set_compressor(compressor) < 0){
printf("Compiled w/o support for compressor: '%s', so sorry.\n",
compressor);
exit(1);
}
/* Initialize buffers */
srccpy = malloc(size);
retcode = posix_memalign( (void **)(&src), 32, size);
retcode = posix_memalign( (void **)(&dest2), 32, size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD);
}
fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
blosc_set_timestamp(¤t);
tshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, (size * 1e6) / (tshuf*MB));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
blosc_set_timestamp(¤t);
tunshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, (nbytes * 1e6) / (tunshuf*MB));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip.
Byte-by-byte comparison is slow, so use 'memcmp' to check whether the
roundtripped data is correct. If not, fall back to the slow path to
print diagnostic messages. */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
if (memcmp(orig, round, size) != 0)
{
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
}
else { i = size; }
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
aligned_free(src); free(srccpy); aligned_free(dest2);
for (i = 0; i < nchunks; i++) {
aligned_free(dest[i]);
}
}
int main() {
static float data[SIZE];
static float data_dest[SIZE];
int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float);
int dsize, csize;
blosc2_sparams sparams = BLOSC_SPARAMS_DEFAULTS;
blosc2_sheader* sheader;
int i, nchunks;
for (i = 0; i < SIZE; i++) {
data[i] = i;
}
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, BLOSC_SHUFFLE, sizeof(float),
isize, data, data_dest, osize);
if (csize == 0) {
printf("Buffer is uncompressible. Giving up.\n");
return 1;
}
else if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize);
/* Create a super-chunk container */
sparams.filters[0] = BLOSC_DELTA;
sparams.filters[1] = BLOSC_SHUFFLE;
sheader = blosc2_new_schunk(&sparams);
/* Now append a couple of chunks */
nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data);
assert(nchunks == 1);
nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data);
assert(nchunks == 2);
/* Retrieve and decompress the chunks (0-based count) */
dsize = blosc2_decompress_chunk(sheader, 0, (void*)data_dest, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
dsize = blosc2_decompress_chunk(sheader, 1, (void*)data_dest, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
printf("Decompression succesful!\n");
for (i = 0; i < SIZE; i++) {
if (data[i] != data_dest[i]) {
printf("i, values: %d, %f, %f\n", i, data[i], data_dest[i]);
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
/* Free resources */
/* Destroy the super-chunk */
blosc2_destroy_schunk(sheader);
/* Destroy the Blosc environment */
blosc_destroy();
return 0;
}
static PyObject *
compress_helper(void * input, size_t nbytes, size_t typesize,
int clevel, int shuffle, char *cname){
int cbytes, blocksize, nthreads;
PyObject *output;
char *output_ptr;
PyThreadState *_save = NULL;
/* Alloc memory for compression */
if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD)))
return NULL;
/* Select compressor */
if (blosc_set_compressor(cname) < 0) {
/* The compressor is not available (should never happen here) */
blosc_error(-1, "compressor not available");
Py_DECREF(output);
return NULL;
}
/* Compress */
// This macro probably doesn't require the Python interpreter but let's leave it outside for safety
output_ptr = PyBytes_AS_STRING(output);
if( RELEASEGIL )
{
// Run with GIL released, tiny overhead penalty from this (although it
// may be significant for smaller chunks.)
_save = PyEval_SaveThread();
blocksize = blosc_get_blocksize();
// if blocksize==0, blosc_compress_ctx will try to auto-optimize it.
nthreads = blosc_get_nthreads();
cbytes = blosc_compress_ctx(clevel, shuffle, typesize, nbytes,
input, output_ptr, nbytes+BLOSC_MAX_OVERHEAD,
cname, blocksize, nthreads);
PyEval_RestoreThread(_save);
_save = NULL;
}
else
{ // Hold onto the Python GIL while compressing
cbytes = blosc_compress(clevel, shuffle, typesize, nbytes,
input, output_ptr,
nbytes+BLOSC_MAX_OVERHEAD);
}
if (cbytes < 0) {
blosc_error(cbytes, "while compressing data");
Py_DECREF(output);
return NULL;
}
/* Attempt to resize, if it's much smaller, a copy is required. */
if (_PyBytes_Resize(&output, cbytes) < 0){
/* the memory exception will have been set, hopefully */
Py_DECREF(output);
return NULL;
}
return output;
}
