/* Check maxout with maxout < BLOSC_MAX_OVERHEAD */ static const char *test_max_overhead(void) { blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, 0); mu_assert("ERROR: cbytes is not correct", cbytes < 0); blosc_destroy(); return 0; }
int main() { static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float); int dsize = SIZE * sizeof(float), csize; int nthreads, pnthreads, i; for (i = 0; i < SIZE; i++) { data[i] = i; } /* Register the filter with the library */ printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Tell Blosc to use some number of threads */ for (nthreads = 1; nthreads <= 4; nthreads++) { pnthreads = blosc_set_nthreads(nthreads); printf("Using %d threads (previously using %d)\n", nthreads, pnthreads); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize); if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } for (i = 0; i < SIZE; i++) { if (data[i] != data_dest[i]) { printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); } /* After using it, destroy the Blosc environment */ blosc_destroy(); return 0; }
int main(){ static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int i; for(i=0; i<SIZE; i++){ data[i] = i; } /* Register the filter with the library */ printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize); if (csize == 0) { printf("Buffer is uncompressible. Giving up.\n"); return 1; } else if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); /* After using it, destroy the Blosc environment */ blosc_destroy(); for(i=0;i<SIZE;i++){ if(data[i] != data_dest[i]) { printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); return 0; }
int main(int argc, char **argv) { int32_t *_src; const char *result; size_t i; printf("STARTING TESTS for %s", argv[0]); blosc_init(); blosc_set_nthreads(1); /* Initialize buffers */ src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size + BLOSC_MAX_OVERHEAD); dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); _src = (int32_t *)src; for (i=0; i < (size/4); i++) { _src[i] = (int32_t)i; } memcpy(srccpy, src, size); /* Run all the suite */ result = all_tests(); if (result != 0) { printf(" (%s)\n", result); } else { printf(" ALL TESTS PASSED"); } printf("\tTests run: %d\n", tests_run); blosc_test_free(src); blosc_test_free(srccpy); blosc_test_free(dest); blosc_test_free(dest2); blosc_destroy(); return result != 0; }
int main(int argc, char **argv) { char *result; printf("STARTING TESTS for %s", argv[0]); blosc_init(); blosc_set_nthreads(1); /* Initialize buffers */ src = malloc(size); srccpy = malloc(size); dest = malloc(size); dest2 = malloc(size); memset(src, 0, size); memcpy(srccpy, src, size); /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size); /* Get a decompressed buffer */ nbytes = blosc_decompress(dest, dest2, size); /* Run all the suite */ result = all_tests(); if (result != 0) { printf(" (%s)\n", result); } else { printf(" ALL TESTS PASSED"); } printf("\tTests run: %d\n", tests_run); free(src); free(srccpy); free(dest); free(dest2); blosc_destroy(); return result != 0; }
static PyObject * PyBlosc_destroy(PyObject *self) { blosc_destroy(); return Py_None; }
int main(int argc, char *argv[]) { int single = 1; int suite = 0; int hard_suite = 0; int extreme_suite = 0; int debug_suite = 0; int nthreads = 4; /* The number of threads */ int size = 2*MB; /* Buffer size */ int elsize = 8; /* Datatype size */ int rshift = 19; /* Significant bits */ int workingset = 256*MB; /* The maximum allocated memory */ int nthreads_, size_, elsize_, rshift_, i; FILE * output_file = stdout; struct timeval last, current; float totaltime; char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]"; if (argc == 1) { printf("%s\n", usage); exit(1); } if (strcmp(argv[1], "single") == 0) { single = 1; } else if (strcmp(argv[1], "suite") == 0) { suite = 1; } else if (strcmp(argv[1], "hardsuite") == 0) { hard_suite = 1; workingset = 64*MB; /* Values here are ending points for loops */ nthreads = 2; size = 8*MB; elsize = 32; rshift = 32; } else if (strcmp(argv[1], "extremesuite") == 0) { extreme_suite = 1; workingset = 32*MB; niter = 1; /* Values here are ending points for loops */ nthreads = 4; size = 16*MB; elsize = 32; rshift = 32; } else if (strcmp(argv[1], "debugsuite") == 0) { debug_suite = 1; workingset = 32*MB; niter = 1; /* Warning: values here are starting points for loops. This is useful for debugging. */ nthreads = 1; size = 16*KB; elsize = 1; rshift = 0; } else { printf("%s\n", usage); exit(1); } if (argc >= 3) { nthreads = atoi(argv[2]); } if (argc >= 4) { size = atoi(argv[3]); } if (argc >= 5) { elsize = atoi(argv[4]); } if (argc >= 6) { rshift = atoi(argv[5]); } if ((argc >= 7) || !(single || suite || hard_suite || extreme_suite)) { printf("%s\n", usage); exit(1); } nchunks = get_nchunks(size, workingset); gettimeofday(&last, NULL); blosc_init(); if (suite) { for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) { do_bench(nthreads_, size, elsize, rshift, output_file); } } else if (hard_suite) { /* Let's start the rshift loop by 4 so that 19 is visited. This is to allow a direct comparison with the plain suite, that runs precisely at 19 significant bits. */ for (rshift_ = 4; rshift_ <= rshift; rshift_ += 5) { for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_; i <= elsize_; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); niter = 1; for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(nthreads_, size_+i, elsize_, rshift_, output_file); gettimeofday(¤t, NULL); totaltime = getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (extreme_suite) { for (rshift_ = 0; rshift_ <= rshift; rshift_++) { for (elsize_ = 1; elsize_ <= elsize; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(nthreads_, size_+i, elsize_, rshift_, output_file); gettimeofday(¤t, NULL); totaltime = getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (debug_suite) { for (rshift_ = rshift; rshift_ <= 32; rshift_++) { for (elsize_ = elsize; elsize_ <= 32; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = size; size_ <= 16*MB; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) { do_bench(nthreads_, size_+i, elsize_, rshift_, output_file); gettimeofday(¤t, NULL); totaltime = getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } /* Single mode */ else { do_bench(nthreads, size, elsize, rshift, output_file); } /* Print out some statistics */ gettimeofday(¤t, NULL); totaltime = getseconds(last, current); printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB); printf("Elapsed time:\t %6.1f s, %.1f MB/s\n", totaltime, totalsize*2*1.1/(MB*totaltime)); /* Free blosc resources */ blosc_free_resources(); blosc_destroy(); return 0; }
int main(int argc, char *argv[]) { char compressor[32]; char shuffle[32] = "shuffle"; char bsuite[32]; int single = 1; int suite = 0; int hard_suite = 0; int extreme_suite = 0; int debug_suite = 0; int nthreads = 4; /* The number of threads */ int size = 2*MB; /* Buffer size */ int elsize = 8; /* Datatype size */ int rshift = 19; /* Significant bits */ #if defined(__arm__) int workingset = 64*MB; /* The maximum allocated memory */ #else int workingset = 256*MB; /* The maximum allocated memory */ #endif int nthreads_, size_, elsize_, rshift_, i; FILE * output_file = stdout; blosc_timestamp_t last, current; float totaltime; char usage[256]; print_compress_info(); strncpy(usage, "Usage: bench [blosclz | lz4 | lz4hc | snappy | zlib] " "[noshuffle | shuffle | bitshuffle] " "[single | suite | hardsuite | extremesuite | debugsuite] " "[nthreads] [bufsize(bytes)] [typesize] [sbits]", 255); if (argc < 2) { printf("%s\n", usage); exit(1); } strcpy(compressor, argv[1]); if (strcmp(compressor, "blosclz") != 0 && strcmp(compressor, "lz4") != 0 && strcmp(compressor, "lz4hc") != 0 && strcmp(compressor, "snappy") != 0 && strcmp(compressor, "zlib") != 0) { printf("No such compressor: '%s'\n", compressor); printf("%s\n", usage); exit(2); } if (argc >= 3) { strcpy(shuffle, argv[2]); if (strcmp(shuffle, "shuffle") != 0 && strcmp(shuffle, "bitshuffle") != 0 && strcmp(shuffle, "noshuffle") != 0) { printf("No such shuffler: '%s'\n", shuffle); printf("%s\n", usage); exit(2); } } if (argc < 4) strcpy(bsuite, "single"); else strcpy(bsuite, argv[3]); if (strcmp(bsuite, "single") == 0) { single = 1; } else if (strcmp(bsuite, "test") == 0) { single = 1; workingset /= 2; } else if (strcmp(bsuite, "suite") == 0) { suite = 1; } else if (strcmp(bsuite, "hardsuite") == 0) { hard_suite = 1; workingset /= 4; /* Values here are ending points for loops */ nthreads = 2; size = 8*MB; elsize = 32; rshift = 32; } else if (strcmp(bsuite, "extremesuite") == 0) { extreme_suite = 1; workingset /= 8; niter = 1; /* Values here are ending points for loops */ nthreads = 4; size = 16*MB; elsize = 32; rshift = 32; } else if (strcmp(bsuite, "debugsuite") == 0) { debug_suite = 1; workingset /= 8; niter = 1; /* Warning: values here are starting points for loops. This is useful for debugging. */ nthreads = 1; size = 16*KB; elsize = 1; rshift = 0; } else { printf("%s\n", usage); exit(1); } printf("Using compressor: %s\n", compressor); printf("Using shuffle type: %s\n", shuffle); printf("Running suite: %s\n", bsuite); if (argc >= 5) { nthreads = atoi(argv[4]); } if (argc >= 6) { size = atoi(argv[5]); } if (argc >= 7) { elsize = atoi(argv[6]); } if (argc >= 8) { rshift = atoi(argv[7]); } if ((argc >= 9) || !(single || suite || hard_suite || extreme_suite)) { printf("%s\n", usage); exit(1); } nchunks = get_nchunks(size, workingset); blosc_set_timestamp(&last); blosc_init(); if (suite) { for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, output_file); } } else if (hard_suite) { /* Let's start the rshift loop by 4 so that 19 is visited. This is to allow a direct comparison with the plain suite, that runs precisely at 19 significant bits. */ for (rshift_ = 4; rshift_ <= rshift; rshift_ += 5) { for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_; i <= elsize_; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); niter = 1; for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (extreme_suite) { for (rshift_ = 0; rshift_ <= rshift; rshift_++) { for (elsize_ = 1; elsize_ <= elsize; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (debug_suite) { for (rshift_ = rshift; rshift_ <= 32; rshift_++) { for (elsize_ = elsize; elsize_ <= 32; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = size; size_ <= 16*MB; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } /* Single mode */ else { do_bench(compressor, shuffle, nthreads, size, elsize, rshift, output_file); } /* Print out some statistics */ blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB); printf("Elapsed time:\t %6.1f s, %.1f MB/s\n", totaltime, totalsize*2*1.1/(MB*totaltime)); /* Free blosc resources */ blosc_free_resources(); blosc_destroy(); return 0; }
int main() { static float data[SIZE]; static float data_dest[SIZE]; int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float); int dsize, csize; blosc2_sparams sparams = BLOSC_SPARAMS_DEFAULTS; blosc2_sheader* sheader; int i, nchunks; for (i = 0; i < SIZE; i++) { data[i] = i; } printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); /* Compress with clevel=5 and shuffle active */ csize = blosc_compress(5, BLOSC_SHUFFLE, sizeof(float), isize, data, data_dest, osize); if (csize == 0) { printf("Buffer is uncompressible. Giving up.\n"); return 1; } else if (csize < 0) { printf("Compression error. Error code: %d\n", csize); return csize; } printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize); /* Create a super-chunk container */ sparams.filters[0] = BLOSC_DELTA; sparams.filters[1] = BLOSC_SHUFFLE; sheader = blosc2_new_schunk(&sparams); /* Now append a couple of chunks */ nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data); assert(nchunks == 1); nchunks = blosc2_append_buffer(sheader, sizeof(float), isize, data); assert(nchunks == 2); /* Retrieve and decompress the chunks (0-based count) */ dsize = blosc2_decompress_chunk(sheader, 0, (void*)data_dest, isize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } dsize = blosc2_decompress_chunk(sheader, 1, (void*)data_dest, isize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); for (i = 0; i < SIZE; i++) { if (data[i] != data_dest[i]) { printf("i, values: %d, %f, %f\n", i, data[i], data_dest[i]); printf("Decompressed data differs from original!\n"); return -1; } } printf("Succesful roundtrip!\n"); /* Free resources */ /* Destroy the super-chunk */ blosc2_destroy_schunk(sheader); /* Destroy the Blosc environment */ blosc_destroy(); return 0; }
// dtor BloscWrapper::~BloscWrapper() { blosc_destroy(); }
static PyObject * PyBlosc_destroy(PyObject *self) { blosc_destroy(); Py_RETURN_NONE; }
int main(int argc, char** argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment argv[4]: compression level argv[5]: shuffle enabled argv[6]: thread count */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } uint32_t compression_level; if (!blosc_test_parse_uint32_t(argv[4], &compression_level) || (compression_level > 9)) { blosc_test_print_bad_arg_msg(4); return EXIT_FAILURE; } uint32_t shuffle_enabled; { if (!blosc_test_parse_uint32_t(argv[5], &shuffle_enabled) || (shuffle_enabled > 2)) { blosc_test_print_bad_arg_msg(5); return EXIT_FAILURE; } } uint32_t blosc_thread_count; if (!blosc_test_parse_uint32_t(argv[6], &blosc_thread_count) || (blosc_thread_count < 1)) { blosc_test_print_bad_arg_msg(6); return EXIT_FAILURE; } /* Initialize blosc before running tests. */ blosc_init(); blosc_set_nthreads(blosc_thread_count); /* Run the test. */ int result = test_getitem(type_size, num_elements, buffer_align_size, compression_level, shuffle_enabled); /* Cleanup blosc resources. */ blosc_destroy(); return result; }
int main() { static int32_t data[SIZE]; static int32_t data_dest[SIZE]; int isize = SIZE * sizeof(int32_t); int dsize; int32_t nbytes, cbytes; blosc2_sparams sparams = BLOSC_SPARAMS_DEFAULTS; blosc2_sheader* sheader; int i, nchunk, nchunks; printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); /* Initialize the Blosc compressor */ blosc_init(); blosc_set_nthreads(2); /* Create a super-chunk container */ sparams.filters[0] = BLOSC_DELTA; sparams.filters[1] = BLOSC_BITSHUFFLE; sheader = blosc2_new_schunk(&sparams); for (nchunk = 1; nchunk <= NCHUNKS; nchunk++) { for (i = 0; i < SIZE; i++) { data[i] = i * nchunk; } nchunks = blosc2_append_buffer(sheader, sizeof(int32_t), isize, data); assert(nchunks == nchunk); } /* Gather some info */ nbytes = sheader->nbytes; cbytes = sheader->cbytes; printf("Compression super-chunk: %d -> %d (%.1fx)\n", nbytes, cbytes, (1. * nbytes) / cbytes); /* Retrieve and decompress the chunks (0-based count) */ dsize = blosc2_decompress_chunk(sheader, 0, (void*)data_dest, isize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression successful!\n"); for (i = 0; i < SIZE; i++) { if (data_dest[i] != i) { printf("Decompressed data differs from original %d, %d, %d!\n", i, data[i], data_dest[i]); return -1; } } printf("Successful roundtrip!\n"); /* Free resources */ /* Destroy the super-chunk */ blosc2_destroy_schunk(sheader); /* Destroy the Blosc environment */ blosc_destroy(); return 0; }