int main(int argc, char **argv) {
printf("STARTING TESTS for %s", argv[0]);
blosc_set_nthreads(1);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest = malloc(size);
dest2 = malloc(size);
memset(src, 0, size);
memcpy(srccpy, src, size);
/* Get a compressed buffer */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size);
/* Get a decompressed buffer */
nbytes = blosc_decompress(dest, dest2, size);
/* Run all the suite */
const char *result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED");
}
printf("\tTests run: %d\n", tests_run);
free(src); free(srccpy); free(dest); free(dest2);
return result != 0;
}
int main() {
static float data[SIZE];
static float data_out[SIZE];
static float data_dest[SIZE];
int isize = SIZE * sizeof(float), osize = SIZE * sizeof(float);
int dsize = SIZE * sizeof(float), csize;
int nthreads, pnthreads, i;
for (i = 0; i < SIZE; i++) {
data[i] = i;
}
/* Register the filter with the library */
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
/* Tell Blosc to use some number of threads */
for (nthreads = 1; nthreads <= 4; nthreads++) {
pnthreads = blosc_set_nthreads(nthreads);
printf("Using %d threads (previously using %d)\n", nthreads, pnthreads);
/* Compress with clevel=5 and shuffle active */
csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
if (csize < 0) {
printf("Compression error. Error code: %d\n", csize);
return csize;
}
printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1. * isize) / csize);
/* Decompress */
dsize = blosc_decompress(data_out, data_dest, dsize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
for (i = 0; i < SIZE; i++) {
if (data[i] != data_dest[i]) {
printf("Decompressed data differs from original!\n");
return -1;
}
}
printf("Succesful roundtrip!\n");
}
/* After using it, destroy the Blosc environment */
blosc_destroy();
return 0;
}
static PyObject *
PyBlosc_set_nthreads(PyObject *self, PyObject *args)
{
int nthreads, old_nthreads;
if (!PyArg_ParseTuple(args, "i:set_nthreads", &nthreads))
return NULL;
old_nthreads = blosc_set_nthreads(nthreads);
return Py_BuildValue("i", old_nthreads);
}
int main(int argc, char **argv) {
int32_t *_src;
const char *result;
size_t i;
int nchildren = 4;
printf("STARTING TESTS for %s\n", argv[0]);
/* Launch several subprocesses */
for (i = 1; i <= nchildren; i++) {
fork();
}
blosc_set_nthreads(4);
/* Initialize buffers */
src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size + BLOSC_MAX_OVERHEAD);
dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
_src = (int32_t *)src;
for (i=0; i < (size/4); i++) {
_src[i] = (int32_t)i;
}
memcpy(srccpy, src, size);
/* Run all the suite */
result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED\n");
}
printf("\tTests run: %d\n", tests_run);
blosc_test_free(src);
blosc_test_free(srccpy);
blosc_test_free(dest);
blosc_test_free(dest2);
return result != 0;
}
int main(int argc, char** argv) {
int32_t* _src;
char* result;
size_t i;
printf("STARTING TESTS for %s", argv[0]);
blosc_init();
blosc_set_nthreads(1);
/* Initialize buffers */
src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size + 16);
dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
_src = (int32_t*)src;
for (i = 0; i < (size / 4); i++) {
_src[i] = (int32_t)i;
}
memcpy(srccpy, src, size);
/* Run all the suite */
result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED");
}
printf("\tTests run: %d\n", tests_run);
blosc_test_free(src);
blosc_test_free(srccpy);
blosc_test_free(dest);
blosc_test_free(dest2);
blosc_destroy();
return result != 0;
}
/* Threaded version for compression/decompression */
int parallel_blosc(void)
{
/* Check whether we need to restart threads */
if (!init_threads_done || pid != getpid()) {
blosc_set_nthreads(nthreads);
}
/* Synchronization point for all threads (wait for initialization) */
WAIT_INIT;
/* Synchronization point for all threads (wait for finalization) */
WAIT_FINISH;
if (giveup_code > 0) {
/* Return the total bytes (de-)compressed in threads */
return params.ntbytes;
}
else {
/* Compression/decompression gave up. Return error code. */
return giveup_code;
}
}
void do_bench(int nthreads, int size, int elsize, int rshift, FILE * ofile) {
void *src, *srccpy;
void **dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j;
struct timeval last, current;
float tmemcpy, tshuf, tunshuf;
int clevel, doshuffle=1;
unsigned char *orig, *round;
blosc_set_nthreads(nthreads);
/* Initialize buffers */
src = malloc(size);
srccpy = malloc(size);
dest2 = malloc(size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
dest[j] = malloc(size+BLOSC_MAX_OVERHEAD);
}
/* Warm destination memory (memcpy() will go a bit faster later on) */
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
fprintf(ofile, "--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
gettimeofday(¤t, NULL);
tmemcpy = get_usec_chunk(last, current);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, size/(tmemcpy*MB/1e6));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
gettimeofday(¤t, NULL);
tshuf = get_usec_chunk(last, current);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, size/(tshuf*MB/1e6));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
gettimeofday(&last, NULL);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
gettimeofday(¤t, NULL);
tunshuf = get_usec_chunk(last, current);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, nbytes/(tunshuf*MB/1e6));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
free(src); free(srccpy); free(dest2);
for (i = 0; i < nchunks; i++) {
free(dest[i]);
}
}
void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize,
int rshift, FILE * ofile) {
void *src, *srccpy;
void *dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j, retcode;
unsigned char *orig, *round;
blosc_timestamp_t last, current;
double tmemcpy, tshuf, tunshuf;
int clevel, doshuffle;
if (strcmp(shuffle, "shuffle") == 0) {
doshuffle = BLOSC_SHUFFLE;
}
else if (strcmp(shuffle, "bitshuffle") == 0) {
doshuffle = BLOSC_BITSHUFFLE;
}
else if (strcmp(shuffle, "noshuffle") == 0) {
doshuffle = BLOSC_NOSHUFFLE;
}
blosc_set_nthreads(nthreads);
if(blosc_set_compressor(compressor) < 0){
printf("Compiled w/o support for compressor: '%s', so sorry.\n",
compressor);
exit(1);
}
/* Initialize buffers */
srccpy = malloc(size);
retcode = posix_memalign( (void **)(&src), 32, size);
retcode = posix_memalign( (void **)(&dest2), 32, size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD);
}
fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
blosc_set_timestamp(¤t);
tshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, (size * 1e6) / (tshuf*MB));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
blosc_set_timestamp(¤t);
tunshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, (nbytes * 1e6) / (tunshuf*MB));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip.
Byte-by-byte comparison is slow, so use 'memcmp' to check whether the
roundtripped data is correct. If not, fall back to the slow path to
print diagnostic messages. */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
if (memcmp(orig, round, size) != 0)
{
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
}
else { i = size; }
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
aligned_free(src); free(srccpy); aligned_free(dest2);
for (i = 0; i < nchunks; i++) {
aligned_free(dest[i]);
}
}
int main(int argc, char** argv) {
/* argv[1]: sizeof(element type)
argv[2]: number of elements
argv[3]: buffer alignment
argv[4]: compression level
argv[5]: shuffle enabled
argv[6]: thread count
*/
/* Verify the correct number of command-line args have been specified. */
if (TEST_ARG_COUNT != argc) {
blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc);
return EXIT_FAILURE;
}
/* Parse arguments */
uint32_t type_size;
if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) {
blosc_test_print_bad_arg_msg(1);
return EXIT_FAILURE;
}
uint32_t num_elements;
if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) {
blosc_test_print_bad_arg_msg(2);
return EXIT_FAILURE;
}
uint32_t buffer_align_size;
if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size)
|| (buffer_align_size & (buffer_align_size - 1))
|| (buffer_align_size < sizeof(void*))) {
blosc_test_print_bad_arg_msg(3);
return EXIT_FAILURE;
}
uint32_t compression_level;
if (!blosc_test_parse_uint32_t(argv[4], &compression_level) || (compression_level > 9)) {
blosc_test_print_bad_arg_msg(4);
return EXIT_FAILURE;
}
uint32_t shuffle_enabled;
{
if (!blosc_test_parse_uint32_t(argv[5], &shuffle_enabled) || (shuffle_enabled > 2)) {
blosc_test_print_bad_arg_msg(5);
return EXIT_FAILURE;
}
}
uint32_t blosc_thread_count;
if (!blosc_test_parse_uint32_t(argv[6], &blosc_thread_count) || (blosc_thread_count < 1)) {
blosc_test_print_bad_arg_msg(6);
return EXIT_FAILURE;
}
/* Initialize blosc before running tests. */
blosc_init();
blosc_set_nthreads(blosc_thread_count);
/* Run the test. */
int result = test_getitem(type_size, num_elements, buffer_align_size,
compression_level, shuffle_enabled);
/* Cleanup blosc resources. */
blosc_destroy();
return result;
}
int main() {
static int32_t data[SIZE];
static int32_t data_dest[SIZE];
int isize = SIZE * sizeof(int32_t);
int dsize;
int32_t nbytes, cbytes;
blosc2_sparams sparams = BLOSC_SPARAMS_DEFAULTS;
blosc2_sheader* sheader;
int i, nchunk, nchunks;
printf("Blosc version info: %s (%s)\n",
BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc_init();
blosc_set_nthreads(2);
/* Create a super-chunk container */
sparams.filters[0] = BLOSC_DELTA;
sparams.filters[1] = BLOSC_BITSHUFFLE;
sheader = blosc2_new_schunk(&sparams);
for (nchunk = 1; nchunk <= NCHUNKS; nchunk++) {
for (i = 0; i < SIZE; i++) {
data[i] = i * nchunk;
}
nchunks = blosc2_append_buffer(sheader, sizeof(int32_t), isize, data);
assert(nchunks == nchunk);
}
/* Gather some info */
nbytes = sheader->nbytes;
cbytes = sheader->cbytes;
printf("Compression super-chunk: %d -> %d (%.1fx)\n",
nbytes, cbytes, (1. * nbytes) / cbytes);
/* Retrieve and decompress the chunks (0-based count) */
dsize = blosc2_decompress_chunk(sheader, 0, (void*)data_dest, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
printf("Decompression successful!\n");
for (i = 0; i < SIZE; i++) {
if (data_dest[i] != i) {
printf("Decompressed data differs from original %d, %d, %d!\n", i, data[i], data_dest[i]);
return -1;
}
}
printf("Successful roundtrip!\n");
/* Free resources */
/* Destroy the super-chunk */
blosc2_destroy_schunk(sheader);
/* Destroy the Blosc environment */
blosc_destroy();
return 0;
}
