static PyObject *
compress_helper(void * input, size_t nbytes, size_t typesize,
int clevel, int shuffle, char *cname){
int cbytes;
PyObject *output = NULL;
/* Alloc memory for compression */
if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD)))
return NULL;
/* Select compressor */
if (blosc_set_compressor(cname) < 0) {
/* The compressor is not available (should never happen here) */
blosc_error(-1, "compressor not available");
return NULL;
}
/* Compress */
cbytes = blosc_compress(clevel, shuffle, typesize, nbytes,
input, PyBytes_AS_STRING(output),
nbytes+BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
blosc_error(cbytes, "while compressing data");
Py_XDECREF(output);
return NULL;
}
/* Attempt to resize, if it's much smaller, a copy is required. */
if (_PyBytes_Resize(&output, cbytes) < 0){
/* the memory exception will have been set, hopefully */
return NULL;
}
return output;
}
// ctor
BloscWrapper::BloscWrapper() {
blosc_init();
if (blosc_set_compressor("zlib") == -1) {
std::cerr << "Blosc reports cannot use zlib. Falling back to default" << std::endl;
}
}
/**
* \brief Converts compressor ID string to category enum
*
* \param[in] strCompressorType Compressor name as string
* \param[in] iCompressionLevel Compression level (compressor dependent)
*/
bool setCompressor( const char *strCompressorType, int iCompressionLevel = -1 )
{
compressor_type_e eCompressorType = CT_NONE;
m_err.clear();
// if no compressor or compression is specified, use standard compressor
// which leads to no compression
if( 0 == iCompressionLevel || !strCompressorType || !*strCompressorType )
{
strCompressorType = COMPRESSOR_DEFAULT_ID;
iCompressionLevel = 0;
}
// checking compressor names
if( 0 == _strcmpi( strCompressorType, BLOSC_LZ4_ID ) )
{
eCompressorType = CT_BLOSC;
}
else if( 0 == _strcmpi( strCompressorType, BLOSC_LZ4HC_ID ) )
{
eCompressorType = CT_BLOSC;
}
else if( 0 == _strcmpi( strCompressorType, BLOSC_DEFAULT_ID ) )
{
eCompressorType = CT_BLOSC;
}
else if( 0 == _strcmpi( strCompressorType, QLIN16_ID ) )
{
eCompressorType = CT_QLIN16;
}
else if( 0 == _strcmpi( strCompressorType, QLOG16_ID ) )
{
eCompressorType = CT_QLOG16;
}
// check and acquire valid settings
if( CT_NONE != eCompressorType )
{
m_strCompressorType = strCompressorType;
m_eCompressorType = eCompressorType;
if( iCompressionLevel >= 0 )
{
m_iCompressionLevel = iCompressionLevel;
}
if( m_eCompressorType == CT_BLOSC )
{
blosc_set_compressor( m_strCompressorType );
}
return true;
}
else return false;
}
/* Append a data buffer to a *packed* super-chunk. */
void* blosc2_packed_append_buffer(void* packed, size_t typesize, size_t nbytes, void* src) {
int cname = *(int16_t*)((uint8_t*)packed + 4);
int clevel = *(int16_t*)((uint8_t*)packed + 6);
void* filters_chunk = (uint8_t*)packed + *(uint64_t*)((uint8_t*)packed + 40);
uint8_t* filters = decode_filters(*(uint16_t*)((uint8_t*)packed + 8));
int cbytes;
void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
void* dest = malloc(nbytes);
char* compname;
int doshuffle;
void* new_packed;
/* Apply filters prior to compress */
if (filters[0] == BLOSC_DELTA) {
doshuffle = filters[1];
if (filters_chunk == NULL) {
/* For packed super-buffers, the filters schunk should exist */
return NULL;
}
delta_encoder8(filters_chunk, 0, (int)nbytes, src, dest);
/* memcpy(dest, src, nbytes); */
src = dest;
}
else {
doshuffle = filters[0];
}
/* Compress the src buffer using super-chunk defaults */
blosc_compcode_to_compname(cname, &compname);
blosc_set_compressor(compname);
cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk,
nbytes + BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
free(chunk);
free(dest);
free(filters);
return NULL;
}
/* We don't need dest and filters anymore */
free(dest);
free(filters);
/* Append the chunk and free it */
new_packed = packed_append_chunk(packed, chunk);
free(chunk);
return new_packed;
}
/* Check bitshuffle */
static const char *test_bitshuffle(void) {
int cbytes2;
/* Get a compressed buffer */
blosc_set_compressor("blosclz"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */
cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: cbytes is not 0", cbytes < size);
/* Activate the BLOSC_BITSHUFFLE variable */
setenv("BLOSC_SHUFFLE", "BITSHUFFLE", 0);
cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
dest, size + BLOSC_MAX_OVERHEAD);
mu_assert("ERROR: BLOSC_SHUFFLE=BITSHUFFLE does not work correctly",
cbytes2 < cbytes * 1.5);
/* Reset env var */
unsetenv("BLOSC_SHUFFLE");
return 0;
}
int main(int argc, char **argv) {
int64_t *_src;
const char *result;
size_t i;
printf("STARTING TESTS for %s", argv[0]);
blosc_init();
blosc_set_compressor("blosclz");
/* Initialize buffers */
src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size + BLOSC_MAX_OVERHEAD);
dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size);
_src = (int64_t *)src;
for (i=0; i < (size / sizeof(int64_t)); i++) {
_src[i] = (int64_t)i;
}
memcpy(srccpy, src, size);
/* Run all the suite */
result = all_tests();
if (result != 0) {
printf(" (%s)\n", result);
}
else {
printf(" ALL TESTS PASSED");
}
printf("\tTests run: %d\n", tests_run);
blosc_test_free(src);
blosc_test_free(srccpy);
blosc_test_free(dest);
blosc_test_free(dest2);
blosc_destroy();
return result != 0;
}
/* Append a data buffer to a super-chunk. */
size_t blosc2_append_buffer(blosc2_sheader* sheader, size_t typesize,
size_t nbytes, void* src) {
int cbytes;
void* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
uint8_t* dec_filters = decode_filters(sheader->filters);
int clevel = sheader->clevel;
char* compname;
int doshuffle, ret;
/* Apply filters prior to compress */
if (dec_filters[0] == BLOSC_DELTA) {
doshuffle = dec_filters[1];
if (sheader->filters_chunk == NULL) {
ret = blosc2_set_delta_ref(sheader, nbytes, src);
if (ret < 0) {
return((size_t)ret);
}
}
}
else {
doshuffle = dec_filters[0];
}
free(dec_filters);
/* Compress the src buffer using super-chunk defaults */
blosc_compcode_to_compname(sheader->compressor, &compname);
blosc_set_compressor(compname);
blosc_set_schunk(sheader);
cbytes = blosc_compress(clevel, doshuffle, typesize, nbytes, src, chunk,
nbytes + BLOSC_MAX_OVERHEAD);
if (cbytes < 0) {
free(chunk);
return cbytes;
}
/* Append the chunk */
return append_chunk(sheader, chunk);
}
void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize,
int rshift, FILE * ofile) {
void *src, *srccpy;
void *dest[NCHUNKS], *dest2;
int nbytes = 0, cbytes = 0;
int i, j, retcode;
unsigned char *orig, *round;
blosc_timestamp_t last, current;
double tmemcpy, tshuf, tunshuf;
int clevel, doshuffle;
if (strcmp(shuffle, "shuffle") == 0) {
doshuffle = BLOSC_SHUFFLE;
}
else if (strcmp(shuffle, "bitshuffle") == 0) {
doshuffle = BLOSC_BITSHUFFLE;
}
else if (strcmp(shuffle, "noshuffle") == 0) {
doshuffle = BLOSC_NOSHUFFLE;
}
blosc_set_nthreads(nthreads);
if(blosc_set_compressor(compressor) < 0){
printf("Compiled w/o support for compressor: '%s', so sorry.\n",
compressor);
exit(1);
}
/* Initialize buffers */
srccpy = malloc(size);
retcode = posix_memalign( (void **)(&src), 32, size);
retcode = posix_memalign( (void **)(&dest2), 32, size);
/* zero src to initialize byte on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD);
}
fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, (size * 1e6) / (tmemcpy*MB));
for (clevel=0; clevel<10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
dest[j], size+BLOSC_MAX_OVERHEAD);
}
}
blosc_set_timestamp(¤t);
tshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, (size * 1e6) / (tshuf*MB));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = size;
}
else {
nbytes = blosc_decompress(dest[j], dest2, size);
}
}
}
blosc_set_timestamp(¤t);
tunshuf = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, (nbytes * 1e6) / (tunshuf*MB));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip.
Byte-by-byte comparison is slow, so use 'memcmp' to check whether the
roundtripped data is correct. If not, fall back to the slow path to
print diagnostic messages. */
orig = (unsigned char *)srccpy;
round = (unsigned char *)dest2;
if (memcmp(orig, round, size) != 0)
{
for(i = 0; i<size; ++i){
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n",
(int)i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
}
else { i = size; }
if (i == size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += (size * nchunks * niter * 10.);
aligned_free(src); free(srccpy); aligned_free(dest2);
for (i = 0; i < nchunks; i++) {
aligned_free(dest[i]);
}
}
static PyObject *
compress_helper(void * input, size_t nbytes, size_t typesize,
int clevel, int shuffle, char *cname){
int cbytes, blocksize, nthreads;
PyObject *output;
char *output_ptr;
PyThreadState *_save = NULL;
/* Alloc memory for compression */
if (!(output = PyBytes_FromStringAndSize(NULL, nbytes+BLOSC_MAX_OVERHEAD)))
return NULL;
/* Select compressor */
if (blosc_set_compressor(cname) < 0) {
/* The compressor is not available (should never happen here) */
blosc_error(-1, "compressor not available");
Py_DECREF(output);
return NULL;
}
/* Compress */
// This macro probably doesn't require the Python interpreter but let's leave it outside for safety
output_ptr = PyBytes_AS_STRING(output);
if( RELEASEGIL )
{
// Run with GIL released, tiny overhead penalty from this (although it
// may be significant for smaller chunks.)
_save = PyEval_SaveThread();
blocksize = blosc_get_blocksize();
// if blocksize==0, blosc_compress_ctx will try to auto-optimize it.
nthreads = blosc_get_nthreads();
cbytes = blosc_compress_ctx(clevel, shuffle, typesize, nbytes,
input, output_ptr, nbytes+BLOSC_MAX_OVERHEAD,
cname, blocksize, nthreads);
PyEval_RestoreThread(_save);
_save = NULL;
}
else
{ // Hold onto the Python GIL while compressing
cbytes = blosc_compress(clevel, shuffle, typesize, nbytes,
input, output_ptr,
nbytes+BLOSC_MAX_OVERHEAD);
}
if (cbytes < 0) {
blosc_error(cbytes, "while compressing data");
Py_DECREF(output);
return NULL;
}
/* Attempt to resize, if it's much smaller, a copy is required. */
if (_PyBytes_Resize(&output, cbytes) < 0){
/* the memory exception will have been set, hopefully */
Py_DECREF(output);
return NULL;
}
return output;
}
