void sz_getvardata_double_(char* varName, int *len, double* data)
{
int i;
size_t r1, r2, r3, r4, r5;
char s2[*len+1];
for(i=0;i<*len;i++)
s2[i]=varName[i];
s2[*len]='\0';
double* tmp_data = (double*)SZ_getVarData(s2, &r5, &r4, &r3, &r2, &r1);
int size = computeDataLength(r5, r4, r3, r2, r1);
memcpy(data, tmp_data, size*sizeof(double));
//free(tmp_data);
}
int SZ_compress_args_float_wRngeNoGzip(unsigned char** newByteData, float *oriData,
int r5, int r4, int r3, int r2, int r1, int *outSize,
int errBoundMode, double absErr_Bound, double rel_BoundRatio)
{
int status = SZ_SCES;
int dataLength = computeDataLength(r5,r4,r3,r2,r1);
float valueRangeSize = 0, medianValue = 0;
float min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
float max = min+valueRangeSize;
double realPrecision = getRealPrecision_float(valueRangeSize, errBoundMode, absErr_Bound, rel_BoundRatio, &status);
if(valueRangeSize <= realPrecision)
{
SZ_compress_args_float_withinRange(newByteData, oriData, dataLength, outSize);
}
else
{
// SZ_compress_args_float_NoCkRngeNoGzip_2D(newByteData, oriData, r2, r1, realPrecision, outSize);
if(r5==0&&r4==0&&r3==0&&r2==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(newByteData, oriData, r1, outSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_1D(newByteData, oriData, r1, realPrecision, outSize, valueRangeSize, medianValue);
}
else if(r5==0&&r4==0&&r3==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_2D_pwr(newByteData, oriData, r2, r1, outSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_2D(newByteData, oriData, r2, r1, realPrecision, outSize, valueRangeSize, medianValue);
}
else if(r5==0&&r4==0)
SZ_compress_args_float_NoCkRngeNoGzip_3D(newByteData, oriData, r3, r2, r1, realPrecision, outSize, valueRangeSize, medianValue);
else if(r5==0)
SZ_compress_args_float_NoCkRngeNoGzip_3D(newByteData, oriData, r4*r3, r2, r1, realPrecision, outSize, valueRangeSize, medianValue);
}
return status;
}
/**
*
*
* @return status SUCCESSFUL (SZ_SCES) or not (other error codes)
* */
int SZ_decompress_args_float(float** newData, int r5, int r4, int r3, int r2, int r1, unsigned char* cmpBytes, int cmpSize)
{
int status = SZ_SCES;
int dataLength = computeDataLength(r5,r4,r3,r2,r1);
//unsigned char* tmpBytes;
int targetUncompressSize = dataLength <<2; //i.e., *4
//tmpSize must be "much" smaller than dataLength
int tmpSize = 12, i;
unsigned char* szTmpBytes;
if(cmpSize!=12)
{
int isZlib = isZlibFormat(cmpBytes[0], cmpBytes[1]);
if(isZlib)
szMode = SZ_BEST_COMPRESSION;
else
szMode = SZ_BEST_SPEED;
if(szMode==SZ_BEST_SPEED)
{
tmpSize = cmpSize;
szTmpBytes = cmpBytes;
}
else if(szMode==SZ_BEST_COMPRESSION || szMode==SZ_DEFAULT_COMPRESSION)
{
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = zlib_uncompress2(cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize);
//szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize);
//memcpy(szTmpBytes, tmpBytes, tmpSize);
//free(tmpBytes); //release useless memory
}
else
{
printf("Wrong value of szMode in the double compressed bytes.\n");
status = SZ_MERR;
return status;
}
}
else
szTmpBytes = cmpBytes;
//TODO: convert szTmpBytes to data array.
TightDataPointStorageF* tdps;
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize);
//writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt");
int dim = computeDimension(r5,r4,r3,r2,r1);
int floatSize = sizeof(float);
if(tdps->isLossless)
{
*newData = (float*)malloc(floatSize*dataLength);
if(sysEndianType==BIG_ENDIAN_SYSTEM)
{
memcpy(*newData, szTmpBytes+8, dataLength*floatSize);
}
else
{
unsigned char* p = szTmpBytes+8;
for(i=0;i<dataLength;i++,p+=floatSize)
(*newData)[i] = bytesToFloat(p);
}
}
else if (dim == 1)
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode);
else
if (dim == 2)
getSnapshotData_float_2D(newData,r2,r1,tdps, errBoundMode);
else
if (dim == 3)
getSnapshotData_float_3D(newData,r3,r2,r1,tdps, errBoundMode);
else
if (dim == 4)
getSnapshotData_float_3D(newData,r4*r3,r2,r1,tdps, errBoundMode);
else
{
printf("Error: currently support only at most 4 dimensions!\n");
status = SZ_DERR;
}
free_TightDataPointStorageF(tdps);
if(szMode!=SZ_BEST_SPEED && cmpSize!=12)
free(szTmpBytes);
SZ_ReleaseHuffman();
return status;
}
int SZ_compress_args_float(unsigned char** newByteData, float *oriData,
int r5, int r4, int r3, int r2, int r1,
int *outSize,
int errBoundMode, double absErr_Bound, double relBoundRatio)
{
int status = SZ_SCES;
int dataLength = computeDataLength(r5,r4,r3,r2,r1);
float valueRangeSize = 0, medianValue = 0;
float min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
float max = min+valueRangeSize;
double realPrecision = getRealPrecision_float(valueRangeSize, errBoundMode, absErr_Bound, relBoundRatio, &status);
if(valueRangeSize <= realPrecision)
{
SZ_compress_args_float_withinRange(newByteData, oriData, dataLength, outSize);
}
else
{
int tmpOutSize = 0;
unsigned char* tmpByteData;
if (r2==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(&tmpByteData, oriData, r1, &tmpOutSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
}
else
if (r3==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_2D_pwr(&tmpByteData, oriData, r2, r1, &tmpOutSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_2D(&tmpByteData, oriData, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
}
else
if (r4==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_3D_pwr(&tmpByteData, oriData, r3, r2, r1, &tmpOutSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_3D(&tmpByteData, oriData, r3, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
}
else
if (r5==0)
{
if(errBoundMode==PW_REL)
SZ_compress_args_float_NoCkRngeNoGzip_3D_pwr(&tmpByteData, oriData, r4*r3, r2, r1, &tmpOutSize, min, max);
else
SZ_compress_args_float_NoCkRngeNoGzip_3D(&tmpByteData, oriData, r4*r3, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
}
else
{
printf("Error: doesn't support 5 dimensions for now.\n");
status = SZ_DERR; //dimension error
}
//Call Gzip to do the further compression.
if(szMode==SZ_BEST_SPEED)
{
*outSize = tmpOutSize;
*newByteData = tmpByteData;
}
else if(szMode==SZ_BEST_COMPRESSION || szMode==SZ_DEFAULT_COMPRESSION)
{
*outSize = (int)zlib_compress2(tmpByteData, tmpOutSize, newByteData, gzipMode);
free(tmpByteData);
}
else
{
printf("Error: Wrong setting of szMode in the float compression.\n");
status = SZ_MERR; //mode error
}
}
SZ_ReleaseHuffman();
return status;
}
static size_t H5Z_filter_sz(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t* buf_size, void** buf)
{
// printf("start in H5Z_filter_sz\n");
//H5Z_SZ_Init_Default();
size_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0;
int dimSize = 0, dataType = 0;
if(cd_nelmts==0) //this is special data such as string, which should not be treated as values.
return nbytes;
SZ_cdArrayToMetaData(cd_nelmts, cd_values, &dimSize, &dataType, &r5, &r4, &r3, &r2, &r1);
/* int i=0;
for(i=0;i<cd_nelmts;i++)
printf("cd_values[%d]=%u\n", i, cd_values[i]);
printf("dimSize=%d, r1=%u, r2=%u, r3=%u, r4=%u, r5=%u\n", dimSize, r1, r2, r3, r4, r5);*/
size_t nbEle = computeDataLength(r5, r4, r3, r2, r1);
if (flags & H5Z_FLAG_REVERSE)
{
//cost_start();
/* decompress data */
if(dataType == SZ_FLOAT)//==0
{
float* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(float);
}
else if(dataType == SZ_DOUBLE)//==1
{
double* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(double);
}
else if(dataType == SZ_INT8)
{
char* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(char);
}
else if(dataType == SZ_UINT8)
{
unsigned char* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(unsigned char);
}
else if(dataType == SZ_INT16)
{
short* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(short);
}
else if(dataType == SZ_UINT16)
{
unsigned short* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(unsigned short);
}
else if(dataType == SZ_INT32)
{
int* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(int);
}
else if(dataType == SZ_UINT32)
{
unsigned int* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(unsigned int);
}
else if(dataType == SZ_INT64)
{
long* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(long);
}
else if(dataType == SZ_UINT64)
{
unsigned long* data = SZ_decompress(dataType, *buf, nbytes, r5, r4, r3, r2, r1);
free(*buf);
*buf = data;
*buf_size = nbEle*sizeof(unsigned long);
}
else
{
printf("Decompression error: unknown data type: %d\n", dataType);
exit(0);
}
//cost_end();
//printf("decompression time = %lf, decompression rate = %lf\n", totalCost, 1.0*nbEle*sizeof(float)/totalCost);
}
else
{
size_t outSize = 0;
//printf("r5=%d, r4=%d, r3=%d, r2=%d, r1=%d, dataType=%d\n", r5, r4, r3, r2, r1, dataType);
//cost_start();
if(dataType == SZ_FLOAT)//==0
{
float* data = (float*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_DOUBLE)//==1
{
double* data = (double*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_INT8)
{
char* data = (char*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_UINT8)
{
unsigned char* data = (unsigned char*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_INT16)
{
short* data = (short*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_UINT16)
{
unsigned short* data = (unsigned short*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_INT32)
{
int* data = (int*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_UINT32)
{
unsigned int* data = (unsigned int*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_INT64)
{
long* data = (long*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else if(dataType == SZ_UINT64)
{
unsigned long* data = (unsigned long*)(*buf);
unsigned char *bytes = SZ_compress(dataType, data, &outSize, r5, r4, r3, r2, r1);
free(*buf);
*buf = bytes;
*buf_size = outSize;
}
else
{
printf("Compression error: unknown data type: %d\n", dataType);
exit(0);
}
//cost_end();
//printf("compression time = %lf, compression rate = %lf\n", totalCost, 1.0*nbEle*sizeof(float)/totalCost);
}
H5Z_SZ_Finalize();
return *buf_size;
}
int main(int argc, char * argv[])
{
int i = 0;
size_t r5=0,r4=0,r3=0,r2=0,r1=0;
char oriDir[640], outputDir[640], outputFilePath[600];
char *cfgFile;
if(argc < 3)
{
printf("Test case: testfloat_compress_ts [config_file] [srcDir] [dimension sizes...]\n");
printf("Example: testfloat_compress_ts sz.config /home/sdi/Data/Hurricane-ISA/consecutive-steps 500 500 100\n");
exit(0);
}
cfgFile=argv[1];
sprintf(oriDir, "%s", argv[2]);
if(argc>=4)
r1 = atoi(argv[3]); //8
if(argc>=5)
r2 = atoi(argv[4]); //8
if(argc>=6)
r3 = atoi(argv[5]); //128
if(argc>=7)
r4 = atoi(argv[6]);
if(argc>=8)
r5 = atoi(argv[7]);
printf("cfgFile=%s\n", cfgFile);
int status = SZ_Init(cfgFile);
if(status == SZ_NSCS)
exit(0);
sprintf(outputDir, "%s", oriDir);
char oriFilePath[600];
size_t nbEle;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
float *data = (float*)malloc(sizeof(float)*dataLength);
SZ_registerVar("xx", SZ_FLOAT, data, REL, 0, 0.0001, 0, r5, r4, r3, r2, r1);
if(status != SZ_SCES)
{
printf("Error: data file %s cannot be read!\n", oriFilePath);
exit(0);
}
size_t outSize;
unsigned char *bytes = NULL;
for(i=1;i<83;i++)
{
printf("simulation time step %d\n", i);
sprintf(oriFilePath, "%s/exalt-%d-2.dat", oriDir, i);
printf("processing file: %s\n", oriFilePath);
float *data_ = readFloatData(oriFilePath, &nbEle, &status);
memcpy(data, data_, nbEle*sizeof(float));
cost_start();
SZ_compress_ts(&bytes, &outSize);
cost_end();
printf("timecost=%f\n",totalCost);
sprintf(outputFilePath, "%s/exalt-%d-2.dat.sz2", outputDir, i);
printf("writing compressed data to %s\n", outputFilePath);
writeByteData(bytes, outSize, outputFilePath, &status);
free(bytes);
free(data_);
}
printf("done\n");
free(data);
SZ_Finalize();
return 0;
}
