/* Count up the zeros in the row, no zeros returned means no zero mapping */
int count_zeros(int ncols, float* row_data, function* parent) {
int zeros_count=0;
int i;
float min=row_data[0];
float max=row_data[0];
function subroutine;
set_function_name(__func__, &subroutine, parent);
/* Find out if we need to use zero bitmaps */
for (i=0;i<ncols;i++) {
if(row_data[i]==0) {
zeros_count++;
} else {
if (min>row_data[i]) min=row_data[i];
if (max<row_data[i]) max=row_data[i];
}
}
/* Just making up an algorithm about whether to store data with zeros mapped out.
Since I don't know the accuracy, take it as 50-50 chance if the range is sqrt(2)
bigger including zeros than excluding them and this is good enough */
if (zeros_count>0 && min>0.0) {
if ((max-min)>(max/sqrt(2))) zeros_count=0; /* turn off zero packing */
}
return zeros_count;
}
/* rle_decode(byte_array, lbrow, lbnpt, mdi) */
static PyObject *rle_decode_py(PyObject *self, PyObject *args)
{
char *bytes_in=NULL;
PyArrayObject *npy_array_out=NULL;
int bytes_in_len;
npy_intp dims[2];
int lbrow, lbnpt, npts;
float mdi;
if (!PyArg_ParseTuple(args, "s#iif", &bytes_in, &bytes_in_len, &lbrow, &lbnpt, &mdi)) return NULL;
// Unpacking algorithm accepts an int - so assert that lbrow*lbnpt does not overflow
if (lbrow > 0 && lbnpt >= INT_MAX / (lbrow+1)) {
PyErr_SetString(PyExc_ValueError, "Resulting unpacked PP field is larger than PP supports.");
return NULL;
} else {
npts = lbnpt*lbrow;
}
// We can't use the macros Py_BEGIN_ALLOW_THREADS / Py_END_ALLOW_THREADS
// because they declare a new scope block, but we want multiple exits.
PyThreadState *_save;
_save = PyEval_SaveThread();
float *dataout = (float*)calloc(npts, sizeof(float));
if (dataout == NULL) {
PyEval_RestoreThread(_save);
PyErr_SetString(PyExc_ValueError, "Unable to allocate memory for wgdos_unpacking.");
return NULL;
}
function func; // function is defined by wgdosstuff.
set_function_name(__func__, &func, 0);
int status = unpack_ppfield(mdi, (bytes_in_len/BYTES_PER_INT_UNPACK_PPFIELD), bytes_in, LBPACK_RLE_PACKED, npts, dataout, &func);
/* Raise an exception if there was a problem with the REL algorithm */
if (status != 0) {
free(dataout);
PyEval_RestoreThread(_save);
PyErr_SetString(PyExc_ValueError, "RLE decode encountered an error.");
return NULL;
}
else {
/* The data came back fine, so make a Numpy array and return it */
dims[0]=lbrow;
dims[1]=lbnpt;
PyEval_RestoreThread(_save);
npy_array_out=(PyArrayObject *) PyArray_SimpleNewFromData(2, dims, NPY_FLOAT, dataout);
if (npy_array_out == NULL) {
PyErr_SetString(PyExc_ValueError, "Failed to make the numpy array for the packed data.");
return NULL;
}
// give ownership of dataout to the Numpy array - Numpy will then deal with memory cleanup.
npy_array_out->flags = npy_array_out->flags | NPY_OWNDATA;
return (PyObject *)npy_array_out;
}
}
int wgdos_decode_field_parameters (
/* IN */
char** data, /* address of PP field to read from */
int unpacked_len, /* Expected length that data should expand to when */
/* unpacked, >=0 */
/* OUT */
float *accuracy, /* Absolute accuracy to which data held in field */
int *ncols, /* Number of columns in field, >=0 */
int *nrows, /* Number of rows in field, >=0 */
const function* const parent
)
{
int log_2_acc; /* Log to base 2 of accuracy */
int status=0;
function subroutine;
set_function_name(__func__, &subroutine, parent);
typedef struct wgdos_field_header {
uint32_t total_length;
uint32_t precision;
uint16_t pts_in_row;
uint16_t rows_in_field;
} wgdos_field_header;
wgdos_field_header* field_header_pointer;
wgdos_field_header field_header;
*accuracy = 0.0;
*ncols = 0;
*nrows = 0;
field_header_pointer=(wgdos_field_header*)*data;
field_header.total_length=ntohl(field_header_pointer->total_length);
field_header.precision=ntohl(field_header_pointer->precision);
field_header.pts_in_row=ntohs(field_header_pointer->pts_in_row);
field_header.rows_in_field=ntohs(field_header_pointer->rows_in_field);
log_2_acc=field_header.precision;
*accuracy=pow(2, log_2_acc);
*ncols=field_header.pts_in_row;
*nrows=field_header.rows_in_field;
/* read ncols, nrows */
if (*ncols <= 0) {
printf("zero/negative ncols\n");
status=-1;
} else if (*nrows <= 0) {
printf("zero/negative nrows\n");
status = -1;
} else if (*nrows * *ncols != unpacked_len) {
printf("size inconsistent %d * %d != %d\n", *nrows, *ncols, unpacked_len);
status = -1;
}
#ifdef DEBUG
snprintf(message, MAX_MESSAGE_SIZE, "WGDOS_decode_field_parameters returned total length %d, precision %d, row length %d and rows %d",
field_header.total_length, field_header.precision, field_header.pts_in_row, field_header.rows_in_field);
MO_syslog(VERBOSITY_MESSAGE, message, &subroutine);
#endif
return status;
}
int wgdos_expand_row_to_data(
int ncols,
float mdi,
float accuracy,
float base,
Boolean *missing_data,
Boolean *zero,
int *data,
float *unpacked_data,
int *mdi_clashes,
const function* const parent
)
{
int non_special_so_far; /* Number of non-special items unpacked so far */
int col; /* Number of items unpacked so far */
int off=1;
function subroutine;
int log_messages=(get_verbosity()>=VERBOSITY_MESSAGE);
set_function_name(__func__, &subroutine, parent);
double dacc, ddata, dbase, dval;
*mdi_clashes = 0;
non_special_so_far = 0;
dacc=accuracy;
dbase=base;
message[0]=0;
for ( col = 0; col < ncols; col ++ ) {
off=1;
if ( missing_data[col] ) {
unpacked_data[col] = mdi;
snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12s", unpacked_data[col],"MDI");
} else if ( zero[col] ) {
unpacked_data[col] = 0.0;
snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12s", unpacked_data[col],"Zero");
} else {
dval=dacc*data[non_special_so_far]+dbase;
unpacked_data[col] = dval;
if ( unpacked_data[col] == mdi ) {
(*mdi_clashes)++;
}
non_special_so_far++;
off=1;
snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12d", unpacked_data[col],data[non_special_so_far-off]);
}
if (log_messages &&(col%4 == 3)) {
snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col);
MO_syslog (VERBOSITY_MESSAGE, message, &subroutine);
message[0]=0;
}
}
if (message[0]!=0 && log_messages) {
snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col);
MO_syslog (VERBOSITY_MESSAGE, message, &subroutine);
message[0]=0;
}
return 0;
} /* end function PP_Wgdos_Expand_Row */
/* wgdos_unpack(byte_array, lbrow, lbnpt, mdi) */
static PyObject *wgdos_unpack_py(PyObject *self, PyObject *args)
{
char *bytes_in=NULL;
PyArrayObject *npy_array_out=NULL;
int bytes_in_len;
npy_intp dims[2];
int lbrow, lbnpt, npts;
float mdi;
if (!PyArg_ParseTuple(args, "s#iif", &bytes_in, &bytes_in_len, &lbrow, &lbnpt, &mdi)) return NULL;
// Unpacking algorithm accepts an int - so assert that lbrow*lbnpt does not overflow
if (lbrow > 0 && lbnpt >= INT_MAX / (lbrow+1)) {
PyErr_SetString(PyExc_ValueError, "Resulting unpacked PP field is larger than PP supports.");
return NULL;
} else{
npts = lbnpt*lbrow;
}
/* Do the unpack of the given byte array */
float *dataout = (float*)calloc(npts, sizeof(float));
if (dataout == NULL) {
PyErr_SetString(PyExc_ValueError, "Unable to allocate memory for wgdos_unpacking.");
return NULL;
}
function func; // function is defined by wgdosstuff.
set_function_name(__func__, &func, 0);
int status = unpack_ppfield(mdi, 0, bytes_in, LBPACK_WGDOS_PACKED, npts, dataout, &func);
/* Raise an exception if there was a problem with the WGDOS algorithm */
if (status != 0) {
free(dataout);
PyErr_SetString(PyExc_ValueError, "WGDOS unpack encountered an error.");
return NULL;
}
else {
/* The data came back fine, so make a Numpy array and return it */
dims[0]=lbrow;
dims[1]=lbnpt;
npy_array_out=(PyArrayObject *) PyArray_SimpleNewFromData(2, dims, NPY_FLOAT, dataout);
if (npy_array_out == NULL) {
PyErr_SetString(PyExc_ValueError, "Failed to make the numpy array for the packed data.");
return NULL;
}
// give ownership of dataout to the Numpy array - Numpy will then deal with memory cleanup.
npy_array_out->flags = npy_array_out->flags | NPY_OWNDATA;
return (PyObject *)npy_array_out;
}
}
int wgdos_decode_row_parameters(
char** data,
float* base,
Boolean *missing_data_present,
Boolean *zeros_bitmap_present,
int *bits_per_value,
int *nop,
const function* const parent
)
{
int int2_pair[2], int2_temp;
int baselen = 1;
int status=0;
union {
int i;
float f;
} basetemp;
function subroutine;
set_function_name(__func__, &subroutine, parent);
/* Read base value for row */
/* Use the union to change byteorder of basetemp value:
needs an integer, so use the int version of basetemp */
memcpy(&basetemp, *data, 4);
basetemp.i = ntohl(basetemp.i);
basetemp.i = ntohl(*(int*)*data);
/* And change the IBM float to an IEEE float: uses the float version of basetemp */
status=convert_float_ibm_to_ieee32(&basetemp.i, (int*)base, &baselen);
if (status <0) {
MO_syslog(VERBOSITY_ERROR, "IEEE/IBM float conversion failed", &subroutine);
}
*data = *data + 4;
/* Read bits_per_value and flags */
int2_temp =ntohl(*((int *)*data));
*nop=int2_temp%65536;
int2_pair[0] = int2_temp >> 16;
*zeros_bitmap_present = ((int2_pair[0] & 128) != 0); /* 8th bit: zero's present? */
*missing_data_present = ((int2_pair[0] & 32) != 0); /* 6th bit: MDI's present? */
*bits_per_value = int2_pair[0] & 31; /* Lowest 5 bits: bits per value (<32) */
*data = *data + 4;
snprintf(message, MAX_MESSAGE_SIZE, "Decoded BitFlags Zero:%d, MDI: %d. %d bits per value base value %f, %d words taken",
*zeros_bitmap_present, *missing_data_present, *bits_per_value, *base, *nop);
MO_syslog(VERBOSITY_MESSAGE, message, &subroutine);
return status;
} /* end function PP_Wgdos_Start_Row */
int unpack_ppfield(float mdi, int data_size, char* data, int pack, int unpacked_size, float* to, function* parent) {
float* unpacked;
int* ip_in;
int* ip_out;
int count;
function subroutine;
set_function_name(__func__, &subroutine, parent);
unpacked=malloc(unpacked_size*sizeof(float));
snprintf(message, MAX_MESSAGE_SIZE, "MDI %f", mdi);
MO_syslog(VERBOSITY_INFO, message, &subroutine);
switch(pack) {
case 0:
MO_syslog(VERBOSITY_INFO, "Unpacked data", &subroutine);
ip_in=(int*)data;
ip_out=(int*)unpacked;
for (count=0; count<data_size; count++) {
ip_out[count]=htonl(ip_in[count]);
}
break;
case 1:
MO_syslog(VERBOSITY_INFO, "WGDOS packed data", &subroutine);
if (wgdos_unpack(data, unpacked_size, unpacked, mdi, parent)) {
MO_syslog(VERBOSITY_INFO, "wgdos_unpack Failed", &subroutine);
free(unpacked);
return 1;
}
break;
case 4:
MO_syslog(VERBOSITY_INFO, "RLE packed data", &subroutine);
ip_in=(int*)data;
ip_out=(int*)data;
for (count=0; count<data_size; count++) {
ip_out[count]=htonl(ip_in[count]);
}
if (runlen_decode(unpacked, unpacked_size, (float*)data, data_size, mdi, &subroutine)) {
MO_syslog(VERBOSITY_INFO, "runlen_decode Failed", &subroutine);
free(unpacked);
return 1;
}
break;
default:
MO_syslog(VERBOSITY_ERROR, "Unrecognised packing code", &subroutine);
return 1;
}
if (to!=NULL) {
memcpy(to, unpacked, unpacked_size*sizeof(float));
}
free(unpacked);
return 0;
}
int byteorder_data_unpack_ppfield(float mdi, int data_size, char* data, int network_order_in,
int pack, int unpacked_size, float* to, int network_order_out,
function* parent) {
int retval=0;
int* ip_in;
int* ip_out;
int count;
char* newdata=NULL;
int temp;
function subroutine;
set_function_name(__func__, &subroutine, parent);
#if __BYTE_ORDER == __LITTLE_ENDIAN
if (network_order_in) {
switch (pack) {
case 1:
newdata=malloc(data_size*sizeof(int));
ip_in=(int*)data;
ip_out=(int*)newdata;
for (count=0; count<data_size; count++) {
ip_out[count]=ntohl(ip_in[count]);
}
break;
default:
newdata=data;
}
}
#endif
retval=unpack_ppfield(mdi, data_size, newdata, pack, unpacked_size, to, &subroutine);
#if __BYTE_ORDER == __LITTLE_ENDIAN
if (network_order_out && (to!=NULL)) {
ip_in=(int*)to;
ip_out=(int*)to;
for (count=0; count<unpacked_size; count++) {
ip_out[count]=htonl(ip_in[count]);
}
}
#endif
if (newdata!=data) {
free (newdata);
}
return retval;
}
int unpack_ppfield32(uint32_t* lookup, char* data, float* to, function* parent) {
int unpacked_size;
int data_size;
int pack;
float mdi;
int ret;
function subroutine;
set_function_name(__func__, &subroutine, parent);
mdi=*(float*)(lookup+MDI);
unpacked_size=lookup[NROWS]*lookup[NCOLS];
data_size = lookup[FIELD_LENGTH] - lookup[EXT];
pack=lookup[PACK] % 10;
ret=unpack_ppfield(mdi, data_size, data, pack, unpacked_size, to, parent);
lookup[FIELD_LENGTH]=unpacked_size + lookup[EXT];
lookup[PACK]=0;
return (ret);
}
int byteorder_unpack_ppfield(int* lookup, char* data, int network_order_in,
float* to, int network_order_out,
function* parent) {
int retval=0;
int unpacked_size;
int data_size;
int pack;
float mdi;
int* ip_in;
int* ip_out;
int count;
char* newdata=NULL;
int temp;
function subroutine;
set_function_name(__func__, &subroutine, parent);
mdi=*(float*)(lookup+MDI);
unpacked_size=lookup[NROWS]*lookup[NCOLS];
data_size = lookup[FIELD_LENGTH] - lookup[EXT];
pack=lookup[PACK] % 10;
byteorder_data_unpack_ppfield(mdi, data_size, data, network_order_in,
pack, unpacked_size, to, network_order_out,
&subroutine);
}
int unpack_ppfield64(uint64_t* lookup, char* data, float* to, function* parent) {
int unpacked_size;
int data_size;
int pack;
float mdi;
double dmdi;
int ret;
function subroutine;
set_function_name(__func__, &subroutine, parent);
if (sizeof(float) != 8) {
dmdi=*(double*)(lookup+MDI);
mdi=(float)dmdi;
} else {
mdi=*(float*)(lookup+MDI);
}
unpacked_size=lookup[NROWS]*lookup[NCOLS];
data_size = (lookup[FIELD_LENGTH] - lookup[EXT]);
pack=lookup[PACK] % 10;
ret=unpack_ppfield(mdi, data_size, data, pack, unpacked_size, to, parent);
lookup[FIELD_LENGTH]=unpacked_size + lookup[EXT];
lookup[PACK]=0;
return (ret);
}
int wgdos_expand_broken_row_to_data(
int ncols,
int badnumbers,
float mdi,
float accuracy,
float base,
Boolean *missing_data,
Boolean *zero,
int *data,
float *unpacked_data,
int *mdi_clashes,
const function* const parent
)
{
int non_special_so_far; /* Number of non-special items unpacked so far */
int col; /* Number of items unpacked so far */
int off=1;
int log_messages=(get_verbosity()>=VERBOSITY_MESSAGE);
function subroutine;
set_function_name(__func__, &subroutine, parent);
*mdi_clashes = 0;
non_special_so_far = 0;
message[0]=0;
for ( col = 0; col < ncols; col ++ ) {
off=1;
if ( missing_data[col]) {
unpacked_data[col] = mdi;
} else if ( zero[col] ) {
unpacked_data[col] = 0.0;
} else {
if (non_special_so_far >= badnumbers) {
unpacked_data[col] = mdi;
} else {
unpacked_data[col] = accuracy * data[non_special_so_far] + base;
if ( unpacked_data[col] == mdi ) {
(*mdi_clashes)++;
}
non_special_so_far++;
off=1;
}
}
if (log_messages && (col%3 == 0)) {
snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col);
MO_syslog (VERBOSITY_MESSAGE, message, &subroutine);
message[0]=0;
}
if (log_messages) {
snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %.16g/%05d", unpacked_data[col],data[non_special_so_far-off]);
}
}
if (log_messages && message[0]!=0) {
snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col);
MO_syslog (VERBOSITY_MESSAGE, message, &subroutine);
message[0]=0;
}
return 0;
} /* end function PP_Wgdos_Expand_Broken_Row */
