/* Count up the zeros in the row, no zeros returned means no zero mapping */ int count_zeros(int ncols, float* row_data, function* parent) { int zeros_count=0; int i; float min=row_data[0]; float max=row_data[0]; function subroutine; set_function_name(__func__, &subroutine, parent); /* Find out if we need to use zero bitmaps */ for (i=0;i<ncols;i++) { if(row_data[i]==0) { zeros_count++; } else { if (min>row_data[i]) min=row_data[i]; if (max<row_data[i]) max=row_data[i]; } } /* Just making up an algorithm about whether to store data with zeros mapped out. Since I don't know the accuracy, take it as 50-50 chance if the range is sqrt(2) bigger including zeros than excluding them and this is good enough */ if (zeros_count>0 && min>0.0) { if ((max-min)>(max/sqrt(2))) zeros_count=0; /* turn off zero packing */ } return zeros_count; }
/* rle_decode(byte_array, lbrow, lbnpt, mdi) */ static PyObject *rle_decode_py(PyObject *self, PyObject *args) { char *bytes_in=NULL; PyArrayObject *npy_array_out=NULL; int bytes_in_len; npy_intp dims[2]; int lbrow, lbnpt, npts; float mdi; if (!PyArg_ParseTuple(args, "s#iif", &bytes_in, &bytes_in_len, &lbrow, &lbnpt, &mdi)) return NULL; // Unpacking algorithm accepts an int - so assert that lbrow*lbnpt does not overflow if (lbrow > 0 && lbnpt >= INT_MAX / (lbrow+1)) { PyErr_SetString(PyExc_ValueError, "Resulting unpacked PP field is larger than PP supports."); return NULL; } else { npts = lbnpt*lbrow; } // We can't use the macros Py_BEGIN_ALLOW_THREADS / Py_END_ALLOW_THREADS // because they declare a new scope block, but we want multiple exits. PyThreadState *_save; _save = PyEval_SaveThread(); float *dataout = (float*)calloc(npts, sizeof(float)); if (dataout == NULL) { PyEval_RestoreThread(_save); PyErr_SetString(PyExc_ValueError, "Unable to allocate memory for wgdos_unpacking."); return NULL; } function func; // function is defined by wgdosstuff. set_function_name(__func__, &func, 0); int status = unpack_ppfield(mdi, (bytes_in_len/BYTES_PER_INT_UNPACK_PPFIELD), bytes_in, LBPACK_RLE_PACKED, npts, dataout, &func); /* Raise an exception if there was a problem with the REL algorithm */ if (status != 0) { free(dataout); PyEval_RestoreThread(_save); PyErr_SetString(PyExc_ValueError, "RLE decode encountered an error."); return NULL; } else { /* The data came back fine, so make a Numpy array and return it */ dims[0]=lbrow; dims[1]=lbnpt; PyEval_RestoreThread(_save); npy_array_out=(PyArrayObject *) PyArray_SimpleNewFromData(2, dims, NPY_FLOAT, dataout); if (npy_array_out == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to make the numpy array for the packed data."); return NULL; } // give ownership of dataout to the Numpy array - Numpy will then deal with memory cleanup. npy_array_out->flags = npy_array_out->flags | NPY_OWNDATA; return (PyObject *)npy_array_out; } }
int wgdos_decode_field_parameters ( /* IN */ char** data, /* address of PP field to read from */ int unpacked_len, /* Expected length that data should expand to when */ /* unpacked, >=0 */ /* OUT */ float *accuracy, /* Absolute accuracy to which data held in field */ int *ncols, /* Number of columns in field, >=0 */ int *nrows, /* Number of rows in field, >=0 */ const function* const parent ) { int log_2_acc; /* Log to base 2 of accuracy */ int status=0; function subroutine; set_function_name(__func__, &subroutine, parent); typedef struct wgdos_field_header { uint32_t total_length; uint32_t precision; uint16_t pts_in_row; uint16_t rows_in_field; } wgdos_field_header; wgdos_field_header* field_header_pointer; wgdos_field_header field_header; *accuracy = 0.0; *ncols = 0; *nrows = 0; field_header_pointer=(wgdos_field_header*)*data; field_header.total_length=ntohl(field_header_pointer->total_length); field_header.precision=ntohl(field_header_pointer->precision); field_header.pts_in_row=ntohs(field_header_pointer->pts_in_row); field_header.rows_in_field=ntohs(field_header_pointer->rows_in_field); log_2_acc=field_header.precision; *accuracy=pow(2, log_2_acc); *ncols=field_header.pts_in_row; *nrows=field_header.rows_in_field; /* read ncols, nrows */ if (*ncols <= 0) { printf("zero/negative ncols\n"); status=-1; } else if (*nrows <= 0) { printf("zero/negative nrows\n"); status = -1; } else if (*nrows * *ncols != unpacked_len) { printf("size inconsistent %d * %d != %d\n", *nrows, *ncols, unpacked_len); status = -1; } #ifdef DEBUG snprintf(message, MAX_MESSAGE_SIZE, "WGDOS_decode_field_parameters returned total length %d, precision %d, row length %d and rows %d", field_header.total_length, field_header.precision, field_header.pts_in_row, field_header.rows_in_field); MO_syslog(VERBOSITY_MESSAGE, message, &subroutine); #endif return status; }
int wgdos_expand_row_to_data( int ncols, float mdi, float accuracy, float base, Boolean *missing_data, Boolean *zero, int *data, float *unpacked_data, int *mdi_clashes, const function* const parent ) { int non_special_so_far; /* Number of non-special items unpacked so far */ int col; /* Number of items unpacked so far */ int off=1; function subroutine; int log_messages=(get_verbosity()>=VERBOSITY_MESSAGE); set_function_name(__func__, &subroutine, parent); double dacc, ddata, dbase, dval; *mdi_clashes = 0; non_special_so_far = 0; dacc=accuracy; dbase=base; message[0]=0; for ( col = 0; col < ncols; col ++ ) { off=1; if ( missing_data[col] ) { unpacked_data[col] = mdi; snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12s", unpacked_data[col],"MDI"); } else if ( zero[col] ) { unpacked_data[col] = 0.0; snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12s", unpacked_data[col],"Zero"); } else { dval=dacc*data[non_special_so_far]+dbase; unpacked_data[col] = dval; if ( unpacked_data[col] == mdi ) { (*mdi_clashes)++; } non_special_so_far++; off=1; snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %012g / %-12d", unpacked_data[col],data[non_special_so_far-off]); } if (log_messages &&(col%4 == 3)) { snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col); MO_syslog (VERBOSITY_MESSAGE, message, &subroutine); message[0]=0; } } if (message[0]!=0 && log_messages) { snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col); MO_syslog (VERBOSITY_MESSAGE, message, &subroutine); message[0]=0; } return 0; } /* end function PP_Wgdos_Expand_Row */
/* wgdos_unpack(byte_array, lbrow, lbnpt, mdi) */ static PyObject *wgdos_unpack_py(PyObject *self, PyObject *args) { char *bytes_in=NULL; PyArrayObject *npy_array_out=NULL; int bytes_in_len; npy_intp dims[2]; int lbrow, lbnpt, npts; float mdi; if (!PyArg_ParseTuple(args, "s#iif", &bytes_in, &bytes_in_len, &lbrow, &lbnpt, &mdi)) return NULL; // Unpacking algorithm accepts an int - so assert that lbrow*lbnpt does not overflow if (lbrow > 0 && lbnpt >= INT_MAX / (lbrow+1)) { PyErr_SetString(PyExc_ValueError, "Resulting unpacked PP field is larger than PP supports."); return NULL; } else{ npts = lbnpt*lbrow; } /* Do the unpack of the given byte array */ float *dataout = (float*)calloc(npts, sizeof(float)); if (dataout == NULL) { PyErr_SetString(PyExc_ValueError, "Unable to allocate memory for wgdos_unpacking."); return NULL; } function func; // function is defined by wgdosstuff. set_function_name(__func__, &func, 0); int status = unpack_ppfield(mdi, 0, bytes_in, LBPACK_WGDOS_PACKED, npts, dataout, &func); /* Raise an exception if there was a problem with the WGDOS algorithm */ if (status != 0) { free(dataout); PyErr_SetString(PyExc_ValueError, "WGDOS unpack encountered an error."); return NULL; } else { /* The data came back fine, so make a Numpy array and return it */ dims[0]=lbrow; dims[1]=lbnpt; npy_array_out=(PyArrayObject *) PyArray_SimpleNewFromData(2, dims, NPY_FLOAT, dataout); if (npy_array_out == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to make the numpy array for the packed data."); return NULL; } // give ownership of dataout to the Numpy array - Numpy will then deal with memory cleanup. npy_array_out->flags = npy_array_out->flags | NPY_OWNDATA; return (PyObject *)npy_array_out; } }
int wgdos_decode_row_parameters( char** data, float* base, Boolean *missing_data_present, Boolean *zeros_bitmap_present, int *bits_per_value, int *nop, const function* const parent ) { int int2_pair[2], int2_temp; int baselen = 1; int status=0; union { int i; float f; } basetemp; function subroutine; set_function_name(__func__, &subroutine, parent); /* Read base value for row */ /* Use the union to change byteorder of basetemp value: needs an integer, so use the int version of basetemp */ memcpy(&basetemp, *data, 4); basetemp.i = ntohl(basetemp.i); basetemp.i = ntohl(*(int*)*data); /* And change the IBM float to an IEEE float: uses the float version of basetemp */ status=convert_float_ibm_to_ieee32(&basetemp.i, (int*)base, &baselen); if (status <0) { MO_syslog(VERBOSITY_ERROR, "IEEE/IBM float conversion failed", &subroutine); } *data = *data + 4; /* Read bits_per_value and flags */ int2_temp =ntohl(*((int *)*data)); *nop=int2_temp%65536; int2_pair[0] = int2_temp >> 16; *zeros_bitmap_present = ((int2_pair[0] & 128) != 0); /* 8th bit: zero's present? */ *missing_data_present = ((int2_pair[0] & 32) != 0); /* 6th bit: MDI's present? */ *bits_per_value = int2_pair[0] & 31; /* Lowest 5 bits: bits per value (<32) */ *data = *data + 4; snprintf(message, MAX_MESSAGE_SIZE, "Decoded BitFlags Zero:%d, MDI: %d. %d bits per value base value %f, %d words taken", *zeros_bitmap_present, *missing_data_present, *bits_per_value, *base, *nop); MO_syslog(VERBOSITY_MESSAGE, message, &subroutine); return status; } /* end function PP_Wgdos_Start_Row */
int unpack_ppfield(float mdi, int data_size, char* data, int pack, int unpacked_size, float* to, function* parent) { float* unpacked; int* ip_in; int* ip_out; int count; function subroutine; set_function_name(__func__, &subroutine, parent); unpacked=malloc(unpacked_size*sizeof(float)); snprintf(message, MAX_MESSAGE_SIZE, "MDI %f", mdi); MO_syslog(VERBOSITY_INFO, message, &subroutine); switch(pack) { case 0: MO_syslog(VERBOSITY_INFO, "Unpacked data", &subroutine); ip_in=(int*)data; ip_out=(int*)unpacked; for (count=0; count<data_size; count++) { ip_out[count]=htonl(ip_in[count]); } break; case 1: MO_syslog(VERBOSITY_INFO, "WGDOS packed data", &subroutine); if (wgdos_unpack(data, unpacked_size, unpacked, mdi, parent)) { MO_syslog(VERBOSITY_INFO, "wgdos_unpack Failed", &subroutine); free(unpacked); return 1; } break; case 4: MO_syslog(VERBOSITY_INFO, "RLE packed data", &subroutine); ip_in=(int*)data; ip_out=(int*)data; for (count=0; count<data_size; count++) { ip_out[count]=htonl(ip_in[count]); } if (runlen_decode(unpacked, unpacked_size, (float*)data, data_size, mdi, &subroutine)) { MO_syslog(VERBOSITY_INFO, "runlen_decode Failed", &subroutine); free(unpacked); return 1; } break; default: MO_syslog(VERBOSITY_ERROR, "Unrecognised packing code", &subroutine); return 1; } if (to!=NULL) { memcpy(to, unpacked, unpacked_size*sizeof(float)); } free(unpacked); return 0; }
int byteorder_data_unpack_ppfield(float mdi, int data_size, char* data, int network_order_in, int pack, int unpacked_size, float* to, int network_order_out, function* parent) { int retval=0; int* ip_in; int* ip_out; int count; char* newdata=NULL; int temp; function subroutine; set_function_name(__func__, &subroutine, parent); #if __BYTE_ORDER == __LITTLE_ENDIAN if (network_order_in) { switch (pack) { case 1: newdata=malloc(data_size*sizeof(int)); ip_in=(int*)data; ip_out=(int*)newdata; for (count=0; count<data_size; count++) { ip_out[count]=ntohl(ip_in[count]); } break; default: newdata=data; } } #endif retval=unpack_ppfield(mdi, data_size, newdata, pack, unpacked_size, to, &subroutine); #if __BYTE_ORDER == __LITTLE_ENDIAN if (network_order_out && (to!=NULL)) { ip_in=(int*)to; ip_out=(int*)to; for (count=0; count<unpacked_size; count++) { ip_out[count]=htonl(ip_in[count]); } } #endif if (newdata!=data) { free (newdata); } return retval; }
int unpack_ppfield32(uint32_t* lookup, char* data, float* to, function* parent) { int unpacked_size; int data_size; int pack; float mdi; int ret; function subroutine; set_function_name(__func__, &subroutine, parent); mdi=*(float*)(lookup+MDI); unpacked_size=lookup[NROWS]*lookup[NCOLS]; data_size = lookup[FIELD_LENGTH] - lookup[EXT]; pack=lookup[PACK] % 10; ret=unpack_ppfield(mdi, data_size, data, pack, unpacked_size, to, parent); lookup[FIELD_LENGTH]=unpacked_size + lookup[EXT]; lookup[PACK]=0; return (ret); }
int byteorder_unpack_ppfield(int* lookup, char* data, int network_order_in, float* to, int network_order_out, function* parent) { int retval=0; int unpacked_size; int data_size; int pack; float mdi; int* ip_in; int* ip_out; int count; char* newdata=NULL; int temp; function subroutine; set_function_name(__func__, &subroutine, parent); mdi=*(float*)(lookup+MDI); unpacked_size=lookup[NROWS]*lookup[NCOLS]; data_size = lookup[FIELD_LENGTH] - lookup[EXT]; pack=lookup[PACK] % 10; byteorder_data_unpack_ppfield(mdi, data_size, data, network_order_in, pack, unpacked_size, to, network_order_out, &subroutine); }
int unpack_ppfield64(uint64_t* lookup, char* data, float* to, function* parent) { int unpacked_size; int data_size; int pack; float mdi; double dmdi; int ret; function subroutine; set_function_name(__func__, &subroutine, parent); if (sizeof(float) != 8) { dmdi=*(double*)(lookup+MDI); mdi=(float)dmdi; } else { mdi=*(float*)(lookup+MDI); } unpacked_size=lookup[NROWS]*lookup[NCOLS]; data_size = (lookup[FIELD_LENGTH] - lookup[EXT]); pack=lookup[PACK] % 10; ret=unpack_ppfield(mdi, data_size, data, pack, unpacked_size, to, parent); lookup[FIELD_LENGTH]=unpacked_size + lookup[EXT]; lookup[PACK]=0; return (ret); }
int wgdos_expand_broken_row_to_data( int ncols, int badnumbers, float mdi, float accuracy, float base, Boolean *missing_data, Boolean *zero, int *data, float *unpacked_data, int *mdi_clashes, const function* const parent ) { int non_special_so_far; /* Number of non-special items unpacked so far */ int col; /* Number of items unpacked so far */ int off=1; int log_messages=(get_verbosity()>=VERBOSITY_MESSAGE); function subroutine; set_function_name(__func__, &subroutine, parent); *mdi_clashes = 0; non_special_so_far = 0; message[0]=0; for ( col = 0; col < ncols; col ++ ) { off=1; if ( missing_data[col]) { unpacked_data[col] = mdi; } else if ( zero[col] ) { unpacked_data[col] = 0.0; } else { if (non_special_so_far >= badnumbers) { unpacked_data[col] = mdi; } else { unpacked_data[col] = accuracy * data[non_special_so_far] + base; if ( unpacked_data[col] == mdi ) { (*mdi_clashes)++; } non_special_so_far++; off=1; } } if (log_messages && (col%3 == 0)) { snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col); MO_syslog (VERBOSITY_MESSAGE, message, &subroutine); message[0]=0; } if (log_messages) { snprintf (message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), " %.16g/%05d", unpacked_data[col],data[non_special_so_far-off]); } } if (log_messages && message[0]!=0) { snprintf(message+strlen(message), MAX_MESSAGE_SIZE-strlen(message), "%3d", col); MO_syslog (VERBOSITY_MESSAGE, message, &subroutine); message[0]=0; } return 0; } /* end function PP_Wgdos_Expand_Broken_Row */