void summary_cut6(double_arr *peakList, hist_t *hist, double *summ)
{
cutSmallPeaks(peakList, 3.0);
gsl_sort(peakList->array, 1, peakList->length);
summ[0] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.649);
summ[1] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.809);
summ[2] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.888);
summ[3] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.930);
summ[4] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.955);
summ[5] = gsl_stats_quantile_from_sorted_data(peakList->array, 1, peakList->length, 0.970);
return;
}
void updateEpsilon_particle_arr(particle_arr *part, double *diffArr)
{
int p = part->p;
int i;
for (i=0; i<p; i++) diffArr[i] = part->array[i]->diff;
gsl_sort(diffArr, 1, p);
part->epsilon = gsl_stats_median_from_sorted_data(diffArr, 1, p);
printf("epsilon_median = %.5f\n", part->epsilon);
printf("Success rate = %.5f\n", p / (double)part->nbAttempts);
return;
}
// feature 03: Mean average deviation from median (in 2D xy)
int LSL_lfeatures_class :: feature_03(LSL_Point3D_container *laserfeat_cluster, Real *out)
{
char ret = 1;
double s_sum = 0;
LSL_Point3D_str pts_median;
std::vector <double> pts_coordx,pts_coordy;
// take x
laserfeat_cluster -> get_coords(pts_coordx, GEOMETRY_COORD_X);
// sort data
gsl_sort (&pts_coordx[0], 1, pts_coordx.size());
// median
pts_median.x = gsl_stats_median_from_sorted_data (&pts_coordx[0], 1, pts_coordx.size());
// take y
laserfeat_cluster -> get_coords(pts_coordy, GEOMETRY_COORD_Y);
// sort data
gsl_sort (&pts_coordy[0], 1, pts_coordy.size());
// median
pts_median.y = gsl_stats_median_from_sorted_data (&pts_coordy[0], 1, pts_coordy.size());
// project on xy plane
pts_median.z = 0;
for(unsigned int i = 0; i < laserfeat_cluster -> pts.size(); i++ )
{
double val = distance_L2_XY (&pts_median, &laserfeat_cluster -> pts[i]);
s_sum += val * val;
}
if(s_sum == 0)
ret = 0;
*out = sqrt( ( 1.0 / (double)laserfeat_cluster -> pts.size() ) * s_sum);
return(ret);
}
StatBox2D::BoxWhiskerData Layout2D::generateBoxWhiskerData(Column *colData,
int from, int to,
int key) {
size_t size = static_cast<size_t>((to - from) + 1);
double *data = new double[size];
for (int i = 0, j = from; j < to + 1; i++, j++) {
data[i] = colData->valueAt(i);
}
// sort the data
gsl_sort(data, 1, size - 1);
StatBox2D::BoxWhiskerData statBoxData;
statBoxData.key = key;
// basic stats
statBoxData.mean = gsl_stats_mean(data, 1, size);
statBoxData.median = gsl_stats_median_from_sorted_data(data, 1, size);
statBoxData.sd = gsl_stats_sd(data, 1, size);
statBoxData.se = statBoxData.sd / sqrt(static_cast<double>(size));
// data bounds
statBoxData.boxWhiskerDataBounds.sd_lower = statBoxData.mean - statBoxData.sd;
statBoxData.boxWhiskerDataBounds.sd_upper = statBoxData.mean + statBoxData.sd;
statBoxData.boxWhiskerDataBounds.se_lower = statBoxData.mean - statBoxData.se;
statBoxData.boxWhiskerDataBounds.se_upper = statBoxData.mean + statBoxData.se;
statBoxData.boxWhiskerDataBounds.perc_1 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.01);
statBoxData.boxWhiskerDataBounds.perc_5 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.05);
statBoxData.boxWhiskerDataBounds.perc_10 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.10);
statBoxData.boxWhiskerDataBounds.perc_25 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.25);
statBoxData.boxWhiskerDataBounds.perc_75 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.75);
statBoxData.boxWhiskerDataBounds.perc_90 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.90);
statBoxData.boxWhiskerDataBounds.perc_95 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.95);
statBoxData.boxWhiskerDataBounds.perc_99 =
gsl_stats_quantile_from_sorted_data(data, 1, size, 0.99);
statBoxData.boxWhiskerDataBounds.max = data[size - 1];
statBoxData.boxWhiskerDataBounds.min = data[0];
// delete the double data pointer
delete[] data;
return statBoxData;
}
static VALUE rb_gsl_sort_narray(VALUE obj)
{
struct NARRAY *na;
size_t size, stride;
double *ptr1, *ptr2;
VALUE ary;
GetNArray(obj, na);
ptr1 = (double*) na->ptr;
size = na->total;
stride = 1;
ary = na_make_object(NA_DFLOAT, na->rank, na->shape, CLASS_OF(obj));
ptr2 = NA_PTR_TYPE(ary, double*);
memcpy(ptr2, ptr1, sizeof(double)*size);
gsl_sort(ptr2, stride, size);
return ary;
}
double as_cv_fixed_params(double beta, double alpha, double s, int n, int num_reps)
{
double X[n];
double T[num_reps];
int i;
srand(time(0));
int k = rand() % 1000000;
double params[3] = {beta, alpha, s};
for (i = 0; i < num_reps; i++)
{
ar1_ts(X, params, n, k + i); // IID draws, because alpha=0
T[i] = as_test_stat(beta, alpha, s, X, n);
}
gsl_sort(T, 1, num_reps);
return gsl_stats_quantile_from_sorted_data(T, 1, num_reps, 0.95);
}
/*
* Rank data using the natural ordering on doubles, ties being resolved by taking their average.
*
* Source: http://commons.apache.org/math/apidocs/src-html/org/apache/commons/math/stat/ranking/NaturalRanking.html#line.190
*/
void rsSpearmannRank(double *ranks, const double *data, const size_t n) {
size_t i;
double *d = malloc(sizeof(double)*n);
size_t *p = malloc(sizeof(size_t)*n);
// copy the input data and sort them
for(i=0; i<n; ++i)
d[i] = data[i];
gsl_sort(d, 1, n);
// get the index of the input data as if they were sorted
gsl_sort_index(p, data, 1, n);
// walk the sorted array, filling output array using sorted positions, resolving ties as we go
size_t pos = 1;
ranks[p[0]] = pos;
size_t n_ties = 1;
size_t * tiesTrace = (size_t*) calloc (1, sizeof(size_t));
tiesTrace[0] = p[0];
for(i=1; i<n; ++i) {
if(d[i] - d[i-1] > 0) {
pos = i + 1;
if(n_ties > 1) {
rsRankingResolveTies(ranks, tiesTrace, n_ties);
}
tiesTrace = (size_t*) realloc(tiesTrace, sizeof(size_t));
n_ties = 1;
tiesTrace[0] = p[i];
} else {
++n_ties;
tiesTrace = (size_t*) realloc(tiesTrace, n_ties * sizeof(size_t));
tiesTrace[n_ties-1] = p[i];
}
ranks[p[i]] = pos;
}
if(n_ties > 1) {
rsRankingResolveTies(ranks, tiesTrace, n_ties);
}
free(tiesTrace);
free(d);
free(p);
}
void _fortetRHS(const double * ts, const int num_steps, double * Is, const int num_intervals,
const double * abgthphi,
double *rhs) {
const double alpha = abgthphi[0];
const double beta = abgthphi[1];
const double gamma = abgthphi[2];
const double theta = abgthphi[3];
const double phi = abgthphi[4];
const double TAU_TOLERANCE = 1e-5;
const double psi = atan(theta);
// sort is
gsl_sort(Is, 1, num_intervals);
double lt, lI, ltau, vthforward, vthbackward,numerator, denominator;
for (int tidx = 0; tidx < num_steps; ++tidx) {
lt = ts[tidx];
rhs[tidx] = .0;
vthforward = 1. - (alpha*(1 - exp(-lt)) +
gamma / sqrt(1.+theta*theta) * ( sin ( theta * ( lt + phi) - psi)
-exp(-lt)*sin(phi*theta - psi) ));
for (int idx = 0; idx < num_intervals; ++idx){
lI = Is[idx];
if(lI >= (lt - TAU_TOLERANCE)) break;
ltau = lt - lI;
vthbackward = 1. - (alpha*(1 - exp(-lI)) +
gamma / sqrt(1.+theta*theta) * ( sin ( theta * ( lI + phi) - psi)
-exp(-lI)*sin(phi*theta - psi) ));
numerator = (vthforward - vthbackward*exp(-ltau))*sqrt(2.);
denominator = beta * sqrt(1. - exp(-2*ltau));
rhs[tidx] += gsl_cdf_gaussian_Q(numerator/denominator, 1.0);
}//end per time loop
}//end all times loop
}
int main(void) {
double data[5] = { 17.2, 18.1, 16.5, 18.3, 12.6 };
double median, upperq, lowerq;
printf("Original dataset: %g, %g, %g, %g, %g\n", data[0], data[1], data[2],
data[3], data[4]);
gsl_sort(data, 1, 5);
printf("Sorted dataset: %g, %g, %g, %g, %g\n", data[0], data[1], data[2],
data[3], data[4]);
median = gsl_stats_median_from_sorted_data(data, 1, 5);
upperq = gsl_stats_quantile_from_sorted_data(data, 1, 5, 0.75);
lowerq = gsl_stats_quantile_from_sorted_data(data, 1, 5, 0.25);
printf("The median is %g\n", median);
printf("The upper quartile is %g\n", upperq);
printf("The lower quartile is %g\n", lowerq);
return EXIT_SUCCESS;
}
/** compute the various kinds of math. operation */
REAL8 math_op(REAL8* data, size_t length, INT4 type) {
UINT4 i;
REAL8 res = 0.0;
switch (type) {
case MATH_OP_ARITHMETIC_SUM: /* sum(data) */
for (i = 0; i < length; ++i) res += *(data++);
break;
case MATH_OP_ARITHMETIC_MEAN: /* sum(data)/length */
for (i = 0; i < length; ++i) res += *(data++);
res /= (REAL8)length;
break;
case MATH_OP_ARITHMETIC_MEDIAN: /* middle element of sort(data) */
gsl_sort(data, 1, length);
if (length/2 == (length+1)/2) /* length is even */ {
res = (data[length/2] + data[length/2+1])/2;
}
else /* length is odd */ {
res = data[length/2];
}
break;
case MATH_OP_HARMONIC_SUM: /* 1 / sum(1 / data) */
for (i = 0; i < length; ++i) res += 1.0 / *(data++);
res = 1.0 / res;
break;
case MATH_OP_HARMONIC_MEAN: /* length / sum(1 / data) */
for (i = 0; i < length; ++i) res += 1.0 / *(data++);
res = (REAL8)length / res;
break;
case MATH_OP_POWERMINUS2_SUM: /* 1 / sqrt ( sum(1 / data/data) )*/
for (i = 0; i < length; ++i) res += 1.0 / (data[i]*data[i]);
res = 1.0 / sqrt(res);
break;
case MATH_OP_POWERMINUS2_MEAN: /* 1 / sqrt ( sum(1/data/data) / length )*/
for (i = 0; i < length; ++i) res += 1.0 / (data[i]*data[i]);
res = 1.0 / sqrt(res / (REAL8)length);
break;
case MATH_OP_MINIMUM: /* first element of sort(data) */
gsl_sort(data, 1, length);
res = data[0];
break;
case MATH_OP_MAXIMUM: /* first element of sort(data) */
gsl_sort(data, 1, length);
res = data[length-1];
break;
default:
XLALPrintError("'%i' is not a valid math. operation", type);
XLAL_ERROR_REAL8(XLAL_EINVAL);
}
return res;
}
int core_oph_quantile (oph_stringPtr byte_array, char *result)
{
#ifdef GSL_SUPPORTED
if (!byte_array || (byte_array->param<0.0) || (byte_array->param>1.0))
{
pmesg(1, __FILE__, __LINE__, "Wrong parameter\n");
return -1;
}
int k;
switch(byte_array->type)
{
case OPH_DOUBLE:{
double sum;
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort((double*)byte_array->extend,1,byte_array->numelem);
double delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((double*)byte_array->extend)+k)) + delta * (*(((double*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_DOUBLE));
break;
}
case OPH_FLOAT:{
float sum;
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort_float((float*)byte_array->extend,1,byte_array->numelem);
float delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((float*)byte_array->extend)+k)) + delta * (*(((float*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_FLOAT));
break;
}
case OPH_INT:{
float sum; //The quantile is a float number
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort_int((int*)byte_array->extend,1,byte_array->numelem);
float delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((int*)byte_array->extend)+k)) + delta * (*(((int*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_FLOAT));
break;
}
case OPH_SHORT:{
float sum; //The quantile is a float number
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort_short((short*)byte_array->extend,1,byte_array->numelem);
float delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((short*)byte_array->extend)+k)) + delta * (*(((short*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_FLOAT));
break;
}
case OPH_BYTE:{
float sum; //The quantile is a float number
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort_char((char*)byte_array->extend,1,byte_array->numelem);
float delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((char*)byte_array->extend)+k)) + delta * (*(((char*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_FLOAT));
break;
}
case OPH_LONG:{
double sum; //The quantile is a double number
memcpy(byte_array->extend, byte_array->content, *byte_array->length);
gsl_sort_long((long*)byte_array->extend,1,byte_array->numelem);
double delta = (byte_array->numelem-1) * byte_array->param;
k = floor(delta);
delta = delta - k;
sum = (1-delta) * (*(((long long*)byte_array->extend)+k)) + delta * (*(((long long*)byte_array->extend)+k+1));
memcpy(result, (void*)(&sum), core_sizeof(OPH_DOUBLE));
break;
}
default:
pmesg(1, __FILE__, __LINE__, "Type non recognized\n");
return -1;
}
return 0;
#else
pmesg(1, __FILE__, __LINE__, "Operation not allowed\n");
return -1;
#endif
}
int main(int argc, char *argv []) {
if(populate_env_variable(REF_ERROR_CODES_FILE, "L2_ERROR_CODES_FILE")) {
printf("\nUnable to populate [REF_ERROR_CODES_FILE] variable with corresponding environment variable. Routine will proceed without error handling\n");
}
if (argc != 15) {
if(populate_env_variable(SPF_BLURB_FILE, "L2_SPF_BLURB_FILE")) {
RETURN_FLAG = 1;
} else {
print_file(SPF_BLURB_FILE);
}
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -1, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 1;
} else {
// ***********************************************************************
// Redefine routine input parameters
char *target_f = strdup(argv[1]);
int bin_size_px = strtol(argv[2], NULL, 0);
double bg_percentile = strtod(argv[3], NULL);
double clip_sigma = strtod(argv[4], NULL);
int median_filter_width_px = strtol(argv[5], NULL, 0);
double min_SNR = strtod(argv[6], NULL);
int min_spatial_width_px = strtol(argv[7], NULL, 0);
int finding_window_lo_px = strtol(argv[8], NULL, 0);
int finding_window_hi_px = strtol(argv[9], NULL, 0);
int max_centering_num_px = strtol(argv[10], NULL, 0);
int centroid_half_window_size_px = strtol(argv[11], NULL, 0);
int min_used_bins = strtol(argv[12], NULL, 0);
int window_x_lo = strtol(argv[13], NULL, 0);
int window_x_hi = strtol(argv[14], NULL, 0);
// ***********************************************************************
// Open target file (ARG 1), get parameters and perform any data format
// checks
fitsfile *target_f_ptr;
int target_f_maxdim = 2, target_f_status = 0, target_f_bitpix, target_f_naxis;
long target_f_naxes [2] = {1,1};
if(!fits_open_file(&target_f_ptr, target_f, IMG_READ_ACCURACY, &target_f_status)) {
if(!populate_img_parameters(target_f, target_f_ptr, target_f_maxdim, &target_f_bitpix, &target_f_naxis, target_f_naxes, &target_f_status, "TARGET FRAME")) {
if (target_f_naxis != 2) { // any data format checks here
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -2, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -3, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, target_f_status);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -4, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, target_f_status);
free(target_f);
return 1;
}
// ***********************************************************************
// Set the range limits
int cut_x [2] = {window_x_lo, window_x_hi};
int cut_y [2] = {1, target_f_naxes[1]};
// ***********************************************************************
// Set parameters used when reading data from target file (ARG 1)
long fpixel [2] = {cut_x[0], cut_y[0]};
long nxelements = (cut_x[1] - cut_x[0]) + 1;
long nyelements = (cut_y[1] - cut_y[0]) + 1;
// ***********************************************************************
// Create arrays to store pixel values from target fits file (ARG 1)
double target_f_pixels [nxelements];
// ***********************************************************************
// Get target fits file (ARG 1) values and store in 2D array
int ii;
double target_frame_values [nyelements][nxelements];
memset(target_frame_values, 0, sizeof(double)*nxelements*nyelements);
for (fpixel[1] = cut_y[0]; fpixel[1] <= cut_y[1]; fpixel[1]++) {
memset(target_f_pixels, 0, sizeof(double)*nxelements);
if(!fits_read_pix(target_f_ptr, TDOUBLE, fpixel, nxelements, NULL, target_f_pixels, NULL, &target_f_status)) {
for (ii=0; ii<nxelements; ii++) {
target_frame_values[fpixel[1]-1][ii] = target_f_pixels[ii];
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -5, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, target_f_status);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
}
// FIND VALUES OF PEAK CENTROID ALONG DISPERSION AXIS
// ***********************************************************************
// 1. Bin array according to bin width given by [bin_size_px]
int disp_nelements = nxelements, spat_nelements = nyelements;
int disp_nelements_binned = (int)floor(disp_nelements/bin_size_px);
double this_frame_values_binned[spat_nelements][disp_nelements_binned];
memset(this_frame_values_binned, 0, sizeof(double)*spat_nelements*disp_nelements_binned);
double this_bin_value;
int bin_number = 0;
int jj;
for (jj=0; jj<spat_nelements; jj++) {
this_bin_value = 0;
bin_number = 0;
for (ii=0; ii<disp_nelements; ii++) {
if (ii % bin_size_px == 0 && ii != 0) {
this_frame_values_binned[jj][bin_number] = this_bin_value;
bin_number++;
this_bin_value = 0;
}
this_bin_value += target_frame_values[jj][ii];
}
}
printf("\nFinding peaks");
printf("\n-------------------------------------\n");
double peaks[disp_nelements_binned];
int num_bins_used = 0;
for (ii=0; ii<disp_nelements_binned; ii++) {
// 1a. Establish if any target flux is in this bin
// First find the mean/sd of the [bg_percentile]th lowest valued pixels as an initial parameters for sigma clip
double this_spat_values[spat_nelements];
double this_spat_values_sorted[spat_nelements];
for (jj=0; jj<spat_nelements; jj++) {
this_spat_values[jj] = this_frame_values_binned[jj][ii];
}
memcpy(this_spat_values_sorted, this_spat_values, sizeof(double)*spat_nelements);
gsl_sort(this_spat_values_sorted, 1, spat_nelements);
int bg_nelements = (int)floor(spat_nelements*bg_percentile);
double bg_values [bg_nelements];
int idx = 0;
for (jj=0; jj<spat_nelements; jj++) {
if (this_spat_values_sorted[jj] != 0) { // avoid 0s set from median filter edges
bg_values[idx] = this_spat_values_sorted[jj];
idx++;
if (idx == bg_nelements)
break;
}
}
if (idx != bg_nelements) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -6, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
double start_mean = gsl_stats_mean(bg_values, 1, bg_nelements);
double start_sd = gsl_stats_sd(bg_values, 1, bg_nelements);
// 1b. Iterative sigma clip around dataset with the initial guess
int retain_indexes[spat_nelements];
double final_mean, final_sd;
int final_num_retained_indexes;
printf("\nBin:\t\t\t\t%d", ii);
printf("\nStart mean:\t\t\t%f", start_mean);
printf("\nStart SD:\t\t\t%f", start_sd);
iterative_sigma_clip(this_spat_values, spat_nelements, clip_sigma, retain_indexes, start_mean, start_sd, &final_mean, &final_sd, &final_num_retained_indexes, FALSE);
printf("\nFinal mean:\t\t\t%f", final_mean);
printf("\nFinal SD:\t\t\t%f", final_sd);
// 2. Smooth array with median filter
double this_spat_values_smoothed[spat_nelements];
memset(this_spat_values_smoothed, 0, sizeof(double)*spat_nelements);
median_filter(this_spat_values, this_spat_values_smoothed, spat_nelements, median_filter_width_px);
// 3. Ascertain if this bin contains target flux
int num_pixels_contain_target_flux = 0;
for (jj=0; jj<spat_nelements-1; jj++) {
if (this_spat_values_smoothed[jj] > final_mean + final_sd*min_SNR) {
num_pixels_contain_target_flux++;
}
}
printf("\nNum pixels (target):\t\t%d", num_pixels_contain_target_flux);
printf("\nDoes bin contain target flux?\t");
if (num_pixels_contain_target_flux >= min_spatial_width_px) {
printf("Yes\n");
} else {
printf("No\n");
peaks[ii] = -1;
continue;
}
// 3. Take derivatives
double this_spat_values_der[spat_nelements-1];
memset(this_spat_values_der, 0, sizeof(double)*spat_nelements-1);
for (jj=1; jj<spat_nelements; jj++) {
this_spat_values_der[jj-1] = this_frame_values_binned[jj][ii] - this_frame_values_binned[jj-1][ii];
}
// 4. Smooth derivatives
double this_spat_values_der_smoothed[spat_nelements-1];
memcpy(this_spat_values_der_smoothed, this_spat_values_der, sizeof(double)*spat_nelements-1);
median_filter(this_spat_values_der, this_spat_values_der_smoothed, spat_nelements-1, median_filter_width_px);
// 5. Pick most positive gradient from window, retaining proper index
double this_spat_values_der_smoothed_windowed[spat_nelements-1];
memcpy(this_spat_values_der_smoothed_windowed, this_spat_values_der_smoothed, sizeof(double)*spat_nelements-1);
for (jj=0; jj<spat_nelements; jj++) {
if (jj >= finding_window_lo_px && jj <= finding_window_hi_px) {
this_spat_values_der_smoothed_windowed[jj] = this_spat_values_der_smoothed_windowed[jj];
} else {
this_spat_values_der_smoothed_windowed[jj] = -1;
}
}
size_t this_pk_idx = gsl_stats_max_index(this_spat_values_der_smoothed_windowed, 1, spat_nelements-1);
printf("Start peak index:\t\t%d\n", this_pk_idx);
// 6. Using this index, walk through centering window [max_centering_num_px] and find derivative turnover point
printf("Found turnover:\t\t\t");
bool found_turnover = FALSE;
for (jj=this_pk_idx; jj<this_pk_idx+max_centering_num_px; jj++) {
if (this_spat_values_der_smoothed[jj] < 0.) {
this_pk_idx = jj;
found_turnover = TRUE;
break;
}
}
if (found_turnover) {
printf("Yes\n");
printf("End peak index:\t\t\t%d\n", this_pk_idx);
} else {
printf("No\n");
peaks[ii] = -1;
continue;
}
// 7. Get parabolic centroid using the full centroid window [centroid_half_window_size_px]
double this_pk_window_idxs[1 + (2*centroid_half_window_size_px)];
double this_pk_window_vals[1 + (2*centroid_half_window_size_px)];
memset(this_pk_window_idxs, 0, sizeof(double)*(1 + (2*centroid_half_window_size_px)));
memset(this_pk_window_vals, 0, sizeof(double)*(1 + (2*centroid_half_window_size_px)));
idx = 0;
for (jj=this_pk_idx-centroid_half_window_size_px; jj<=this_pk_idx+centroid_half_window_size_px; jj++) {
this_pk_window_idxs[idx] = jj;
this_pk_window_vals[idx] = this_frame_values_binned[jj][ii];
idx++;
}
int order = 2;
double coeffs [order+1];
memset(coeffs, 0, sizeof(double)*order+1);
double chi_squared;
if (calc_least_sq_fit(2, 1 + (2*centroid_half_window_size_px), this_pk_window_idxs, this_pk_window_vals, coeffs, &chi_squared)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -7, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
double fitted_peak_idx = -coeffs[1]/(2*coeffs[2]);
printf("Fitted peak index:\t\t%f\n", fitted_peak_idx);
// 8. Ensure fitted peak location is within finding window
printf("Is fitted peak within window?\t");
if (fitted_peak_idx > finding_window_lo_px && fitted_peak_idx < finding_window_hi_px) {
printf("Yes\n");
num_bins_used++;
} else {
printf("No\n");
peaks[ii] = -1;
continue;
}
peaks[ii] = fitted_peak_idx;
/*for (jj=0; jj<spat_nelements-1; jj++) {
printf("%d\t%f\t%f\n", jj, this_spat_values_der_smoothed[jj], this_spat_values_der[jj]);
}*/ // DEBUG
}
printf("\nNum bins used:\t%d\n", num_bins_used);
if (num_bins_used < min_used_bins) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -8, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
// ***********************************************************************
// Create [FRFIND_OUTPUTF_PEAKS_FILE] output file and print a few
// parameters
FILE *outputfile;
outputfile = fopen(SPFIND_OUTPUTF_PEAKS_FILE, FILE_WRITE_ACCESS);
if (!outputfile) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -9, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(target_f);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
char timestr [80];
memset(timestr, '\0', sizeof(char)*80);
find_time(timestr);
fprintf(outputfile, "#### %s ####\n\n", SPFIND_OUTPUTF_PEAKS_FILE);
fprintf(outputfile, "# Lists the coordinates of the peaks found using the spfind routine.\n\n");
fprintf(outputfile, "# Run filename:\t%s\n", target_f);
fprintf(outputfile, "# Run datetime:\t%s\n\n", timestr);
for (ii=0; ii<disp_nelements_binned; ii++) {
if (peaks[ii] == -1)
continue;
fprintf(outputfile, "%f\t%f\n", cut_x[0]+(ii*bin_size_px) + (double)bin_size_px/2., peaks[ii]);
}
fprintf(outputfile, "%d", EOF);
// ***********************************************************************
// Clean up heap memory
free(target_f);
// ***********************************************************************
// Close [FRFIND_OUTPUTF_PEAKS_FILE] output file and target file (ARG 1)
if (fclose(outputfile)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -10, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
if(fits_close_file(target_f_ptr, &target_f_status)) fits_report_error (stdout, target_f_status);
return 1;
}
if(fits_close_file(target_f_ptr, &target_f_status)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", -11, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error (stdout, target_f_status);
return 1;
}
// ***********************************************************************
// Write success to [ERROR_CODES_FILE]
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATFI", RETURN_FLAG, "Status flag for L2 spfind routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 0;
}
}
nodo * crop_network(nodo *my_node,int N,edge *my_edge,int Nlinks,long double Nloops_rand,long double Nloops_exp,int contmax)
{
nodo *crop_node;
double *importance,*importance2;
size_t *importance_index;
vector <int> aux;
edge *new_edge;
int Nlinks_eff;
for (int i=0;i<Nlinks;i++) my_edge[i].label=-1;
/* for(int i=0;i<Nlinks;i++) {*/
/* printf("Link[%d] (%d->%d).importance=%d\n",i,my_edge[i].in,my_edge[i].out,my_edge[i].importance);*/
/* }*/
//printf("defino un nuevo vector de links\n");
for(int i=0;i<Nlinks;i++) if (my_edge[i].importance != 0) aux.push_back(i);
new_edge=(edge*)malloc(aux.size()*sizeof(edge));
for(int i=0;i<aux.size();i++) {
new_edge[i].in = my_edge[aux[i]].in;
new_edge[i].out = my_edge[aux[i]].out;
new_edge[i].importance = my_edge[aux[i]].importance;
new_edge[i].label= aux[i];
}
Nlinks_eff=aux.size();
/* for(int i=0;i<Nlinks_eff;i++) {*/
/* printf("new_links[%d] (%d->%d).importance=%d\n",i,new_edge[i].in,new_edge[i].out,new_edge[i].importance);*/
/* }*/
importance=(double*)calloc(Nlinks_eff,sizeof(double));
importance2=(double*)calloc(Nlinks_eff,sizeof(double));
importance_index=(size_t*)malloc(Nlinks_eff*sizeof(size_t));
for (int i=0;i<Nlinks_eff;i++) importance[i]=importance2[i]=new_edge[i].importance*1.;
gsl_sort(importance2,1,Nlinks_eff);
gsl_sort_index(importance_index,importance,1,Nlinks_eff); //ya tenemos ordenados los edges por importancia, y su antiguo nombre esta en index
//printf("primero establecemos los labels en la lista larga!\n");
for (int i=0;i<Nlinks_eff;i++) {
//printf("para el new_link[%d][%d->%d]:\n",i,new_edge[i].in,new_edge[i].out);
int index=0;
for (int j=0;j<new_edge[i].in;j++){
index += my_node[j].kout;
// printf("+M[%d].kout(%d) ",j,my_node[j].kout);
}
// printf("\n");
int nodoi=new_edge[i].in;
for (int j=0;j<my_node[nodoi].kout;j++){
if (new_edge[i].out==my_node[nodoi].out_nodos[j]) break;
index ++;
// printf("+ 1(%d)",my_node[nodoi].out_nodos[j]);
}
// printf("\n");
//printf("en el largo es Links[%d][%d->%d]\n\n",index,my_edge[index].in,my_edge[index].out);
my_edge[index].label=i;
new_edge[i].label2=index;
}
//for (int i=0;i<Nlinks_eff;i++) printf("importance2[%d]=%.0lf label2[%d]=%d i[%d]=%d %d->%d \n",i,importance2[i],i,new_edge[i].label2,i,importance_index[i],new_edge[i].in,new_edge[i].out);//el nodo "inicial(label)" es el nuevo
//for (int i=0;i<Nlinks_eff;i++) printf("importance[%d]=%.0lf i[%d]=%d \n",i,importance[i],i,importance_index[i]);//el nodo "inicial(label)" es el nuevo (label_index)
int cont=Nlinks_eff; printf("Nlinks_eff=%d\n",Nlinks_eff);
//printf("entramos al bucle\n");
resta(Nloops_exp,Nloops_rand,Nlinks_eff,new_edge,importance_index,my_node,N,Nlinks_eff,contmax);
//printf("salimos del bucle\n");
aux.clear();
free(new_edge);
free(importance);
free(importance2);
free(importance_index);
return crop_node;
}
int main (void)
{
int i, j, warn, counter;
double lim[15];
lim[0]=0.01;
//lim[1]=0.02;
lim[1]=0.03;
//lim[3]=0.04;
lim[2]=0.05;
//lim[5]=0.09;
lim[3]=0.11;
//lim[7]=0.14;
lim[4]=0.17;
//lim[9]=0.208;
lim[5]=0.26;
//lim[11]=0.32;
lim[6]=0.4;
//lim[13]=0.51;
lim[7]=0.64;
//lim[15]=0.8;
lim[8]=1.0;
//lim[17]=1.25;
lim[9]=1.56;
//lim[19]=1.95;
lim[10]=2.44;
//lim[21]=3.05;
lim[11]=3.81;
//lim[23]=4.76;
lim[12]=5.96;
//lim[25]=7.45;
lim[13]=9.31;
//lim[27]=11.64;
lim[14]=14.55;
//lim[29]=18.18;
counter = 0.0;
double pos[N],vel[N],err[N],vel_sq, sum;
double mean, median, skew, sd, kurtosis, error, sum1, sum2;
FILE *velo, *sig, *script;
velo = fopen("organized.dat", "r");
sig = fopen("sigma.dat","w");
for(i=0.0; i<N; i++)
{
fscanf(velo, "%lf %lf %lf", &pos[i], &vel[i], &err[i]);
}
//printf ("The sample mean is %g\n", mean);
fclose(velo);
for(j=0.0; j<14.0; j=j+1.0)
{
counter = 0.0;
for(i = 0.0;(i)<=N; i++)
{
if( lim[j] <= pos[i] && pos[i] < lim[(j)+1] )
{
counter++;
gsl_sort (vel, 1, counter);
mean = gsl_stats_mean(vel, 1, counter);
median = gsl_stats_median_from_sorted_data (vel, 1, counter);
sd = gsl_stats_sd (vel, 1, counter);
skew = gsl_stats_skew(vel, 1, counter);
kurtosis = gsl_stats_kurtosis (vel, 1, counter);
sum1 = (vel[i]-mean)*(vel[i]-mean);
sum1++;
sum2 = (vel[i]-mean)*err[i];
sum2++;
error = 1.0/(sqrt(counter)) * pow(sum1,-0.5) * sum2;
}
}
printf("Particulas entre %lf y %lf es %d\n",lim[j],lim[j+1], counter);
printf("Error %lf\n", error);
printf ("The sample mean is %g\n", mean);
printf ("The median is %g\n", median);
printf ("Standard deviation is %g\n", sd);
printf ("Skewness is %g\n", skew);
printf ("kurtosis is %g\n\n", kurtosis);
//printf("Sigma in the bin is: %.2f\n\n", sig);
fprintf(sig, "%lf\t %lf\t %lf\t %d\n", (lim[j]+lim[j+1])*0.5, sd, error, counter);
}
fclose(sig);
script = fopen( "script1.gpl", "w" );
fprintf(script, "set grid\nset terminal png\nset output 'Sigma_vs_rad.png'\nset nokey\n");
fprintf( script, "set title 'Sigma vs Radius'\n" );
fprintf( script, "set logscale x\n" );
fprintf( script, "set xlabel 'Radius in Arcmin'\n" );
fprintf( script, "set xrange [0.01:20]\n" );
//fprintf( script, "set logscale x 10\n" );
fprintf( script, "set ylabel 'Sigma (km/s)'\n" );
fprintf( script, "plot 'sigma.dat' u 1:2:3 pt 7 ps 1 with errorbars\n");
fclose(script);
warn = system("gnuplot script1.gpl");
return(warn);
}
void LALCleanCOMPLEX8SFT (LALStatus *status,/**< pointer to LALStatus structure */
SFTtype *sft, /**< SFT to be cleaned */
INT4 width, /**< maximum width to be cleaned -- set sufficiently large if all bins in each line are to be cleaned*/
INT4 window,/**< window size for noise floor estimation in vicinity of a line */
LineNoiseInfo *lineInfo, /**< list of lines */
RandomParams *randPar /**< parameters for generating random noise */)
{
/* function to clean the SFT based on the line information read earlier */
INT4 nLines, count, leftCount, rightCount, lineBin, minBin, maxBin, k, tempk;
INT4 leftWingBins, rightWingBins, length, sumBins;
REAL8 deltaF, f0, tBase, bias;
REAL8 stdPow, medianPow;
REAL8 *tempDataPow=NULL;
REAL8 *lineFreq=NULL;
REAL8 *leftWing=NULL;
REAL8 *rightWing=NULL;
COMPLEX8 *inData;
/* FILE *fp=NULL;
INT4 seed, ranCount;
RandomParams *randPar=NULL; */ /* 09/09/05 gam; randPar now a function argument */
static REAL4Vector *ranVector=NULL;
REAL4 *randVal;
/* --------------------------------------------- */
INITSTATUS(status);
ATTATCHSTATUSPTR (status);
/* Make sure the arguments are not NULL: */
ASSERT (sft, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
inData = sft->data->data;
ASSERT (inData, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
ASSERT (lineInfo, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
ASSERT (lineInfo->nLines, status, SFTCLEANH_EVAL, SFTCLEANH_MSGEVAL);
ASSERT (lineInfo->lineFreq, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
ASSERT (lineInfo->leftWing, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
ASSERT (lineInfo->rightWing, status, SFTCLEANH_ENULL, SFTCLEANH_MSGENULL);
ASSERT (window > 0, status, SFTCLEANH_EVAL, SFTCLEANH_MSGEVAL);
ASSERT (width >= 0, status, SFTCLEANH_EVAL, SFTCLEANH_MSGEVAL);
length = sft->data->length;
ASSERT (length > 0, status, SFTCLEANH_EHEADER, SFTCLEANH_MSGEVAL);
/* get the value of RngMedBias from the window size */
TRY( LALRngMedBias( status->statusPtr, &bias, 2*window ), status );
/* copy pointers from input */
nLines = lineInfo->nLines;
lineFreq = lineInfo->lineFreq;
leftWing = lineInfo->leftWing;
rightWing = lineInfo->rightWing;
deltaF = sft->deltaF;
tBase = 1.0/deltaF;
f0 = sft->f0;
minBin = lround(f0/deltaF);
maxBin = minBin + length - 1;
/* allocate memory for storing sft power */
tempDataPow = LALMalloc(2*window*sizeof(REAL8));
/* 09/09/05 gam; randPar now a function argument */
/* fp=fopen("/dev/urandom", "r");
ASSERT(fp, status, SFTCLEANH_EFILE, SFTCLEANH_MSGEFILE);
ranCount = fread(&seed, sizeof(seed), 1, fp);
ASSERT(ranCount==1, status, SFTCLEANH_EREAD, SFTCLEANH_MSGEREAD);
fclose(fp); */
/* calculate total number of bins to see how many random numbers must be generated */
sumBins = 0;
for (count = 0; count < nLines; count++)
{
{
INT4 tempSumBins;
tempSumBins = floor(tBase*leftWing[count]) + floor(tBase*rightWing[count]);
sumBins += tempSumBins < 2*width ? tempSumBins : 2*width;
}
}
/* TRY ( LALCreateRandomParams (status->statusPtr, &randPar, seed), status); */ /* 09/09/05 gam; randPar now a function argument */
TRY ( LALCreateVector (status->statusPtr, &ranVector, 2*(sumBins + nLines)), status);
TRY ( LALNormalDeviates (status->statusPtr, ranVector, randPar), status);
/* TRY ( LALDestroyRandomParams (status->statusPtr, &randPar), status); */ /* 09/09/05 gam; randPar now a function argument */
tempk = 0;
/* loop over the lines */
for (count = 0; count < nLines; count++){
/* find frequency bins for line frequency and wings */
lineBin = lround(tBase * lineFreq[count]);
leftWingBins = floor(tBase * leftWing[count]);
rightWingBins = floor(tBase * rightWing[count]);
/* check that frequency is within band of sft and line is not too wide*/
if ((lineBin >= minBin) && (lineBin <= maxBin) && (leftWingBins <= width) && (rightWingBins <= width)){
/* estimate the sft power in "window" # of bins each side */
for (k = 0; k < window ; k++){
if (maxBin - lineBin - rightWingBins - k > 0)
inData = sft->data->data + lineBin - minBin + rightWingBins + k + 1;
else
inData = sft->data->data + length - 1;
tempDataPow[k] = (crealf(*inData))*(crealf(*inData)) + (cimagf(*inData))*(cimagf(*inData));
if (lineBin - minBin -leftWingBins - k > 0)
inData = sft->data->data + lineBin - minBin - leftWingBins - k - 1;
else
inData = sft->data->data;
tempDataPow[k+window] = (crealf(*inData))*(crealf(*inData)) + (cimagf(*inData))*(cimagf(*inData));
}
gsl_sort( tempDataPow, 1, 2*window);
medianPow = gsl_stats_median_from_sorted_data(tempDataPow, 1, 2*window);
stdPow = sqrt(medianPow/(2 * bias));
/* set sft value at central frequency to noise */
inData = sft->data->data + lineBin - minBin;
randVal = ranVector->data + tempk;
*(inData) = crectf( stdPow * (*randVal), cimagf(*(inData)) );
tempk++;
randVal = ranVector->data + tempk;
*(inData) = crectf( crealf(*(inData)), stdPow * (*randVal) );
tempk++;
/* now go left and set the left wing to noise */
/* make sure that we are always within the sft band */
/* and set bins to zero only if Wing width is smaller than "width" */
if ((leftWingBins <= width)){
for (leftCount = 0; leftCount < leftWingBins; leftCount++){
if ( (lineBin - minBin - leftCount > 0)){
inData = sft->data->data + lineBin - minBin - leftCount - 1;
randVal = ranVector->data + tempk;
*(inData) = crectf( stdPow * (*randVal), cimagf(*(inData)) );
tempk++;
randVal = ranVector->data + tempk;
*(inData) = crectf( crealf(*(inData)), stdPow * (*randVal) );
tempk++;
}
}
}
/* now go right making sure again to stay within the sft band */
if ((rightWingBins <= width )){
for (rightCount = 0; rightCount < rightWingBins; rightCount++){
if ( (maxBin - lineBin - rightCount > 0)){
inData = sft->data->data + lineBin - minBin + rightCount + 1;
randVal = ranVector->data + tempk;
*(inData) = crectf( stdPow * (*randVal), cimagf(*(inData)) );
tempk++;
randVal = ranVector->data + tempk;
*(inData) = crectf( crealf(*(inData)), stdPow * (*randVal) );
tempk++;
}
}
}
}
} /* end loop over lines */
/* free memory */
LALFree(tempDataPow);
TRY (LALDestroyVector (status->statusPtr, &ranVector), status);
DETATCHSTATUSPTR (status);
/* normal exit */
RETURN (status);
}
void Column::calculateStatistics() {
m_column_private->statistics = ColumnStatistics();
ColumnStatistics& statistics = m_column_private->statistics;
QVector<double>* rowValues = reinterpret_cast<QVector<double>*>(data());
int notNanCount = 0;
double val;
double columnSum = 0.0;
double columnProduct = 1.0;
double columnSumNeg = 0.0;
double columnSumSquare = 0.0;
statistics.minimum = INFINITY;
statistics.maximum = -INFINITY;
QMap<double, int> frequencyOfValues;
QVector<double> rowData;
rowData.reserve(rowValues->size());
for (int row = 0; row < rowValues->size(); ++row) {
val = rowValues->value(row);
if (std::isnan(val) || isMasked(row))
continue;
if (val < statistics.minimum){
statistics.minimum = val;
}
if (val > statistics.maximum){
statistics.maximum = val;
}
columnSum+= val;
columnSumNeg += (1.0 / val);
columnSumSquare += pow(val, 2.0);
columnProduct *= val;
if (frequencyOfValues.contains(val)){
frequencyOfValues.operator [](val)++;
}
else{
frequencyOfValues.insert(val, 1);
}
++notNanCount;
rowData.push_back(val);
}
if (notNanCount == 0) {
setStatisticsAvailable(true);
return;
}
if (rowData.size() < rowValues->size()){
rowData.squeeze();
}
statistics.arithmeticMean = columnSum / notNanCount;
statistics.geometricMean = pow(columnProduct, 1.0 / notNanCount);
statistics.harmonicMean = notNanCount / columnSumNeg;
statistics.contraharmonicMean = columnSumSquare / columnSum;
double columnSumVariance = 0;
double columnSumMeanDeviation = 0.0;
double columnSumMedianDeviation = 0.0;
double sumForCentralMoment_r3 = 0.0;
double sumForCentralMoment_r4 = 0.0;
gsl_sort(rowData.data(), 1, notNanCount);
statistics.median = (notNanCount % 2 ? rowData.at((notNanCount-1)/2) :
(rowData.at((notNanCount-1)/2) +
rowData.at(notNanCount/2))/2.0);
QVector<double> absoluteMedianList;
absoluteMedianList.reserve(notNanCount);
absoluteMedianList.resize(notNanCount);
int idx = 0;
for(int row = 0; row < rowValues->size(); ++row){
val = rowValues->value(row);
if ( std::isnan(val) || isMasked(row) )
continue;
columnSumVariance+= pow(val - statistics.arithmeticMean, 2.0);
sumForCentralMoment_r3 += pow(val - statistics.arithmeticMean, 3.0);
sumForCentralMoment_r4 += pow(val - statistics.arithmeticMean, 4.0);
columnSumMeanDeviation += fabs( val - statistics.arithmeticMean );
absoluteMedianList[idx] = fabs(val - statistics.median);
columnSumMedianDeviation += absoluteMedianList[idx];
idx++;
}
statistics.meanDeviationAroundMedian = columnSumMedianDeviation / notNanCount;
statistics.medianDeviation = (notNanCount % 2 ? absoluteMedianList.at((notNanCount-1)/2) :
(absoluteMedianList.at((notNanCount-1)/2) + absoluteMedianList.at(notNanCount/2))/2.0);
const double centralMoment_r3 = sumForCentralMoment_r3 / notNanCount;
const double centralMoment_r4 = sumForCentralMoment_r4 / notNanCount;
statistics.variance = columnSumVariance / notNanCount;
statistics.standardDeviation = sqrt(statistics.variance);
statistics.skewness = centralMoment_r3 / pow(statistics.standardDeviation, 3.0);
statistics.kurtosis = (centralMoment_r4 / pow(statistics.standardDeviation, 4.0)) - 3.0;
statistics.meanDeviation = columnSumMeanDeviation / notNanCount;
double entropy = 0.0;
QList<int> frequencyOfValuesValues = frequencyOfValues.values();
for (int i = 0; i < frequencyOfValuesValues.size(); ++i){
double frequencyNorm = static_cast<double>(frequencyOfValuesValues.at(i)) / notNanCount;
entropy += (frequencyNorm * log2(frequencyNorm));
}
statistics.entropy = -entropy;
setStatisticsAvailable(true);
}
int main (int argc, char * argv[]) {
parse_opts(argc,argv);
if(!doInit()) {
printf("Error Initializing Stuff.\n");
exit(EXIT_FAILURE);
} else { //o calibrar colores o seguir la linea
//bool rotate = true;
double pval;
bool stop = true;
int left, right;
while(!ag_psh_is_pushed(&push, &pval));
UDELAY(750000);
printf("pval is: %.2f\n", pval);
ag_lgt_set_led(&lright, true);
ag_lgt_set_led(&lleft, true);
if (mode) { //calibrar colores
int i = 0;
double acumr [calib], minr, maxr, upr, lowr;
double acuml [calib], minl, maxl, upl, lowl;
for (;i<calib;i++) {
acumr[i] = (double)ag_read_int(&lright);
acuml[i] = (double)ag_read_int(&lleft);
UDELAY(100);
}
gsl_stats_minmax(&minr, &maxr, acumr, 1, calib);
gsl_stats_minmax(&minl, &maxl, acuml, 1, calib);
gsl_sort (acumr,1,calib);
gsl_sort (acuml,1,calib);
upl = gsl_stats_quantile_from_sorted_data (acuml,1,calib,0.95);//uq
lowl = gsl_stats_quantile_from_sorted_data (acuml,1,calib,0.05);//lq
upr = gsl_stats_quantile_from_sorted_data (acumr,1,calib,0.95);//uq
lowr = gsl_stats_quantile_from_sorted_data (acumr,1,calib,0.05);//lq
for (i = 0 ; i < 2; i++){
printf("COLOR: %s, SENSOR: %s\n", white ? "WHITE" : "BLACK", i == 0 ? "LEFT" : "RIGHT");
printf("min_v: %d, max_v: %d\n", i == 0 ? (int)minl : (int)minr, i == 0 ? (int)maxl : (int)maxr);
printf("low_q: %d, up_q :%d\n", i == 0 ? (int)lowl : (int)lowr, i == 0 ? (int)upl : (int)upr);
printf("\n");
}
} else { //seguir linea
while(!ag_psh_is_pushed(&push, &pval)) {
//printf("pval is: %.2f\n", pval);
if (stop){
stop = false;
move_all(MY_FWD,vel);
}
right = ag_read_int(&lright);
left = ag_read_int(&lleft);
if(! IS_WHITE(right) || ! IS_WHITE(left)){
stop_all();
stop = true;
if (IS_BLACK(left)){
rotate_robot(vel, true);
while(IS_BLACK(ag_read_int(&lleft)));
UDELAY(TDELAY);
stop_all();
} else if (IS_BLACK(right)) {
rotate_robot(vel, false);
while(IS_BLACK(ag_read_int(&lright)));
UDELAY(TDELAY);
stop_all();
}
}
}
}
}
lego_shutdown();
exit(EXIT_SUCCESS);
}
/**
* Estimates the FAP by Monte Carlo bootstrapping of the original data. "trials" bootstrapped data sets are
* generated using the random number generator "rng"; for each data set, the routine ok_periodogram_ls estimates
* z_max, which is collected into an array and returned into "zmax". Bootstrapped datasets are built by selecting
* with replacement from the input dataset, keeping times of observation fixed.
* @param data Input matrix containing the data; each row containing (t_i, x_i, sigma_i)
* @param trials Number of bootstrap trials
* @param samples Number of frequencies sampled
* @param Pmin Minimum period sampled
* @param Pmax Maximum period sampled
* @param method Method used to compute periodogram (ignored)
* @param timecol Time column (e.g. 0) in the matrix data
* @param valcol Value column (e.g. 1) in the matrix data
* @param sigmacol Sigma column (e.g. 2) in the matrix data
* @param rng A pre-allocated random number generator
* @param p If specified, returns additional info for the periodogram and reuses matrices to save space/speed. If you pass
* a value different than NULL, you are responsible for deallocating the workspace and its fields. p->zm returns a sorted
* vector of the maximum powers in each synthetic trial.
* @param prog An ok_progress* callback; if different from NULL, can be used to stop or report progress.
* @return A matrix containing: {PS_TIME, PS_Z, PS_FAP, PS_Z_LS} (period, power, bootstrapped FAP, unnormalized
* LS power). You are responsible for deallocating it.
*/
gsl_matrix* ok_periodogram_boot(const gsl_matrix* data, const unsigned int trials, const unsigned int samples,
const double Pmin, const double Pmax, const int method,
const unsigned int timecol, const unsigned int valcol, const unsigned int sigcol,
const unsigned long int seed, ok_periodogram_workspace* p, ok_progress prog) {
int nthreads = omp_get_max_threads();
ok_periodogram_workspace * w[nthreads];
gsl_matrix * mock[nthreads];
gsl_rng * rng[nthreads];
rng[0] = gsl_rng_alloc(gsl_rng_default);
gsl_rng_set(rng[0], seed);
for (int i = 0; i < nthreads; i++) {
w[i] = (ok_periodogram_workspace*) malloc(sizeof (ok_periodogram_workspace));
w[i]->per = NULL;
w[i]->buf = NULL;
w[i]->calc_z_fap = false;
mock[i] = ok_matrix_copy(data);
if (i > 0) {
rng[i] = gsl_rng_alloc(gsl_rng_default);
gsl_rng_set(rng[i], seed + i);
}
}
gsl_matrix* ret = ok_matrix_copy(ok_periodogram_ls(data, samples, Pmin, Pmax, method, timecol, valcol, sigcol, NULL));
gsl_vector* zmax = (p != NULL && p->zm != NULL ? p->zm : gsl_vector_alloc(trials));
bool abort = false;
#pragma omp parallel for
for (int i = 0; i < trials; i++) {
if (!abort) {
int nt = omp_get_thread_num();
ok_bootstrap_matrix_mean(data, T_TIME, T_VAL, mock[nt], rng[nt]);
ok_periodogram_ls(mock[nt], samples, Pmin, Pmax, method, timecol, valcol, sigcol, w[nt]);
zmax->data[i] = w[nt]->zmax;
if (nt == 0 && prog != NULL) {
int ret = prog(i * nthreads, trials, NULL,
"ok_periodogram_boot");
if (ret == PROGRESS_STOP) {
abort = true;
#pragma omp flush (abort)
}
}
}
}
gsl_sort(zmax->data, 1, trials);
for (int i = 0; i < ret->size1; i++) {
if (MGET(ret, i, PS_Z) > zmax->data[trials - 1])
MSET(ret, i, PS_FAP, 1. / (double) trials);
else if (MGET(ret, i, PS_Z) < zmax->data[0])
MSET(ret, i, PS_FAP, 1.);
else {
int idx = ok_bsearch(zmax->data, MGET(ret, i, PS_Z), trials);
MSET(ret, i, PS_FAP, (double) (trials - idx) / (double) trials);
}
}
for (int i = 0; i < nthreads; i++) {
gsl_matrix_free(w[i]->buf);
gsl_matrix_free(w[i]->per);
free(w[i]);
gsl_matrix_free(mock[i]);
gsl_rng_free(rng[i]);
}
if (p != NULL) {
if (p->zm != NULL)
gsl_vector_free(zmax);
}
return ret;
}
void _fortetError(const double * ts, const int num_steps, double * Is, const int num_intervals,
const double * abgthphi,
double *error) {
const double alpha = abgthphi[0];
const double beta = abgthphi[1];
const double gamma = abgthphi[2];
const double theta = abgthphi[3];
const double phi = abgthphi[4];
// printf("Fortet: theta=%.2f,gamma=%.2f\n", theta, gamma);
const double TAU_TOLERANCE = 1e-5;
const double psi = atan(theta);
//Sort is:
gsl_sort(Is, 1, num_intervals);
double lt, lI, ltau, vthforward, vthbackward;
double numerator, denominator, lhs;
double max_lhs = .0;
double normalizing_const = 1.0*num_intervals;
double exp_factor;
for (int tidx = 0; tidx < num_steps; ++tidx) {
lt = ts[tidx];
vthforward = 1. - (alpha*(1 - exp(-lt)) +
gamma / sqrt(1.+theta*theta) * ( sin ( theta * ( lt + phi) - psi)
-exp(-lt)*sin(phi*theta - psi) ));
//Calculate LHS:
numerator = vthforward * sqrt(2.);
denominator = beta * sqrt(1. - exp(-2.*lt));
lhs = gsl_cdf_gaussian_Q(numerator/denominator, 1.0);
max_lhs = GSL_MAX(lhs, max_lhs);
error[tidx] = lhs;
//Calculate RHS:
for (int idx = 0; idx < num_intervals; ++idx){
lI = Is[idx];
if(lI >= (lt - TAU_TOLERANCE)) break;
ltau = lt - lI;
exp_factor = exp(-lI);
vthbackward = 1. - (alpha*(1. - exp_factor) +
gamma / sqrt(1.+theta*theta) * ( sin ( theta * ( lI + phi) - psi)
-exp_factor*sin(phi*theta - psi) ));
exp_factor = exp(-ltau);
numerator = (vthforward - vthbackward*exp_factor)*sqrt(2.);
denominator = beta * sqrt(1. - exp_factor*exp_factor);
error[tidx] -= gsl_cdf_gaussian_Q(numerator/denominator, 1.0) / normalizing_const;
}//end rhs (intervals) loop
}//end error (time steps) loop
//scale by max_lhs:
for (int tidx = 0; tidx < num_steps; ++tidx) {
error[tidx] = fabs(error[tidx]) / max_lhs;
}
}
int main (void)
{
int i, j, warn, lim[13], counter;
lim[0]=0.0;
lim[1]=1.0;
lim[2]=2.0;
lim[3]=3.0;
lim[4]=4.0;
lim[5]=5.0;
lim[6]=8.0;
lim[7]=12.0;
lim[8]=16.0;
lim[9]=20.0;
lim[10]=24.0;
lim[11]=28.0;
lim[12]=45.0;
counter = 0.0;
double pos[N],vel[N],err[N],vel_sq, sum;
double mean, median, skew, sd, kurtosis;
FILE *velo, *sig, *script;
velo = fopen("velocity.dat", "r");
sig = fopen("sigma.dat","w");
for(i=0.0; i<N; i++)
{
fscanf(velo, "%lf %lf %lf", &pos[i], &vel[i], &err[i]);
}
//printf ("The sample mean is %g\n", mean);
fclose(velo);
for(j=0.0; j<12.0; j=j+1.0)
{
counter = 0.0;
for(i = 0.0;(i)<=N; i++)
{
if( lim[j] <= pos[i] && pos[i] < lim[(j)+1] )
{
counter++;
gsl_sort (vel, 1, counter);
mean = gsl_stats_mean(vel, 1, counter);
median = gsl_stats_median_from_sorted_data (vel, 1, counter);
sd = gsl_stats_sd (vel, 1, counter);
skew = gsl_stats_skew(vel, 1, counter);
kurtosis = gsl_stats_kurtosis (vel, 1, counter);
}
}
printf("Particulas entre %d y %d es %d\n",lim[j],lim[j +1], counter);
printf ("The sample mean is %g\n", mean);
printf ("The median is %g\n", median);
printf ("Standard deviation is %g\n", sd);
printf ("Skewness is %g\n", skew);
printf ("kurtosis is %g\n\n", kurtosis);
//printf("Sigma in the bin is: %.2f\n\n", sig);
fprintf(sig, "%.2f\t %.2f\n", (lim[j]+lim[j+1])*0.5, sd);
}
fclose(sig);
script = fopen( "script1.gpl", "w" );
fprintf(script, "set grid\nset terminal png\nset output 'Sigma_vs_rad.png'\nset nokey\n");
fprintf( script, "set title 'Sigma vs Radius'\n" );
//fprintf( script, "set logscale x\n" );
fprintf( script, "set xlabel 'Radius in Arcmin'\n" );
//fprintf( script, "set yrange [25:100]\n" );
fprintf( script, "set ylabel 'Sigma (km/s)'\n" );
fprintf( script, "plot 'sigma.dat' u 1:2 pt 7 ps 1\n");
fclose(script);
warn = system("gnuplot script1.gpl");
return(warn);
}
int reformat_data(DATA *data) {
int i, j, id, cur;
double spec[data->nprey][data->_K_];
double allspec[data->nctrl];
int ctrlCounts[data->nprey];
// fprintf(stderr, "%d\n", data->nctrl);
/* reformat control data: do this only if(K > 5) */
for(i=0;i<data->nprey;i++) {
for(j=0;j<data->_K_;j++) spec[i][j] = 0.0;
}
if(data->nctrl > data->_K_) {
for(i=0;i<data->nprey;i++) {
cur = 0;
for(j=0;j<data->nctrl;j++) allspec[j] = 0;
for(j=0;j<data->preyNinter[i];j++) {
id = data->p2i[i][j];
// fprintf(stderr, "%s %s\n", data->bait[id], data->prey[id]);
if(data->ctrl[data->i2IP[id]]) {
allspec[cur] = data->d[id];
cur++;
}
}
ctrlCounts[i] = cur;
if(cur > 0) {
gsl_sort(allspec, 1, data->nctrl);
for(j=0;j<data->nctrl;j++) {
spec[i][j] = allspec[data->nctrl-1-j];
if(j >= data->_K_) break;
}
}
}
}
/* write interaction_intermediate */
FILE *fpi = fopen("interaction.intermediate", "w");
if(data->nctrl > data->_K_) {
for(i=0;i<data->ninter;i++) {
if(data->ctrl[data->i2IP[i]] == 0) {
if(data->type) fprintf(fpi, "%s\t%s\t%s\t%f\n", data->ip[i], data->bait[i], data->prey[i], data->d[i]);
else fprintf(fpi, "%s\t%s\t%s\t%d\n", data->ip[i], data->bait[i], data->prey[i], ((int) data->d[i]));
}
}
for(i=0;i<data->nprey;i++) {
if(ctrlCounts[i] > 0) {
for(j=0;j<data->_K_;j++) {
if(spec[i][j] > 0.0) {
if(data->type) fprintf(fpi, "CTRL%d\tCTRL%d\t%s\t%.3f\n", j+1, j+1, data->PREY[i], spec[i][j]);
else fprintf(fpi, "CTRL%d\tCTRL%d\t%s\t%d\n", j+1, j+1, data->PREY[i], (int) spec[i][j]);
}
}
}
}
}
else {
for(i=0;i<data->ninter;i++) {
if(data->type) fprintf(fpi, "%s\t%s\t%s\t%f\n", data->ip[i], data->bait[i], data->prey[i], data->d[i]);
else fprintf(fpi, "%s\t%s\t%s\t%d\n", data->ip[i], data->bait[i], data->prey[i], ((int) data->d[i]));
}
}
fclose(fpi);
/* write bait.new */
FILE *fpb = fopen("bait.new", "w");
if(data->nctrl > data->_K_) {
for(i=0;i<data->nIP;i++) {
if(data->ctrl[i] == 0) {
fprintf(fpb, "%s\t%s\tT\n", data->IP[i], data->BAIT[data->IP2b[i]]);
}
}
for(j=0;j<data->_K_;j++) fprintf(fpb, "CTRL%d\tCTRL%d\tC\n", j+1, j+1);
}
else {
for(i=0;i<data->nIP;i++) {
fprintf(fpb, "%s\t%s\t%s\n", data->IP[i], data->BAIT[data->IP2b[i]], data->ctrl[i] ? "C" : "T");
}
}
fclose(fpb);
return 0;
}
void generate_kmeans_centres(const double * X,const int dim_x,const int dim_n,const int dim_b,double * centres){
int i,N, iter,k,num_ix,num_empty_clusters;
int* ind_,*empty_clusters,*minDi,*ix;
size_t *sDi;
double * M, * D,*minDv,*X_ix,*X_ix_m,*X_ink,*sDv;
double dist_old, dist_new;
dist_old = 10000;
gsl_permutation *ind;
const gsl_rng_type *T;
gsl_rng * r;
// finish declaration
N = dim_n;
gsl_rng_env_setup();
T = gsl_rng_default;
r = gsl_rng_alloc(T);
gsl_rng_set(r,3);
ind = gsl_permutation_alloc(N);
gsl_permutation_init(ind);
gsl_ran_shuffle(r,ind->data,N,sizeof(size_t));
// gsl_permutation_fprintf(stdout,ind,"%u");
ind_ = malloc(dim_b*sizeof(int));
for (i=0;i<dim_b;i++){
ind_[i] = (int)(gsl_permutation_get(ind,i));
}
M = malloc(dim_x*dim_b*sizeof(double));
D = malloc(dim_b*dim_n*sizeof(double));
minDv = malloc(dim_n*sizeof(double));
minDi = malloc(dim_n*sizeof(int));
sDv = malloc(dim_n*sizeof(double));
sDi = malloc(dim_n*sizeof(int));
ix = malloc(dim_n*sizeof(int));
X_ix_m= malloc(dim_x*1*sizeof(double));
X_ink = malloc(dim_x*sizeof(double));
ccl_get_sub_mat_cols(X,dim_x,dim_n,ind_,dim_b,M);
empty_clusters = malloc(dim_b*sizeof(int));
num_empty_clusters = 0;
for (iter=0;iter<1001;iter++){
num_empty_clusters = 0;
ccl_mat_distance(M,dim_x,dim_b,X,dim_x,dim_n,D);
ccl_mat_min(D,dim_b,dim_n,1,minDv,minDi);
memcpy(sDv,minDv,dim_n*sizeof(double));
memset(empty_clusters,0,dim_b*sizeof(int));
for (k=0;k<dim_b;k++){
memset(ix,0,dim_n*sizeof(int));
num_ix = ccl_find_index_int(minDi,dim_n,1,k,ix);
// print_mat_i(ix,1,dim_n);
X_ix = malloc(dim_x*num_ix*sizeof(double));
if(num_ix!=0){// not empty
ccl_get_sub_mat_cols(X,dim_x,dim_n,ix,num_ix,X_ix);
ccl_mat_mean(X_ix,dim_x,num_ix,0,X_ix_m);
ccl_mat_set_col(M,dim_x,dim_b,k,X_ix_m);
}
else{
empty_clusters[num_empty_clusters] = k;
num_empty_clusters ++;
}
free(X_ix);
}
dist_new = ccl_vec_sum(minDv,dim_n);
if (num_empty_clusters == 0){
if(fabs(dist_old-dist_new)<1E-10) {
memcpy(centres,M,dim_x*dim_b*sizeof(double));
return;
}
}
else{
// print_mat_i(empty_clusters,1,num_empty_clusters);
gsl_sort_index(sDi,sDv,1,dim_n);
gsl_sort(sDv,1,dim_n);
for (k=0;k<num_empty_clusters;k++){
int ii = (int) sDi[dim_n-k-1];
//print_mat_d(X,dim_x,dim_n);
ccl_get_sub_mat_cols(X,dim_x,dim_n,&ii,1,X_ink);
ccl_mat_set_col(M,dim_x,dim_b,empty_clusters[k],X_ink);
}
}
dist_old = dist_new;
}
memcpy(centres,M,dim_x*dim_b*sizeof(double));
gsl_permutation_free(ind);
gsl_rng_free(r);
free(ind_);
free(empty_clusters);
free(minDi);
free(M);
free(minDv);
free(X_ink);
free(ix);
free(sDi);
free(sDv);
free(X_ix_m);
free(D);
}
void sigma_clip_median__cy(double* pixels, int n_pixels, int n_images, double* output,
double nsigma, int max_repeat)
{
int x,y,l, repeat;
int n_bytes = n_images*sizeof(bool);
// printf("n bytes to hold mask: %d\n", n_bytes);
bool *good_value = (bool*)malloc(n_images*sizeof(bool));
// printf("\n\n\ngood-value=%x\n\n\n",good_value);
double *pixelvalue = (double*)malloc(n_images*sizeof(double));
for (x=0; x<n_pixels ; x++) {
// set output to NaN, this is the default
output[x] = nan;
/* printf("\n\n\n\n\n\n\rColumn %d", x+1); */
//
// Figure out how many pixels are not NaN
//
int n_good_pixels = 0;
for (y=0; y<n_images; y++) {
if (!isnan(pixels[x*n_images+y])) {
good_value[y] = true;
pixelvalue[n_good_pixels] = pixels[x*n_images+y];
n_good_pixels++;
}
}
if (n_good_pixels < 1) {
// all pixels appear to be NaNs, nothing left to do
continue;
}
//
// Now sort all values
//
gsl_sort(pixelvalue, 1, n_good_pixels);
/* for (l=0; l<n_good_pixels; l++) { */
/* printf("%2d -> %.0f\n", l, pixelvalue[l]); */
/* } */
/* printf("\n"); */
double median, sigma_plus, sigma_minus, upper, lower, sigma;
int start=0;
int end=n_good_pixels;
int i, new_start, new_end, n_valid_values;
new_start = start;
new_end = end;
/* printf("Starting iterating: start/end=%d,%d, new-start/end=%d,%d\n", start, end, new_start, new_end); */
/* for (i=0; i<end; i++) { */
/* printf("%2d: %f\n", i, pixelvalue[i]); */
/* } */
for (repeat=0; repeat<max_repeat && (end-start)>=3; repeat++) {
end = new_end;
start = new_start;
n_valid_values = end - start;
/* printf("Iteration %d: start/end=%d,%d #=%d\n", repeat, start, end, n_valid_values); */
// Compute median and the sigma-widths
median = gsl_stats_median_from_sorted_data(&pixelvalue[start], 1, n_valid_values);
sigma_minus = median - gsl_stats_quantile_from_sorted_data(&pixelvalue[start], 1, n_valid_values, 0.16);
sigma_plus = gsl_stats_quantile_from_sorted_data(&pixelvalue[start], 1, n_valid_values, 0.84) - median;
sigma = 0.5 * (gsl_stats_quantile_from_sorted_data(&pixelvalue[start], 1, n_valid_values, 0.84) -
gsl_stats_quantile_from_sorted_data(&pixelvalue[start], 1, n_valid_values, 0.16));
// Compute the valid range of pixels
lower = median - nsigma * sigma;
upper = median + nsigma * sigma;
/* lower = median - nsigma * sigma_minus; */
/* upper = median + nsigma * sigma_plus; */
// Now figure out the start and end range of pixels within the range
/* printf("Lower limit:\n"); */
for (new_start=start; new_start<end; new_start++) {
if (pixelvalue[new_start] >= lower) {
/* printf("Value %d is %f >= %f, taking as new lower limit\n", */
/* new_start, pixelvalue[new_start], lower); */
break;
}
/* printf("Value %d is %f < %f, skipping\n", */
/* new_start, pixelvalue[new_start], lower); */
}
for (new_end = end; new_end > new_start;) {
new_end--;
if (pixelvalue[new_end] <= upper) {
/* printf("Value %d is %f <= %f, taking as new upper limit\n", */
/* new_end, pixelvalue[new_end], upper); */
// Make stick to nomenclature that end means the index of the first
// entry that is no longer contained in the list
new_end++;
break;
}
/* printf("Value %d is %f > %f, skipping\n", */
/* new_end, pixelvalue[new_end], upper); */
}
/* printf("Was: %d/%d now is %d,%d\n", */
/* start, end, new_start, new_end); */
/* printf("Iteration %d: median=%f (sigma= %f / %f) lower/upper: %f %f #=%d\n", */
/* repeat, median, sigma_minus, sigma_plus, lower, upper, n_valid_values); */
/* printf("%d --> %d D=%d (%d)\n", */
/* pixelvalue, &pixelvalue[start], &pixelvalue[start]-pixelvalue, start); */
/* printf("Remaining valid values:\n"); */
/* for (i=new_start; i<new_end; i++) { */
/* printf("%f\n", pixelvalue[i]); */
/* } */
if (new_start == start && new_end == end) {
// This iteration hasn't changed anything, so future iteration won't change
// anything either, so we can stop right here
/* printf("No change in this iteration skipping rest of work.\n"); */
break;
}
}
if ((new_end - new_start) >= 3) {
end = new_end;
start = new_start;
}
//
// Now we have all pixels limited down to iteratively select only the
// 3-sigma range for each pixel; Now compute the mean value.
//
output[x] = gsl_stats_median_from_sorted_data(&pixelvalue[start], 1, (end-start));
// output[x] = (double)(end-start); //gsl_stats_mean(&pixelvalue[start], 1, (end-start));
// printf("filtered output: %f (%d, %d)\n", output[x], start, end);
}
// printf("\n\ngood-value=%x\n\n",good_value);
/* free(good_value); */
// printf("Freeing memory\n");
free((void*)good_value);
// printf("done freeing\n");
// printf("Work done in c_sigclip.c\n");
return;
}
int main(int argc, char *argv[])
{
if(argc<3)
{
printf("Not enough arguments\n");
printf("Number of arguments:%d\n",argc);
exit(1);
}
if(!CheckFile(argv[1]))
{
printf("\nfile argv[1]:%s does not exist!.\n",argv[1]);
// USAGE;
exit(1);
}
char *name_file_input = argv[1];
char *name_file_output = argv[2];
int i, j;
int nb_lines, nb_counts, nb_massRanges;
char char_dumb[20];
int *treeId;
double mass, dmass, median, quartil1, quartil3, sigma2, mean;
double *nodeMass, *diskStellarMass, *diskGasMass, *diskCircularVelocity, *spheroidStellarMass, *spheroidGasMass, *bulgeFractionInRange;
FILE *file_input, *file_output;
////////////////////////////////////////////////////////////////////////////////////////////////////
file_input = fopen(name_file_input,"r");
nb_lines = 0;
while( !feof(file_input) )
{
for(i=0; i<5; i++)
fscanf(file_input,"%s", char_dumb);
nb_lines++;
}
fclose(file_input);
nb_lines=nb_lines-2;
printf("Number of lines:%d\n",nb_lines);
treeId = malloc((size_t)nb_lines*sizeof(int));
nodeMass = malloc((size_t)nb_lines*sizeof(double));
diskStellarMass = malloc((size_t)nb_lines*sizeof(double));
diskGasMass = malloc((size_t)nb_lines*sizeof(double));
diskCircularVelocity = malloc((size_t)nb_lines*sizeof(double));
spheroidStellarMass = malloc((size_t)nb_lines*sizeof(double));
spheroidGasMass = malloc((size_t)nb_lines*sizeof(double));
file_input = fopen(name_file_input,"r");
for(i=0; i<5; i++)
fscanf(file_input,"%s", char_dumb);
for(i=0; i<nb_lines; i++ )
fscanf(file_input,"%lf %lf %lf %lf %lf"
, &nodeMass[i], &spheroidStellarMass[i], &spheroidGasMass[i], &diskStellarMass[i], &diskGasMass[i]
);
fclose(file_input);
////////////////////////////////////////////////////////////////////////////////////////////////////
file_output = fopen(name_file_output,"w");
fprintf(file_output, "#1.haloMass #2.quartil1 #3.median #4.quartil3 #5.quartil4\n");
nb_massRanges = 60;
dmass = (11.2-10.0)/nb_massRanges;
for(mass=10.0; mass<11.2; mass+=dmass)
{
j = 0;
for(i=0; i<nb_lines; i++ )
if( (log10(diskStellarMass[i] + spheroidStellarMass[i]) >= mass) && (log10(diskStellarMass[i] + spheroidStellarMass[i]) < mass+dmass)
&& (spheroidStellarMass[i]/(diskStellarMass[i] + spheroidStellarMass[i] + diskGasMass[i])<=0.6) )
j++;
//&& (diskStellarMass[i] + spheroidStellarMass[i]> 0.0)
nb_counts = j;
//printf("number of matches:%d\n",nb_counts);
bulgeFractionInRange = malloc((size_t)nb_counts*sizeof(double));
j = 0;
for(i=0; i<nb_lines; i++ )
if( (log10(diskStellarMass[i] + spheroidStellarMass[i]) >= mass) && (log10(diskStellarMass[i] + spheroidStellarMass[i]) < mass+dmass)
&& (spheroidStellarMass[i]/(diskStellarMass[i] + spheroidStellarMass[i] + diskGasMass[i])<=0.6))
{
//&& (diskStellarMass[i] + spheroidStellarMass[i]> 0.0)
bulgeFractionInRange[j] = spheroidStellarMass[i]/(diskGasMass[i] + spheroidGasMass[i] + diskStellarMass[i] + spheroidStellarMass[i]);
j++;
}
gsl_sort(bulgeFractionInRange, 1, nb_counts);
mean = gsl_stats_mean (bulgeFractionInRange, 1, nb_counts);
median = gsl_stats_median_from_sorted_data(bulgeFractionInRange, 1, nb_counts);
sigma2 = gsl_stats_variance (bulgeFractionInRange, 1, nb_counts);
quartil1 = gsl_stats_quantile_from_sorted_data (bulgeFractionInRange, 1, nb_counts, 0.25);
quartil3 = gsl_stats_quantile_from_sorted_data (bulgeFractionInRange, 1, nb_counts, 0.75);
//quartil4 = gsl_stats_quantile_from_sorted_data (gasFractionInRange, 1, nb_counts, 1.0);
fprintf(file_output,"%f %f %f %f %f %f\n", mass, quartil1, median, quartil3, mean, sigma2);
}
fclose(file_output);
return 0;
}
void compute_global_autocorrelations(struct State *S)
/* Compute population rate autocorrelation */
{
struct Network *ntw = &S->ntw;
int n_neurons_sample = 100;
const size_t n_bins = 201;
const double max_lag = 100; /* maximal lag in ms */
double bin_width = 2 * max_lag / (double) n_bins; /* 2*max_lag */
char filename[100];
gsl_histogram *h = gsl_histogram_alloc(n_bins);
gsl_histogram_set_ranges_uniform(h, -max_lag, max_lag);
FILE *f;
struct Dynamic_Array poptrain;
size_t num_spikes_total = 0;
for (int i = 0; i < n_neurons_sample; i++)
num_spikes_total += ntw->cell[i].spike_train.n;
poptrain.data = emalloc(num_spikes_total * sizeof(double));
poptrain.n = 0;
poptrain.size = num_spikes_total;
report("Computing global (population rate) autocorrelation...\n");
fill_population_spike_train(S, &poptrain, n_neurons_sample);
gsl_sort(poptrain.data, 1, num_spikes_total);
sprintf(filename, "global_autocorrelation_%s", S->sim.suffix);
double t_sp;
f = fopen(filename, "w");
size_t l = 0;
size_t r = 0;
for (size_t j = 0; j < poptrain.n; j++) {
t_sp = poptrain.data[j];
/* look for left index */
while (l < poptrain.n && poptrain.data[l] < t_sp - max_lag)
l++;
/* look for right index */
while (r < poptrain.n && poptrain.data[r] < t_sp + max_lag)
r++;
/* And feed the histogram with the relevant spikes */
for (size_t k = l; k < r; k++)
gsl_histogram_increment(h, t_sp - poptrain.data[k]);
}
/* gsl_histogram_fprintf(f, h, "%g", "%g"); */
/* correct for boundary effects and substract mean */
double w;
double T = S->sim.total_time - S->sim.offset;
/* double nu = num_spikes_total / (T * (double) n_neurons_sample); */
double lower, upper, ac;
int status;
for (size_t j = 0; j < n_bins; j++) {
w = T - fabs( (n_bins - 1.0) / 2.0 - j ) * bin_width;
ac = gsl_histogram_get(h, j) / (w * n_neurons_sample);
ac -= (pow(num_spikes_total / (T * n_neurons_sample), 2) * bin_width);
/* ac /= pow(nu * bin_width, 2); [> Normalization <] */
status = gsl_histogram_get_range(h, j, &lower, &upper);
if (status==0) {
fprintf(f, "% 9.4f % 9.4f % 9.6f\n", lower, upper, ac);
} else {
report("Somehting wrong here.\n");
exit(2);
}
}
fclose(f);
free(poptrain.data);
gsl_histogram_free(h);
}
double nsl_stats_median(double data[], size_t stride, size_t n, nsl_stats_quantile_type type) {
gsl_sort(data, stride, n);
return nsl_stats_median_sorted(data,stride,n,type);
}
int
main()
{
gsl_rng *r = gsl_rng_alloc(gsl_rng_default);
const double tol1 = 1.0e-8;
const double tol2 = 1.0e-3;
gsl_ieee_env_setup();
{
const size_t N = 2000000;
double *data = random_data(N, r);
double data2[] = { 4.0, 7.0, 13.0, 16.0 };
size_t i;
test_basic(2, data, tol1);
test_basic(100, data, tol1);
test_basic(1000, data, tol1);
test_basic(10000, data, tol1);
test_basic(50000, data, tol1);
test_basic(80000, data, tol1);
test_basic(1500000, data, tol1);
test_basic(2000000, data, tol1);
for (i = 0; i < 4; ++i)
data2[i] += 1.0e9;
test_basic(4, data2, tol1);
free(data);
}
{
/* dataset from Jain and Chlamtac paper */
const size_t n_jain = 20;
const double data_jain[] = { 0.02, 0.15, 0.74, 3.39, 0.83,
22.37, 10.15, 15.43, 38.62, 15.92,
34.60, 10.28, 1.47, 0.40, 0.05,
11.39, 0.27, 0.42, 0.09, 11.37 };
double expected_jain = 4.44063435326;
test_quantile(0.5, data_jain, n_jain, expected_jain, tol1, "jain");
}
{
size_t n = 1000000;
double *data = malloc(n * sizeof(double));
double *sorted_data = malloc(n * sizeof(double));
gsl_rstat_workspace *rstat_workspace_p = gsl_rstat_alloc();
double p;
size_t i;
for (i = 0; i < n; ++i)
{
data[i] = gsl_ran_gaussian_tail(r, 1.3, 1.0);
gsl_rstat_add(data[i], rstat_workspace_p);
}
memcpy(sorted_data, data, n * sizeof(double));
gsl_sort(sorted_data, 1, n);
/* test quantile calculation */
for (p = 0.1; p <= 0.9; p += 0.1)
{
double expected = gsl_stats_quantile_from_sorted_data(sorted_data, 1, n, p);
test_quantile(p, data, n, expected, tol2, "gauss");
}
/* test mean, variance */
{
const double expected_mean = gsl_stats_mean(data, 1, n);
const double expected_var = gsl_stats_variance(data, 1, n);
const double expected_sd = gsl_stats_sd(data, 1, n);
const double expected_skew = gsl_stats_skew(data, 1, n);
const double expected_kurtosis = gsl_stats_kurtosis(data, 1, n);
const double expected_median = gsl_stats_quantile_from_sorted_data(sorted_data, 1, n, 0.5);
const double mean = gsl_rstat_mean(rstat_workspace_p);
const double var = gsl_rstat_variance(rstat_workspace_p);
const double sd = gsl_rstat_sd(rstat_workspace_p);
const double skew = gsl_rstat_skew(rstat_workspace_p);
const double kurtosis = gsl_rstat_kurtosis(rstat_workspace_p);
const double median = gsl_rstat_median(rstat_workspace_p);
gsl_test_rel(mean, expected_mean, tol1, "mean");
gsl_test_rel(var, expected_var, tol1, "variance");
gsl_test_rel(sd, expected_sd, tol1, "stddev");
gsl_test_rel(skew, expected_skew, tol1, "skew");
gsl_test_rel(kurtosis, expected_kurtosis, tol1, "kurtosis");
gsl_test_abs(median, expected_median, tol2, "median");
}
free(data);
free(sorted_data);
gsl_rstat_free(rstat_workspace_p);
}
gsl_rng_free(r);
exit (gsl_test_summary());
}
double nsl_stats_quantile(double data[], size_t stride, size_t n, double p, nsl_stats_quantile_type type) {
gsl_sort(data, stride, n);
return nsl_stats_quantile_sorted(data,stride,n,p,type);
}
int main(int argc, char *argv []) {
if(populate_env_variable(REF_ERROR_CODES_FILE, "L2_ERROR_CODES_FILE")) {
printf("\nUnable to populate [REF_ERROR_CODES_FILE] variable with corresponding environment variable. Routine will proceed without error handling\n");
}
if (argc != 7) {
if(populate_env_variable(SPE_BLURB_FILE, "L2_SPE_BLURB_FILE")) {
RETURN_FLAG = 1;
} else {
print_file(SPE_BLURB_FILE);
}
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -1, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 1;
} else {
// ***********************************************************************
// Redefine routine input parameters
char *input_f = strdup(argv[1]);
char *method = strdup(argv[2]);
char *ss_method = strdup(argv[3]);
double target_half_aperture_px = strtod(argv[4], NULL);
int sky_window_half_aperture_px = strtol(argv[5], NULL, 0);
char *output_f = strdup(argv[6]);
// ***********************************************************************
// Open input file (ARG 1), get parameters and perform any data format
// checks
fitsfile *input_f_ptr;
int input_f_maxdim = 2, input_f_status = 0, input_f_bitpix, input_f_naxis;
long input_f_naxes [2] = {1,1};
if(!fits_open_file(&input_f_ptr, input_f, IMG_READ_ACCURACY, &input_f_status)) {
if(!populate_img_parameters(input_f, input_f_ptr, input_f_maxdim, &input_f_bitpix, &input_f_naxis, input_f_naxes, &input_f_status, "INPUT FRAME")) {
if (input_f_naxis != 2) { // any data format checks here
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -2, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -3, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -4, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
return 1;
}
// ***********************************************************************
// Set the range limits
int cut_x [2] = {1, input_f_naxes[0]};
int cut_y [2] = {1, input_f_naxes[1]};
// ***********************************************************************
// Set parameters used when reading data from input file (ARG 1)
long fpixel [2] = {cut_x[0], cut_y[0]};
long nxelements = (cut_x[1] - cut_x[0]) + 1;
long nyelements = (cut_y[1] - cut_y[0]) + 1;
// ***********************************************************************
// Create arrays to store pixel values from input fits file (ARG 1)
double input_f_pixels [nxelements];
// ***********************************************************************
// Get input fits file (ARG 1) values and store in 2D array
int ii;
double input_frame_values [nyelements][nxelements];
memset(input_frame_values, 0, sizeof(double)*nxelements*nyelements);
for (fpixel[1] = cut_y[0]; fpixel[1] <= cut_y[1]; fpixel[1]++) {
memset(input_f_pixels, 0, sizeof(double)*nxelements);
if(!fits_read_pix(input_f_ptr, TDOUBLE, fpixel, nxelements, NULL, input_f_pixels, NULL, &input_f_status)) {
for (ii=0; ii<nxelements; ii++) {
input_frame_values[fpixel[1]-1][ii] = input_f_pixels[ii];
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -5, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
return 1;
}
}
// ***********************************************************************
// Open [SPFIND_OUTPUTF_PEAKS_FILE] input file
FILE *inputfile;
if (!check_file_exists(SPFIND_OUTPUTF_PEAKS_FILE)) {
inputfile = fopen(SPFIND_OUTPUTF_PEAKS_FILE , "r");
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -6, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 1;
}
// ***********************************************************************
// Find some [SPFIND_OUTPUTF_PEAKS_FILE] input file details
char input_string [150];
int row_count = 0;
while(!feof(inputfile)) {
memset(input_string, '\0', sizeof(char)*150);
fgets(input_string, 150, inputfile);
if (strtol(&input_string[0], NULL, 0) > 0) { // check the line begins with a positive number (usable)
row_count++;
}
}
rewind(inputfile);
// ***********************************************************************
// Store [SPFIND_OUTPUTF_PEAKS_FILE] data
double x_coords[row_count];
memset(x_coords, 0, sizeof(double)*(row_count));
double y_coords[row_count];
memset(y_coords, 0, sizeof(double)*(row_count));
double coord_x, coord_y;
int idx = 0;
while(!feof(inputfile)) {
memset(input_string, '\0', sizeof(char)*150);
fgets(input_string, 150, inputfile);
if (strtol(&input_string[0], NULL, 0) > 0) { // check the line begins with a positive number (usable)
sscanf(input_string, "%lf\t%lf\n", &coord_x, &coord_y);
x_coords[idx] = coord_x;
y_coords[idx] = coord_y;
idx++;
}
}
double output_frame_values[nxelements];
memset(output_frame_values, 0, sizeof(double)*nxelements);
// ***********************************************************************
// EXTRACTION
if (strcmp(method, "simple") == 0) {
// ***********************************************************************
// PARTIAL PIXEL APERTURE
int ii, jj;
double y;
y = y_coords[0]; // should only be one bin in [SPFIND_OUTPUTF_PEAKS_FILE] data file
double this_col_value;
for (ii=0; ii<nxelements; ii++) {
this_col_value = 0.;
// ***********************************************************************
// Does [y] violate the img boundaries?
if ((y + target_half_aperture_px > nyelements) || (y - target_half_aperture_px <= 0)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -7, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
fclose(inputfile);
return 1;
}
// ***********************************************************************
// Extract flux within aperture
double y_low, y_high;
y_low = y-target_half_aperture_px-0.5;
y_high = y+target_half_aperture_px+0.5;
int y_low_floor, y_high_floor;
y_low_floor = floor(y-target_half_aperture_px-0.5); // 0.5 as taking (half_ap*2) + 1 as total aperture
y_high_floor = floor(y+target_half_aperture_px+0.5);
for (jj=y_low_floor; jj<=y_high_floor; jj++) {
if (jj == y_low_floor) { // outside pixel where partial flux needs to be taken into account
double partial_fraction_of_bin = (y_low_floor + 1) - y_low;
this_col_value += partial_fraction_of_bin * input_frame_values[jj][ii];
} else if (jj == y_high_floor) { // outside pixel where partial flux needs to be taken into account
double partial_fraction_of_bin = y_high - y_high_floor;
this_col_value += partial_fraction_of_bin * input_frame_values[jj][ii];
} else {
this_col_value += input_frame_values[jj][ii];
}
}
output_frame_values[ii] = this_col_value;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -8, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
fclose(inputfile);
return 1;
}
// ***********************************************************************
// SKY SUBTRACTION
if (strcmp(ss_method, "none") == 0) {
} else if (strcmp(ss_method, "median") == 0) {
// ***********************************************************************
// MEDIAN SUBTRACTION
int ii, jj;
double y;
y = y_coords[0]; // should only be one bin in [SPFIND_OUTPUTF_PEAKS_FILE] data file
double this_col_value;
for (ii=0; ii<nxelements; ii++) {
this_col_value = 0.;
// ***********************************************************************
// Does [y] violate the img boundaries?
if ((y + target_half_aperture_px + sky_window_half_aperture_px > nyelements) || (y - target_half_aperture_px - sky_window_half_aperture_px <= 0)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -9, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
fclose(inputfile);
return 1;
}
// ***********************************************************************
// Find pixels for sky aperture
int ap_lower_lo, ap_lower_hi, ap_upper_lo, ap_upper_hi;
ap_lower_lo = floor(y-target_half_aperture_px-0.5) - sky_window_half_aperture_px - 1;
ap_lower_hi = floor(y-target_half_aperture_px-0.5) - 1;
ap_upper_lo = floor(y+target_half_aperture_px+0.5) + 1;
ap_upper_hi = floor(y+target_half_aperture_px+0.5) + sky_window_half_aperture_px + 1;
int n_ap_values = (ap_lower_hi-ap_lower_lo) + (ap_upper_hi-ap_upper_lo) + 2;
double ap_values[n_ap_values];
int idx = 0;
for (jj=ap_lower_lo; jj<=ap_lower_hi; jj++) {
ap_values[idx] = input_frame_values[jj][ii];
idx++;
}
for (jj=ap_upper_lo; jj<=ap_upper_hi; jj++) {
ap_values[idx] = input_frame_values[jj][ii];
idx++;
}
// DEBUG
/*for (jj=0; jj<idx; jj++)
printf("%f,", ap_values[jj]);
printf("\n");
for (jj=0; jj<idx; jj++)
printf("%d,", jj);
printf("\n");*/
// Take median
double ap_values_sorted [n_ap_values];
memcpy(ap_values_sorted, ap_values, sizeof(double)*n_ap_values);
gsl_sort(ap_values_sorted, 1, (ap_lower_hi-ap_lower_lo) + (ap_upper_hi-ap_upper_lo) + 2);
double ap_values_median = gsl_stats_median_from_sorted_data(ap_values_sorted, 1, n_ap_values);
this_col_value = ap_values_median * ((target_half_aperture_px*2) + 1); // need to scale up to target extraction aperture size
output_frame_values[ii] -= this_col_value;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -10, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, input_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
fclose(inputfile);
return 1;
}
// ***********************************************************************
// Set output frame parameters
fitsfile *output_f_ptr;
int output_f_status = 0;
long output_f_naxes [2] = {nxelements, 1};
long output_f_fpixel = 1;
// ***********************************************************************
// Create and write [output_frame_values] to output file (ARG 6)
if (!fits_create_file(&output_f_ptr, output_f, &output_f_status)) {
if (!fits_create_img(output_f_ptr, INTERMEDIATE_IMG_ACCURACY[0], 2, output_f_naxes, &output_f_status)) {
if (!fits_write_img(output_f_ptr, INTERMEDIATE_IMG_ACCURACY[1], output_f_fpixel, nxelements, output_frame_values, &output_f_status)) {
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -11, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, output_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
if(fits_close_file(output_f_ptr, &output_f_status)) fits_report_error (stdout, output_f_status);
fclose(inputfile);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -12, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, output_f_status);
free(input_f);
free(output_f);
free(method);
free(ss_method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
if(fits_close_file(output_f_ptr, &output_f_status)) fits_report_error (stdout, output_f_status);
fclose(inputfile);
return 1;
}
} else {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -13, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error(stdout, output_f_status);
free(input_f);
free(output_f);
free(method);
if(fits_close_file(input_f_ptr, &input_f_status)) fits_report_error (stdout, input_f_status);
fclose(inputfile);
return 1;
}
// ***********************************************************************
// Free arrays on heap
free(input_f);
free(output_f);
free(method);
free(ss_method);
// ***********************************************************************
// Close input file (ARG 1) and output file (ARG 6)
if(fits_close_file(input_f_ptr, &input_f_status)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -14, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error (stdout, input_f_status);
return 1;
}
if(fits_close_file(output_f_ptr, &output_f_status)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", -15, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
fits_report_error (stdout, output_f_status);
return 1;
}
if (fclose(inputfile)) {
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATTR", -16, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 1;
}
// Write success to [ERROR_CODES_FILE]
write_key_to_file(ERROR_CODES_FILE, REF_ERROR_CODES_FILE, "L2STATEX", RETURN_FLAG, "Status flag for L2 spextract routine", ERROR_CODES_FILE_WRITE_ACCESS);
return 0;
}
}
static void smfFlagSpikesPar( void *job_data_ptr, int *status ) {
size_t box; /* Box size */
size_t bstride; /* Vector stride between bolometer samples */
double *dat = NULL; /* Pointer to bolo data */
double dnew; /* Data value being added into the filter box */
double dold; /* Data value being removed from the filter box*/
int iadd; /* Index within box at which to store new value*/
dim_t ibolo; /* Bolometer index */
dim_t ibox; /* Index within box */
dim_t iold; /* Index of oldest value in "w2" */
dim_t inbox; /* Number of values in current filter box */
int inoise; /* Index within noisebox element to be removed */
int iremove; /* Index within box of element to be removed */
dim_t itime; /* Time-slice index */
dim_t lasttime; /* Last time-slice index to check for spikes */
double lmedian; /* Median value in previous filter box */
smf_qual_t mask; /* quality bit mask */
double median; /* Median value in current filter box */
dim_t nbolo; /* Number of bolometers */
size_t newstride; /* Vector stride to new time sample */
size_t nflag; /* Number of samples flagged */
int nn; /* Number of good values in noise box */
double noise; /* Local noise estimate */
double *noisebox = NULL; /* Pointer to array holding values for
calculating local noise */
double nsum; /* Sum of values in noise box */
double nsum2; /* Sum of squared values in noise box */
dim_t ntime; /* Number of time-slices */
double *pdat = NULL; /* Pointer to next bolo data value */
double *pdat0 = NULL; /* Pointer to first bolo data value */
double *pdat1 = NULL; /* Pointer to last bolo data value */
smfFlagSpikesData *pdata=NULL;
double *pn = NULL; /* Pointer to next "noisebox" value */
smf_qual_t *pqua = NULL; /* Pointer to next quality flag */
smf_qual_t *pqua0 = NULL; /* Pointer to first bolo quality value */
smf_qual_t *pqua1 = NULL; /* Pointer to last bolo quality value */
double *pw1 = NULL; /* Pointer to next "w1" value */
double *pw2 = NULL; /* Pointer to next "w2" value */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
int spike; /* Is current time slice a spike? */
double thresh; /* threshold for spikes */
size_t tstride; /* Vector stride between time samples */
double umedian; /* Lagged median value */
double *w1 = NULL; /* Array holding sorted data values */
double *w2 = NULL; /* Array holding un-sorted data values */
/* Retrieve job data */
pdata = job_data_ptr;
box = pdata->box;
bstride = pdata->bstride;
dat = pdata->dat;
mask = pdata->mask;
nbolo = pdata->nbolo;
ntime = pdata->ntime;
qua = pdata->qua;
thresh = pdata->thresh;
tstride = pdata->tstride;
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "",
"smfFlagSpikesPar: thread starting on bolos %zu -- %zu",
status, pdata->b1, pdata->b2 );
/* Initialise the number of spikes found. */
nflag = 0;
/* Store the largest time slice index to be checked. */
lasttime = ntime - ( box + 1 )/2;
/* Store the offset from the central value in a filter box to the next
time slice value to enter the filter box (i.e. the time slice just
in front of the current filter box). */
newstride = ( ( box + 1 )/2 )*tstride;
/* Allocate work arrays. */
w1 = astMalloc( sizeof( *w1 )*box );
w2 = astMalloc( sizeof( *w2 )*box );
noisebox = astMalloc( sizeof( *noisebox )*box );
/* Initialise pointers to the first data value and quality value for the
first bolometer. */
pdat0 = dat + pdata->b1*bstride;
pqua0 = qua + pdata->b1*bstride;
/* Initialise pointers to the last data value and quality value for the first
bolometer. */
pdat1 = dat + ( ntime - 1 )*tstride + pdata->b1*bstride;
pqua1 = qua + ( ntime - 1 )*tstride + pdata->b1*bstride;
/* We process each bolometer in turn for this block. */
for( ibolo=pdata->b1; ibolo<=pdata->b2 && *status == SAI__OK; ibolo++ ) {
/* Ignore bad bolometers. */
if( !( *pqua0 & SMF__Q_BADB ) ) {
/* Initialise the running sums used to calculate the standard deviation
of the values in the initial noise box. */
nsum = 0.0;
nsum2 = 0.0;
nn = 0;
/* The highest element in the noise box is the element just below the
central element of the filter box (so the central filter box element
does not affect the noise level estimate). Initially, the lower half
of the noise box is off the edge of the array and so we fill it with bad
values. */
pn = noisebox;
for( ibox = 0; ibox < ( box + 1 )/2; ibox++ ) *(pn++) = VAL__BADD;
/* Initialise the filter box to contain the first "box" values from the
current bolometer time-series. Do not store bad or flagged values in
the filter box. The good values are stored at the start of the "w1"
array, with no gaps. The "w2" array holds all values in the box, good
or bad, in the order they occur in the time-series (i.e. un-sorted). */
pdat = pdat0;
pqua = pqua0;
pw1 = w1;
pw2 = w2;
for( ibox = 0; ibox < box; ibox++ ) {
if( !( *pqua & mask ) && *pdat != VAL__BADD ) {
*(pw2++) = *(pw1++) = *pdat;
} else {
*(pw2++) = VAL__BADD;
}
/* Also store values these data values in the second half of the noise
box. Check we have not reached the end of the noise box. */
if( pn - noisebox < (int) box ) {
/* Store the value in the next element of the noise box, and if the value is
good, update the running sums used for calculating the noise level. */
if( ( *(pn++) = pw2[-1] ) != VAL__BADD ) {
nsum += *pdat;
nsum2 += (*pdat)*(*pdat);
nn++;
}
}
/* Get pointers to the next data and quality values for the current
bolometer. */
pdat += tstride;
pqua += tstride;
}
/* Calculate the standard deviation of the values in the initial noise box.
We require at least three values. */
if( nn > 3 ) {
noise = ( nsum2 - (nsum*nsum)/nn )/( nn - 1 );
if( noise > 0 ) {
noise = sqrt( noise );
} else {
noise = VAL__BADD;
}
} else {
noise = VAL__BADD;
}
/* Store the index within the noise box at which to store the next value
(the new value will over-write the oldest value in the noise box -
i.e. the first element). */
inoise = 0;
/* Initialise the index at which to store the next bolometer data value in the
"w2" array. The first new value added to the box will over-write element
zero - the oldest value in the box. */
iold = 0;
/* Note the number of good values stored in the filter box. */
inbox = (int)( pw1 - w1 );
/* If there are any bad data values, pad out the w1 array with bad
values. */
for( ibox = inbox; ibox < box; ibox++ ) w1[ ibox ] = VAL__BADD;
/* If any good values are stored in the filter box, we now sort them. */
if( inbox > 0 ) gsl_sort( w1, 1, inbox );
/* Get pointers to the data value and quality value at the centre of the
first filter box. */
pdat = pdat0 + (box/2)*tstride;
pqua = pqua0 + (box/2)*tstride;
/* Initialise the median value in the previous box to "unknown". */
lmedian = VAL__BADD;
/* Loop round all time slices, except for the first and last "box/2"
slices, which are left unchanged. "itime" is the index of the time
slice at the centre of the filter box. */
for( itime = box/2; itime <= lasttime; itime++ ) {
/* Assume this time slice is not a spike. */
spike = 0;
/* If the current sorted filter box contains an odd number of good values,
use the central good value as the median value. If the box contains an
even number of good values, use the mean of the two central values as
the median value. If the box is empty use VAL__BADD. */
if( inbox == 0 ) {
median = VAL__BADD;
} else if( inbox % 2 == 1 ) {
median = w1[ inbox/2 ];
} else {
ibox = inbox/2;
median = 0.5*( w1[ ibox ] + w1[ ibox - 1 ] );
}
/* If we have all the values we need, we can check for a spike. */
if( !( *pqua & mask ) && *pdat != VAL__BADD &&
median != VAL__BADD && noise != VAL__BADD ) {
/* The median is a robust but intrinsically noisey estimator of the expected
value. So we smooth the median values a bit by using the mean of the
current median and the previous median. */
if( lmedian != VAL__BADD ) {
umedian = 0.5*( median + lmedian );
} else {
umedian = median;
}
/* If the central value in the filter box deviates from the median by more than
"thresh" times the current noise estimate, flag it as a spike. Note, we can
modify the quality array directly because we will not be re-reading the value
being modified (all the reading is done at the leading edge of the filter
box). */
if( fabs( *pdat - umedian ) > thresh*noise ) {
*pqua |= SMF__Q_SPIKE;
nflag++;
spike = 1;
}
}
/* Save the median in the current box so that we can use it to smooth the
next box. */
lmedian = median;
/* Now move the filter box on by one time sample so that we are ready for
the next "itime" value. Do not do this if we have just done the last
itime value. */
if( itime < lasttime ) {
/* Get the data value for the time slice that is about to enter the filter
box. Set it bad if it is flagged in the quality array. */
dnew = pdat[ newstride ];
if( pqua[ newstride ] & mask ) dnew = VAL__BADD;
/* Get the data value for the time slice that is about to leave the filter
box. */
dold = w2[ iold ];
/* Store the new value in "w2" in place of the old value, and then
increment the index of the next "w2" value to be removed, wrapping back
to the start when the end of the array is reached. */
w2[ iold++ ] = dnew;
if( iold == box ) iold = 0;
/* If the new value to be added into the box is good... */
if( dnew != VAL__BADD ) {
/* Find the index (iadd) within the w1 box at which to store the new
value so as to maintain the ordering of the values in the box. Could do
a binary chop here, but the box size is presumably going to be very
small and so it's probably not worth it. At the same time, look for
the value that is leaving the box (if it is not bad). */
iremove = -1;
iadd = -1;
if( dold != VAL__BADD ) {
for( ibox = 0; ibox < inbox; ibox++ ) {
if( iremove == -1 && w1[ ibox ] == dold ) {
iremove = ibox;
if( iadd != -1 ) break;
}
if( iadd == -1 && w1[ ibox ] >= dnew ) {
iadd = ibox;
if( iremove != -1 ) break;
}
}
} else {
for( iadd = 0; iadd < (int) inbox; iadd++ ) {
if( w1[ iadd ] >= dnew ) break;
}
}
/* If the new value is larger than any value currently in w1, we add it
to the end. */
if( iadd == -1 ) iadd = inbox;
/* If the value being removed is bad, shuffle all the good values greater
than the new value up one element, and increment the number of good
values for this bolometer box. */
if( iremove == -1 ) {
for( ibox = inbox; (int) ibox > iadd; ibox-- ) {
w1[ ibox ] = w1[ ibox - 1 ];
}
inbox++;
/* If the value being removed is good and the value being added is greater
than the value being removed, shuffle all the intermediate values down
one element within the box. */
} else if( (int) iadd > iremove ) {
for( ibox = iremove; (int) ibox < iadd - 1; ibox++ ) {
w1[ ibox ] = w1[ ibox + 1 ];
}
iadd--;
/* If the value being removed is good and the value being added is less
than or equal to the value being removed, shuffle all the intermediate
values up one element within the box. */
} else {
for( ibox = iremove; (int) ibox > iadd; ibox-- ) {
w1[ ibox ] = w1[ ibox - 1 ];
}
}
/* Store the new value in the box. */
w1[ iadd ] = dnew;
/* If the value being added is bad but the value being removed is good... */
} else if( dold != VAL__BADD ){
/* Find the index (iremove) within the w1 box at which is stored the value
that is leaving the box. */
iremove = -1;
for( iremove = 0; iremove < (int) inbox; iremove++ ) {
if( w1[ iremove ] == dold ) break;
}
/* Move all the larger values down one element in "w1" to fill the gap
left by the removal. */
for( iremove++; iremove < (int) inbox; iremove++ ) {
w1[ iremove - 1 ] = w1[ iremove ];
}
/* Over-write the un-used last element with a bad value, and decrement
the number of values in "w1". */
w1[ iremove - 1 ] = VAL__BADD;
inbox--;
}
/* Get the new value (the central value in the filter box) to add to the
noise box. If the central value was found to be a spike, do not
include it in the noise box as it would upset the estimate of the
local noise. */
if( spike ) {
dnew = VAL__BADD;
} else {
dnew = *pdat;
if( *pqua & mask ) dnew = VAL__BADD;
}
/* Get the old value to remove from the noise box. */
dold = noisebox[ inoise ];
/* If the value being added is good, add it into the running sums used to
calculate the standard deviation of the values in the local noise box. */
if( dnew != VAL__BADD ) {
nsum += dnew;
nsum2 += dnew*dnew;
nn++;
}
/* If the value being removed is good, remove it from the running sums used
to calculate the standard deviation of the values in the local noise box. */
if( dold != VAL__BADD ) {
nsum -= dold;
nsum2 -= dold*dold;
nn--;
}
/* Calculate the standard deviation of the values in the new noise box.
We require at least three values. */
if( nn > 3 ) {
noise = ( nsum2 - (nsum*nsum)/nn )/( nn - 1 );
if( noise > 0 ) {
noise = sqrt( noise );
} else {
noise = VAL__BADD;
}
} else {
noise = VAL__BADD;
}
/* Store the new value in the noise box, over-writing the oldest value,
and increment the index at which to store the next value. If the index
reaches the end of the array, wrap around to the start of the array. */
noisebox[ inoise++ ] = dnew;
if( inoise == (int) box ) inoise = 0;
}
/* Get pointers to the next central data and quality value for the current
bolometer. */
pdat += tstride;
pqua += tstride;
}
}
/* Get pointers to the first data value and quality value for the next
bolometer. */
pdat0 += bstride;
pqua0 += bstride;
/* Get pointers to the last data value and quality value for the next
bolometer. */
pdat1 += bstride;
pqua1 += bstride;
}
/* Free resources. */
w1 = astFree( w1 );
w2 = astFree( w2 );
noisebox = astFree( noisebox );
/* Store number of flagged samples */
pdata->nflag = nflag;
/* Debugging message indicating thread finished work */
msgOutiff( SMF__TIMER_MSG, "",
"smfFlagSpikesPar: thread finishing bolos %zu -- %zu",
status, pdata->b1, pdata->b2 );
}
