void smf_calc_smoothedwvm ( ThrWorkForce *wf, const smfArray * alldata,
const smfData * adata, AstKeyMap* extpars, double **wvmtau,
size_t *nelems, size_t *ngoodvals, int * status ) {
size_t i;
size_t nrelated = 0; /* Number of entries in smfArray */
size_t nframes = 0; /* Number of timeslices */
size_t ngood = 0; /* Number of elements with good tau */
double *taudata = NULL; /* Local version of WVM tau */
const smfArray * thesedata = NULL; /* Collection of smfDatas to analyse */
smfArray * tmpthesedata = NULL; /* Local version of adata in a smfArray */
if (*status != SAI__OK) return;
if (alldata && adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm can not be given non-NULL alldata and non-NULL adata arguments"
" (possible programming error)", status );
return;
}
if (!alldata && !adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm: One of alldata or adata must be non-NULL",
status);
return;
}
if (!wvmtau) {
*status = SAI__ERROR;
errRep("", "Must supply a non-NULL pointer for wvmtau argument"
" (possible programming error)", status );
return;
}
/* if we have a single smfData put it in a smfArray */
if (alldata) {
if (alldata->ndat == 0 ) {
*status = SAI__ERROR;
errRep("", "No smfDatas present in supplied smfArray for WVM smoothing"
" (possible programming error)", status );
return;
}
thesedata = alldata;
} else {
tmpthesedata = smf_create_smfArray( status );
if (tmpthesedata) {
tmpthesedata->owndata = 0; /*not owned by the smfArray */
/* we know that the smfData here will not be touched in this
function so we do the BAD thing of casting const to non-const */
smf_addto_smfArray( tmpthesedata, (smfData *)adata, status );
}
thesedata = tmpthesedata;
}
/* Check that we have headers and that the smfData are the same length */
nrelated = thesedata->ndat;
for (i = 0; i < nrelated; i++ ) {
smfData * data = (thesedata->sdata)[i];
smfHead * hdr = data->hdr;
dim_t thisframes = 0;
if ( !hdr) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has no header. Aborting WVM smoothing",
status, i );
return;
}
smf_get_dims( data, NULL, NULL, NULL, &thisframes, NULL, NULL, NULL, status );
if (!nframes) nframes = thisframes;
if (thisframes != nframes) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has different length. Aborting WVM smoothing",
status, i );
return;
}
}
/* We will need the earliest and last airmass value in order
to calculate a zenith tau */
/* As a first step, just fill the time series with calculated WVM
tau values even though we know there are about 240 fewer tau
readings in reality. This initial approach will make it easier to
use the smoothed data directly rather than having to interpolate
from the 1.2 second data back into the 200 Hz data. */
taudata = astCalloc( nframes, sizeof(*taudata) );
if (*status == SAI__OK) {
double amprev = VAL__BADD;
double steptime;
size_t maxgap;
struct timeval tv1;
struct timeval tv2;
smfCalcWvmJobData *job_data = NULL;
int nworker;
/* We need to know the steptime so we can define the max good gap
in seconds and convert it to steps*/
steptime = (thesedata->sdata)[0]->hdr->steptime;
maxgap = (size_t)( 5.0 / steptime ); /* 5 seconds is just larger than 2 WVM readings */
/* Assume all files have the same airmass information */
smf_find_airmass_interval( (thesedata->sdata)[0]->hdr, &rev, NULL, NULL, NULL, status );
smf_timerinit( &tv1, &tv2, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if (*status == SAI__OK) {
dim_t tstep;
int iworker;
smfCalcWvmJobData *pdata = NULL;
/* Get the number of time slices to process in each thread. */
if( nworker > (int) nframes ) {
tstep = 1;
} else {
tstep = nframes/nworker;
}
/* to return the same values for one thread and multiple threads
we need to break the threads on wvm sample boundaries wherever
possible. We make an initial estimate of the number of WVM measurements
by assuming one every two seconds. */
{
smfData * curdata = NULL;
size_t nwvm = 0;
double prevtime = VAL__BADD;
double curtime;
size_t *boundaries = astGrow(NULL, nframes*(size_t)(steptime/2.0), sizeof(*boundaries));
for (i=0; i<nframes; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
if (!curdata) {
curtime = VAL__BADD;
} else {
curtime = curdata->hdr->state->wvm_time;
}
if (curtime != prevtime || nwvm == 0 ) {
/* Store the index in the boundaries array */
nwvm++;
boundaries = astGrow(boundaries, nwvm, sizeof(*boundaries));
if (!boundaries) { /* this is serious */
if (*status == SAI__OK) *status = SAI__ERROR;
errRep("", "Error allocating temporary memory for WVM calculation\n",
status );
break;
}
boundaries[nwvm-1] = i;
prevtime = curtime;
}
}
/* No point using too many threads */
if (*status == SAI__OK) {
if (nworker >= (int)nwvm) {
nworker = nwvm;
/* Allocate a measurement per thread */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->t1 = boundaries[iworker];
if (iworker+1 < nworker) pdata->t2 = boundaries[iworker+1]-1;
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
} else {
/* Allocate the workers to slices of approximate size tstep */
size_t prevend = 0; /* End of previous slice */
size_t prevbnd = 0; /* Index into previous boundaries[] array selection */
for( iworker = 0; iworker < nworker; iworker++ ) {
size_t belowidx = prevend+1;
size_t aboveidx = nframes;
size_t lbnd;
size_t ubnd;
size_t j;
size_t guess;
pdata = job_data + iworker;
if (iworker == 0) { /* always start at the beginning */
pdata->t1 = 0;
} else { /* Start one after the previous block */
pdata->t1 = prevend + 1;
}
/* Now we have to find the end of this slice */
guess = (iworker*tstep) + tstep - 1;
if (guess <= pdata->t1) guess = pdata->t1 + tstep;
/* find nearest boundaries */
for (j=prevbnd; j<nwvm; j++) {
if ( boundaries[j] > guess ) {
aboveidx = boundaries[j];
ubnd = j;
if (j>0) {
belowidx = boundaries[j-1];
lbnd = j -1 ;
} else {
lbnd = 0;
}
break;
}
}
/* Choose the closest, making sure that we are not choosing t1 */
if ( (guess - belowidx < aboveidx - guess) && belowidx > pdata->t1 ) {
pdata->t2 = belowidx - 1;
prevbnd = lbnd;
} else {
pdata->t2 = aboveidx - 1;
prevbnd = ubnd;
}
prevend = pdata->t2;
if (prevend == nframes - 1 && iworker < nworker-1 ) {
/* we have run out of slices so just use fewer workers */
nworker = iworker + 1;
break;
}
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
}
/* Tidy up */
boundaries = astFree( boundaries );
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to fix the steps in each bolo, and then wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray *thrdata = NULL;
pdata = job_data + iworker;
pdata->nframes = nframes;
pdata->airmass = amprev; /* really need to get it from the start of each chunk */
pdata->taudata = taudata;
pdata->maxgap = maxgap;
/* Need to copy the smfDatas and create a new smfArray for each
thread */
thrdata = smf_create_smfArray( status );
for (i=0;i<nrelated;i++) {
smfData *tmpdata = NULL;
tmpdata = smf_deepcopy_smfData( wf, (thesedata->sdata)[i], 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( tmpdata, 0, status );
smf_addto_smfArray( thrdata, tmpdata, status );
}
pdata->thesedata = thrdata;
/* Need to do a deep copy of ast data and unlock them */
pdata->extpars = astCopy(extpars);
astUnlock( pdata->extpars, 1 );
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf__calc_wvm_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now free the resources we allocated during job creation
and calculate the number of good values */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray * thrdata;
pdata = job_data + iworker;
astLock( pdata->extpars, 0 );
pdata->extpars = astAnnul( pdata->extpars );
thrdata = pdata->thesedata;
for (i=0;i<thrdata->ndat;i++) {
smf_lock_data( (thrdata->sdata)[i], 1, status );
}
smf_close_related( wf, &thrdata, status );
ngood += pdata->ngood;
}
}
job_data = astFree( job_data );
msgOutiff( MSG__NORM, "", FUNC_NAME ": %f s to calculate unsmoothed WVM tau values",
status, smf_timerupdate(&tv1,&tv2,status) );
}
if (*status == SAI__OK && extpars) {
/* Read extpars to see if we need to smooth */
double smoothtime = VAL__BADD;
if (astMapGet0D( extpars, "SMOOTHWVM", &smoothtime ) ) {
if (smoothtime != VAL__BADD && smoothtime > 0.0) {
smfData * data = (thesedata->sdata)[0];
double steptime = data->hdr->steptime;
dim_t boxcar = (dim_t)( smoothtime / steptime );
msgOutiff( MSG__VERB, "",
"Smoothing WVM data with %f s tophat function",
status, smoothtime );
smf_tophat1D( taudata, nframes, boxcar, NULL, 0, 0.0, status );
/* The tophat smoothing puts a bad value at the start and end of
the time series so we replace that with the adjacent value since
the step time is much smaller than WVM readout time. If more than
one value is bad we do not try to find the good value. */
taudata[0] = taudata[1];
taudata[nframes-1] = taudata[nframes-2];
}
}
}
/* Use this to get the raw WVM output for debugging */
/*
if (*status == SAI__OK) {
smfData *data = (thesedata->sdata)[0];
smfHead *hdr = data->hdr;
printf("# IDX TAU RTS_NUM RTS_END WVM_TIME\n");
for (i=0; i<nframes;i++) {
JCMTState * state;
state = &(hdr->allState)[i];
printf("%zu %.*g %d %.*g %.*g\n", i, DBL_DIG, taudata[i], state->rts_num,
DBL_DIG, state->rts_end, DBL_DIG, state->wvm_time);
}
} */
/* Free resources */
if (tmpthesedata) smf_close_related( wf, &tmpthesedata, status );
if (*status != SAI__OK) {
if (taudata) taudata = astFree( taudata );
*nelems = 0;
*ngoodvals = 0;
} else {
*wvmtau = taudata;
*nelems = nframes;
*ngoodvals = ngood;
}
}
static void smf1_calcmodel_smo_job( void *job_data, int *status ) {
/*
* Name:
* smf1_calcmodel_smo_job
* Purpose:
* Smooth each bolometer.
* Invocation:
* void smf1_calcmodel_smo_job( void *job_data, int *status )
* Arguments:
* job_data = void * (Given)
* Pointer to the data needed by the job. Should be a pointer to a
* smfCalcmodelSmoJobData structure.
* status = int * (Given and Returned)
* Pointer to global status.
* Description:
* This routine runs in a worker thread to smooth the specified bolometer
* time streams using the specified filter.
*/
/* Local Variables: */
dim_t b1;
dim_t b2;
dim_t i;
dim_t nbolo;
dim_t ntslice;
double *boldata = NULL;
double *model_data;
double *res_data;
double *w1 = NULL;
int *w3 = NULL;
smf_filt_t filter_type;
size_t boxcar;
size_t bstride;
size_t tstride;
size_t *w2 = NULL;
smfCalcmodelSmoJobData *pdata;
smf_qual_t *bolqua = NULL;
smf_qual_t *qua_data;
struct timeval tv1;
struct timeval tv2;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Get a pointer to the job data, and then extract its contents into a
set of local variables. */
pdata = (smfCalcmodelSmoJobData *) job_data;
b1 = pdata->b1;
b2 = pdata->b2;
boxcar = pdata->boxcar;
bstride = pdata->bstride;
filter_type = pdata->filter_type;
model_data = pdata->model_data;
nbolo = pdata->nbolo;
ntslice = pdata->ntslice;
qua_data = pdata->qua_data;
res_data = pdata->res_data;
tstride = pdata->tstride;
/* Check we have something to do. */
if( b1 < nbolo ) {
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "", "smf_calcmodel_smo: thread starting on bolos %" DIM_T_FMT
" -- %" DIM_T_FMT,
status, b1, b2 );
smf_timerinit( &tv1, &tv2, status);
/* Allocate work space */
boldata = astMalloc( ntslice*sizeof( *boldata ) );
bolqua = astMalloc( ntslice*sizeof( *bolqua ) );
if( filter_type == SMF__FILT_MEDIAN ) {
w1 = astMalloc( boxcar*sizeof( *w1 ) );
w2 = astMalloc( boxcar*sizeof( *w2 ) );
w3 = astMalloc( boxcar*sizeof( *w3 ) );
}
/* Loop round all the bolometers to be processed by this thread. */
for( i = b1; i <= b2 && *status == SAI__OK; i++ ) {
/* Smooth the data and subtract it from res */
size_t j;
size_t boloff = i*bstride;
/* Copy the data for this bolometer into a temporary buffer */
for (j=0; j<ntslice; j++) {
size_t thisidx = boloff+j*tstride;
boldata[j] = res_data[thisidx];
bolqua[j] = qua_data[thisidx];
}
/* Smooth that bolometer time series */
if( filter_type == SMF__FILT_MEDIAN ) {
smf_median_smooth( (dim_t) boxcar, SMF__FILT_MEDIAN, 0.0, ntslice,
res_data+boloff, qua_data+boloff, 1, SMF__Q_FIT,
boldata, w1, w2, w3, status );
} else {
smf_tophat1D( boldata, ntslice, boxcar, bolqua, SMF__Q_FIT,
0.0, status );
}
/* Remove this model from the residual data and copy the data to the model */
for (j=0; j<ntslice; j++) {
size_t thisidx = boloff+j*tstride;
/* Modify RES if this section has smoothable quality */
if (! (qua_data[thisidx] & SMF__Q_FIT) ) {
if (boldata[j] == VAL__BADD) {
/* Need to set to bad quality but since SMO can not
introduce bad values we shouldn't get here. For now
we use the COM quality bit just in case. */
qua_data[thisidx] |= SMF__Q_COM;
} else if (res_data[thisidx] != VAL__BADD) {
res_data[thisidx] -= boldata[j];
}
/* Store the model */
model_data[thisidx] = boldata[j];
} else {
/* store bad value in the model */
model_data[thisidx] = VAL__BADD;
}
}
}
/* Free work space. */
w1 = astFree( w1 );
w2 = astFree( w2 );
w3 = astFree( w3 );
boldata = astFree( boldata );
bolqua = astFree( bolqua );
/* Report the time taken in this thread. */
msgOutiff( SMF__TIMER_MSG, "",
"smf_calcmodel_smo: thread finishing bolos %" DIM_T_FMT
" -- %" DIM_T_FMT "(%.3f sec)",
status, b1, b2, smf_timerupdate( &tv1, &tv2, status ) );
}
}
void smf_filter2d_whiten( ThrWorkForce *wf, smfFilter *filt, smfData *map,
double minfreq, double maxfreq, size_t smooth,
int *status ) {
double A; /* Amplitude 1/f component */
double B; /* exponent of 1/f component */
double df; /* Frequency spacing in ref pspec */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t ndims=0; /* Number of real-space dimensions */
size_t nf=0; /* Number of frequencies in ref pspec */
smfData *map_fft=NULL; /* FFT of the map */
smfData *pspec=NULL; /* Az-averaged PSPEC of map */
double *pspec_data=NULL; /* Pointer to DATA comp. of pspec */
double *smoothed_filter=NULL; /* Smoothed power spectrum */
double W; /* White noise level */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter supplied.", status );
return;
}
if( filt->ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied filter is not for 2-d map.",
status );
return;
}
if( smf_isfft( map, NULL, NULL, NULL, NULL, &ndims, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied map is FFT instead of real space!",
status );
return;
}
/* Check for reasonable frequencies -- noting that maxfreq=0 defaults to
nyquist. */
if( maxfreq && (maxfreq < minfreq) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxfreq < minfreq!", status );
return;
}
/* Calculate azimuthally-averaged angular power spectrum */
map_fft = smf_fft_data( wf, map, NULL, 0, 0, status );
smf_fft_cart2pol( wf, map_fft, 0, 1, status );
pspec = smf_fft_2dazav( map_fft, &df, status );
if( *status == SAI__OK ) {
nf = pspec->dims[0];
pspec_data = pspec->pntr[0];
}
smf_close_file( &map_fft, status );
/* Fit power-law + white level model */
smf_fit_pspec( pspec_data, pspec->dims[0], 10, df, minfreq, 0.1,
maxfreq, &A, &B, &W, status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": P(f) = %lg*f^%lg + %lg\n", status, A, B, W );
if( smooth ) {
int whichaxis=0; /* index of higher-resolution axis */
double df_s; /* Frequency spacing of smoothed_filter */
double nf_s; /* Size of the smoothed 1-d filter */
/* Re-sample the radial power spectrum measured in the map on to
an array that corresponds to the highest-resolution (longer
real-space) dimension of the supplied filter */
if( filt->rdims[1] > filt->rdims[0] ) {
whichaxis = 1;
}
df_s = filt->df[whichaxis];
nf_s = filt->rdims[whichaxis]/2 + 1;
smoothed_filter = astMalloc( nf_s*sizeof(*smoothed_filter) );
if( *status == SAI__OK ) {
/* Nearest-neighbour... */
for( i=0; i<nf_s; i++ ) {
size_t nearest = round(i * df_s / df );
if( nearest >= nf ) nearest = nf-1;
smoothed_filter[i] = pspec_data[nearest];
}
}
/* Smooth pspec to estimate the radial power spectrum, but normalize
by the white-noise level from the fit to preserve
normalization -------------------------------------------------------- */
smf_tophat1D( smoothed_filter, nf_s, smooth, NULL, 0, 0.5, status );
/* Replace bad values (where not enough values to calculate median) with
the original values. Then normalize by the white noise level,
and take inverse square root */
if( *status == SAI__OK ) {
for( i=0; i<nf_s; i++ ) {
if( smoothed_filter[i] != VAL__BADD ) {
smoothed_filter[i] = 1./sqrt(smoothed_filter[i]/W);
} else if( i < smooth ) {
/* Filter to 0 at low frequencies where we can't estimate median */
smoothed_filter[i] = 0;
} else {
/* Close to Nyquist we set the filter gain to 1 so that it doesn't
do anything */
smoothed_filter[i] = 1;
}
}
}
/* Copy radial filter into 2d filter */
if( *status == SAI__OK ) {
size_t d; /* radial distance (FFT pixels) in smoothed filter */
double model; /* the value of the model (smoothed filter) at d */
double x; /* x- spatial frequency */
double y; /* y- spatial frequency */
for( i=0; i<filt->fdims[0]; i++ ) {
x = FFT_INDEX_TO_FREQ(i,filt->rdims[0]) * filt->df[0];
for( j=0; j<filt->fdims[1]; j++ ) {
y = FFT_INDEX_TO_FREQ(j,filt->rdims[1]) * filt->df[1];
d = (size_t) round(sqrt(x*x + y*y)/df_s);
if( d < nf_s) {
model = smoothed_filter[d];
} else {
model = 1;
}
filt->real[i + j*filt->fdims[0]] *= model;
if( filt->imag ) filt->imag[i + j*filt->fdims[0]] *= model;
}
}
}
} else {
/* --- otherwise use the smooth fitted model ---------------------------- */
if( *status == SAI__OK ) {
double d;
double model;
double x;
double y;
/* We fit a model to the power spectrum, but we want to apply its
complement to the FFT of the data. So we normalize by the
white-noise level, and take the square root of the model before
writing its complement to the filter buffer */
A = sqrt(A / W);
B = B / 2.;
for( i=0; i<filt->fdims[0]; i++ ) {
x = FFT_INDEX_TO_FREQ(i,filt->rdims[0]) * filt->df[0];
for( j=0; j<filt->fdims[1]; j++ ) {
y = FFT_INDEX_TO_FREQ(j,filt->rdims[1]) * filt->df[1];
d = sqrt(x*x + y*y);
model = 1. + A * pow(d,B);
filt->real[i + j*filt->fdims[0]] /= model;
if( filt->imag ) filt->imag[i + j*filt->fdims[0]] /= model;
}
}
}
}
/* Clean up */
if( pspec ) smf_close_file( &pspec, status );
if( smoothed_filter ) smoothed_filter = astFree( smoothed_filter );
}