void smf_bolonoise( ThrWorkForce *wf, smfData *data, double gfrac,
size_t window, double f_low,
double f_white1, double f_white2,
int nep, size_t len, double *whitenoise, double *fratio,
smfData **fftpow,int *status ) {
double *base=NULL; /* Pointer to base coordinates of array */
size_t bstride; /* bolometer index stride */
double df=1; /* Frequency step size in Hz */
size_t i; /* Loop counter */
size_t i_low; /* Index in power spectrum to f_low */
size_t i_w1; /* Index in power spectrum to f_white1 */
size_t i_w2; /* Index in power spectrum to f_white2 */
size_t j; /* Loop counter */
size_t mingood; /* Min. required no. of good values in bolometer */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t nf=0; /* Number of frequencies */
size_t ngood; /* Number of good samples */
dim_t ntslice; /* Number of time slices */
double p_low; /* Power at f_low */
double p_white; /* Average power from f_white1 to f_white2 */
smfData *pow=NULL; /* Pointer to power spectrum data */
smf_qual_t *qua=NULL; /* Pointer to quality component */
double steptime=1; /* Length of a sample in seconds */
size_t tstride; /* time index stride */
if (*status != SAI__OK) return;
/* Check inputs */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
if( !data->hdr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData has no header", status );
return;
}
/* Obtain dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( *status==SAI__OK ) {
steptime = data->hdr->steptime;
if( steptime < VAL__SMLD ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": FITS header error, STEPTIME must be > 0",
status);
} else {
/* Frequency steps in the FFT */
df = 1. / (steptime * (double) ntslice );
}
}
/* Initialize arrays */
if( whitenoise ) for(i=0; i<nbolo; i++) whitenoise[i] = VAL__BADD;
if( fratio ) for(i=0; i<nbolo; i++) fratio[i] = VAL__BADD;
/* FFT the data and convert to polar power spectral density form */
pow = smf_fft_data( wf, data, NULL, 0, len, status );
smf_convert_bad( wf, pow, status );
smf_fft_cart2pol( wf, pow, 0, 1, status );
{
dim_t fdims[2];
smf_isfft( pow, NULL, NULL, fdims, NULL, NULL, status );
if( *status == SAI__OK ) nf=fdims[0];
}
/* Check for reasonble frequencies, and integer offsets in the array */
i_low = smf_get_findex( f_low, df, nf, status );
i_w1 = smf_get_findex( f_white1, df, nf, status );
i_w2 = smf_get_findex( f_white2, df, nf, status );
/* Get the quality pointer from the smfData so that we can mask known
bad bolometer. */
qua = smf_select_qualpntr( data, NULL, status );
/* The minimum required number of good values in a bolometer. */
mingood = ( gfrac > 0.0 ) ? ntslice*gfrac : 0;
/* Loop over detectors */
for( i=0; (*status==SAI__OK)&&(i<nbolo); i++ )
if( !qua || !(qua[i*bstride]&SMF__Q_BADB) ) {
/* Pointer to start of power spectrum */
base = pow->pntr[0];
base += nf*i;
/* Smooth the power spectrum */
smf_boxcar1D( base, nf, 1, window, NULL, 0, 1, NULL, status );
/* Measure the power */
if( *status == SAI__OK ) {
p_low = base[i_low];
smf_stats1D( base+i_w1, 1, i_w2-i_w1+1, NULL, 0, 0, &p_white, NULL, NULL,
&ngood, status );
/* It's OK if bad status was generated as long as a mean was calculated */
if( *status==SMF__INSMP ) {
errAnnul( status );
/* if we had no good data there was probably a problem with SMF__Q_BADB
so we simply go to the next bolometer */
if (ngood == 0) continue;
}
/* Count the number of initially good values for the current
bolometer. */
if( (*status==SAI__OK) && qua ) {
ngood = 0;
for( j=0; j<ntslice; j++ ) {
if( qua[i*bstride + j*tstride] == 0 ) ngood++;
}
/* Set bolometer to bad if no power detected, or the number of good
values is too low. */
if( (p_low <= 0) || (p_white <= 0) || (ngood < mingood) ) {
for( j=0; j<ntslice; j++ ) {
qua[i*bstride + j*tstride] |= SMF__Q_BADB;
}
}
}
}
if( (*status==SAI__OK) && (!qua || !(qua[i*bstride]&SMF__Q_BADB)) ) {
/* Power ratio requested */
if ( fratio ) {
fratio[i] = p_low/p_white;
}
/* Store values */
if( whitenoise ) {
/* Integrate the PSD by multiplying the average white noise
level by total number of samples and the frequency spacing:
this calculates the time-domain variance (in 200 Hz SCUBA-2
samples for example) assuming this level holds at all
frequencies. */
whitenoise[i] = p_white * ntslice * df;
/* If NEP set, scale this to variance in a 1-second average by
dividing by the sampling frequency (equivalent to
multiplying by sample length). */
if( nep ) {
whitenoise[i] *= steptime;
}
}
}
}
/* Clean up if the caller does not want to take over the power spectrum */
if( pow ) {
if (fftpow) {
*fftpow = pow;
} else {
smf_close_file( &pow, status );
}
}
}
void smfParallelTime( void *job_data_ptr, int *status ) {
smfArray **array=NULL; /* array of smfArrays that we're working on */
smfTimeChunkData *data=NULL; /* Pointer to job data */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
dim_t nbolo; /* Number of bolos in smfData */
dim_t ndata; /* Size of DATA component in smfData */
dim_t ntslice; /* Number of time slices in smfData */
dim_t nsub; /* Number of subarrays */
double *val=NULL; /* Pointer to DATA component of smfData */
if( *status != SAI__OK ) return;
/* Pointer to the data that this thread will process */
data = job_data_ptr;
/* Check for valid inputs */
if( !data ) {
*status = SAI__ERROR;
errRep( "", "smfParallelTime: job_data_ptr is NULL.", status );
return;
}
if( !(array=data->data) ) {
*status = SAI__ERROR;
errRep( "", "smfParallelTime: data array is NULL.", status );
return;
}
/* Message indicating the thread started */
msgSeti( "C1", data->chunk1);
msgSeti( "C2", data->chunk2);
msgOutif( MSG__DEBUG, "",
"-- parallel time: thread starting on chunks ^C1 -- ^C2",
status );
/* Loop over time chunk. Some chunks may be flagged to skip if
chunk1=nchunks */
for( i=data->chunk1; (i<=data->chunk2)&&(i<data->nchunks)&&(*status==SAI__OK);
i++ ) {
nsub = array[i]->ndat;
/* Initialize filter -- assume same data dimensions for all subarrays */
if( data->type == 2 ) {
filt = smf_create_smfFilter( array[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
}
/* Loop over subarray */
for( j=0; (j<nsub)&&(*status==SAI__OK); j++ ) {
/* Change to bolo-ordered data */
if( data->type == 0 ) {
/* --- Re-order the data --- */
smf_dataOrder( array[i]->sdata[j], 0, status );
}
smf_get_dims( array[i]->sdata[j], NULL, NULL, &nbolo, &ntslice, &ndata,
NULL, NULL, status );
if( data->type == 1 ) {
/* --- Boxcar smooth the data --- */
for( k=0; (*status==SAI__OK)&&(k<nbolo); k++ ) {
val = array[i]->sdata[j]->pntr[0];
val += k*ntslice;
/* Boxcar smooth each bolometer by 500 samples */
smf_boxcar1D( val, ntslice, 500, NULL, 0, status );
}
}
if( data->type == 2 ) {
/* --- FFT filter the data --- */
smf_filter_execute( NULL, array[i]->sdata[j], filt, 0, 0, status );
}
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/* Message indicating the thread started */
msgSeti( "C1", data->chunk1);
msgSeti( "C2", data->chunk2);
msgOutif( MSG__DEBUG, "",
"-- parallel time: thread finished chunks ^C1 -- ^C2",
status );
/* Try setting some status in thread to test merging mechanism */
*status = SMF__INSMP;
errRepf( " ", "Set SMF__INSMP in thread chunks %zu -- %zu", status,
data->chunk1, data->chunk2 );
}
static void smf1_calcmodel_noi( void *job_data_ptr, int *status ) {
/*
* Name:
* smf1_calcmodel_noi
* Purpose:
* Executed in a worker thread to do various calculations for
* smf_calmodel_noi.
* Invocation:
* smf1_calcmodel_noi( void *job_data_ptr, int *status )
* Arguments:
* job_data_ptr = SmfCalcModelNoiData * (Given)
* Data structure describing the job to be performed by the worker
* thread.
* status = int * (Given and Returned)
* Inherited status.
*/
/* Local Variables: */
SmfCalcModelNoiData *pdata;
dim_t ibolo;
dim_t itime;
double *pm;
double *pr;
double mean;
double sum;
dim_t nsum;
size_t ibase;
smf_qual_t *pq;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Get a pointer that can be used for accessing the required items in the
supplied structure. */
pdata = (SmfCalcModelNoiData *) job_data_ptr;
if( pdata->operation == 1 ) {
/* Initialise returned chisquared increment and count of values. */
pdata->chisquared = 0.0;
pdata->nchisq = 0;
/* Loop round all bolos to be processed by this thread, maintaining the
index of the first time slice for the current bolo. */
ibase = pdata->b1*pdata->bstride;
for( ibolo = pdata->b1; ibolo <= pdata->b2; ibolo++ ) {
/* Get a pointer ot the first quality value for the current bolo, and
check that the whole bolometer has not been flagged as bad. */
pq = pdata->qua_data + ibase;
if( !( *pq & SMF__Q_BADB ) ) {
/* Get a pointer to the first residual for the current bolo, and then
loop round all time slices. */
pr = pdata->res_data + ibase;
for( itime = 0; itime < pdata->ntslice; itime++ ) {
/* Get a pointer to the model value (noise) to be used with the current
bolometer sample. Multiple bolometer samples may share the same noise
value, so we caclulate the index into the model_data array separately,
rather than just using "ibase" as for the residual and quality
pointers. */
pm = pdata->model_data + ibolo*pdata->mbstride +
( itime % pdata->mntslice )*pdata->mtstride;
/* If the noise is positive and the bolometer sample is good, increment
the chi-squared value and the number of samples included in the sum. */
if( *pm > 0 && !(*pq & SMF__Q_GOOD) ) {
pdata->chisquared += (*pr)*(*pr) / (*pm);
pdata->nchisq++;
}
/* Move residual and quality pointers on to the next time slice. */
pq += pdata->tstride;
pr += pdata->tstride;
}
}
/* Increment the index of the first value associated with the next
bolometer. */
ibase += pdata->bstride;
}
/* Set the noise to the variance of the neighbouring residuals. */
} else if( pdata->operation == 2 ) {
ibase = pdata->b1*pdata->bstride;
for( ibolo = pdata->b1; ibolo <= pdata->b2; ibolo++ ) {
pq = pdata->qua_data + ibase;
if( !( *pq & SMF__Q_BADB ) ) {
pr = pdata->res_data + ibase;
pm = pdata->model_data + ibase;
smf_boxcar1D( pr, pdata->ntslice, pdata->tstride, pdata->box, pq,
SMF__Q_FIT, 0, pm, status);
}
ibase += pdata->bstride;
}
/* Replace bad values and unity values with the mean noise value in each
bolometer. */
} else if( pdata->operation == 3 ) {
/* Loop round all bolos to be processed by this thread, maintaining the
index of the first time slice for the current bolo. */
ibase = pdata->b1*pdata->bstride;
for( ibolo = pdata->b1; ibolo <= pdata->b2; ibolo++ ) {
/* Get a pointer to the first quality value for the current bolo, and
check that the whole bolometer has not been flagged as bad. */
pq = pdata->qua_data + ibase;
if( !( *pq & SMF__Q_BADB ) ) {
/* This operation is only used in cases where the NOI model has the same
shape and size as the residuals. Get a pointer to the first NOI value
for the current bolometer. */
pm = pdata->model_data + ibase;
/* Loop round all time slices, foring the sums needed to calculate the
mean of the non-bad, non-unity noise values. */
sum = 0.0;
nsum = 0;
for( itime = 0; itime < pdata->ntslice; itime++ ) {
/* If the noise value is usable, increment the sums */
if( *pm > 0 && *pm != VAL__BADD && *pm != 1.0 ) {
sum += *pm;
nsum++;
}
/* Point to the next NOI value. */
pm += pdata->tstride;
}
/* If any bad or unity noise values were found, replace them with the
mean noise value. */
if( nsum < pdata->ntslice && nsum > 0 ) {
mean = sum/nsum;
pm = pdata->model_data + ibase;
for( itime = 0; itime < pdata->ntslice; itime++ ) {
if( *pm < 0 || *pm == VAL__BADD || *pm == 1.0 ) {
*pm = mean;
}
pm += pdata->tstride;
}
}
}
/* Increment the index of the first value associated with the next
bolometer. */
ibase += pdata->bstride;
}
} else {
*status = SAI__ERROR;
errRepf( "", "smf_calcmodel_noi: Illegal operation %d", status, pdata->operation );
}
}
void smf_clean_dksquid( smfData *indata, smf_qual_t mask, size_t window, smfData *model,
int calcdk, int nofit, int replacebad, int *status ) {
dim_t b; /* Bolometer index */
size_t bstride; /* Bolometer index stride */
double corr; /* Linear correlation coefficient */
double *corrbuf=NULL; /* Array of correlation coeffs all bolos this col */
int needDA=0; /* Do we need dksquids from the DA? */
int *dkgood=NULL; /* Flag for non-constant dark squid */
double *dksquid=NULL; /* Buffer for smoothed dark squid */
double *dkav=NULL; /* Buffer for average dark squid */
double firstdk; /* First value in dksquid signal */
double gain; /* Gain parameter from template fit */
double *gainbuf=NULL; /* Array of gains for all bolos in this col */
size_t i; /* Loop counter */
size_t jt1;
size_t jt2;
size_t jf1; /* Starting tslice that should be fit */
size_t jf2; /* Final tslice that should be fit */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
size_t nbad=0; /* Number of new bad bolos due to bad dark squid */
dim_t nbolo; /* Number of bolometers */
dim_t ncol; /* Number of columns */
dim_t ndata; /* Number of data points */
size_t nfit; /* number of samples over good range to fit */
size_t ngood=0; /* number of good dark squids */
dim_t nrow; /* Number of rows */
size_t ntot;
dim_t ntslice; /* Number of time slices */
double offset; /* Offset parameter from template fit */
double *offsetbuf=NULL; /* Array of offsets for all bolos in this col */
int pass; /* two passes over data to get estimate of average */
smf_qual_t *qua=NULL;/* Pointer to quality array */
size_t tstride; /* Time slice index stride */
if (*status != SAI__OK) return;
/* Check for NULL smfData pointer */
if( !indata ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData pointer is NULL", status );
return;
}
/* Decide if we need the DA extension or not */
if( (!model) || (model && calcdk) ) {
needDA = 1;
if( !indata->da) {
/* Check for NULL smfDA */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no smfDA struct in smfData",
status);
return;
} else if( !indata->da->dksquid) {
/* Check for NULL dksquid */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no dksquid array in smfData",
status);
return;
}
/* Assert the correct data order here */
smf_dataOrder( indata->da->dksquid, 1, status );
}
/* Check for 3-d data and get dimensions */
smf_get_dims( indata, &nrow, &ncol, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* Identify the range of data that should be fit using SMF__Q_BOUND */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jf1, &jf2,
status );
} else {
jf1 = 0;
jf2 = ntslice-1;
}
nfit = jf2-jf1+1;
/* Total total range only using SMF__Q_PAD */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jt1, &jt2,
status );
} else {
jt1 = 0;
jt2 = ntslice-1;
}
ntot = jt2-jt1+1;
if( model ) {
/* Check for valid model dimensions if supplied */
if( model->dtype != SMF__DOUBLE ) {
msgSetc("DT", smf_dtype_str(model->dtype, status) );
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Data type ^DT for model not supported.",
status );
return;
}
if( (model->ndims != 2) ||
(model->dims[0] != ntslice+nrow*3) ||
(model->dims[1] != ncol) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": model has incorrect dimensions", status );
return;
}
} else {
/* Otherwise allocate space for local dksquid buffer */
dksquid = astCalloc( ntslice, sizeof(*dksquid) );
}
/* Pointer to quality */
qua = smf_select_qualpntr( indata, 0, status );
/* Two passes: in the first we calculate an average dark squid to use as
a surrogate for columns with dead dark squids. In the second we do the
actual cleaning etc. */
dkgood = astCalloc( ncol, sizeof(*dkgood) );
dkav = astCalloc( ntslice, sizeof(*dkav) );
for( pass=0; (*status==SAI__OK)&&(pass<2); pass++ ) {
/* Loop over columns */
for( i=0; (*status==SAI__OK)&&(i<ncol); i++ ) {
/* Point dksquid, gainbuf, offsetbuf and corrbuf to the right
place in model if supplied. */
if( model ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
gainbuf = dksquid + ntslice;
offsetbuf = gainbuf + nrow;
corrbuf = offsetbuf + nrow;
}
/* First pass is just to copy the dark squid over to the model and replace
dead dark squids with the average */
if( pass==0 ) {
/* Copy dark squids from the DA extension into dksquid */
if( needDA && calcdk && model ) {
double *ptr = indata->da->dksquid->pntr[0];
for( j=0; j<ntslice; j++ ) {
dksquid[j] = ptr[i+ncol*j];
}
}
/* Check for a good dark squid by seeing if it ever changes */
firstdk = VAL__BADD;
for( j=jt1; j<=jt2; j++ ) {
if( dksquid[j] != VAL__BADD ) {
if( firstdk == VAL__BADD ) {
firstdk = dksquid[j];
} else if( dksquid[j] != firstdk ) {
dkgood[i] = 1;
ngood++;
/* Add good squid to average dksquid */
for( k=jt1; k<=jt2; k++ ) {
dkav[k] += dksquid[k];
}
break;
}
}
}
}
/* Second pass actually do the fitting / replace with average dksquid if
dksquid was dead */
if( pass==1 ) {
/* Do some dksquid initialization if requested */
if( (*status==SAI__OK) && needDA && calcdk && model && dkgood[i] ) {
/* Smooth the dark squid template */
smf_boxcar1D( &dksquid[jt1], ntot, window, NULL, 0, status );
}
/* Initialize fit coeffs to VAL__BADD */
if( (*status == SAI__OK) && model ) {
for( j=0; j<nrow; j++ ) {
gainbuf[j] = VAL__BADD;
offsetbuf[j] = VAL__BADD;
corrbuf[j] = VAL__BADD;
}
}
/* Loop over rows, removing the fitted dksquid template. */
for( j=0; (!nofit) && (*status==SAI__OK) && (j<nrow); j++ ) {
/* Calculate bolometer index from row/col counters */
if( SC2STORE__COL_INDEX ) {
b = i*nrow + j;
} else {
b = i + j*ncol;
}
/* If dark squid is bad, flag entire bolo as bad if it isn't already */
if( !dkgood[i] && qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
nbad++;
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
/* Try to fit if we think we have a good dark squid and bolo, and only
the goodrange of data (excluding padding etc.) */
if((!qua && dkgood[i]) ||
(qua && dkgood[i] && !(qua[b*bstride]&SMF__Q_BADB))) {
double *d_d;
int *d_i;
switch( indata->dtype ) {
case SMF__DOUBLE:
d_d = (double *) indata->pntr[0];
smf_templateFit1D( &d_d[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL,
mask, mask, nfit, tstride, &dksquid[jf1], 1,
1, &gain, &offset, &corr, status );
break;
case SMF__INTEGER:
d_i = (int *) indata->pntr[0];
smf_templateFit1I( &d_i[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL, mask,
mask, nfit, tstride, &dksquid[jf1], 1, 1,
&gain, &offset, &corr, status );
break;
default:
msgSetc( "DT", smf_dtype_string( indata, status ));
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": Unsupported data type for dksquid cleaning (^DT)",
status );
}
if( *status == SMF__INSMP || *status == SMF__DIVBZ ) {
int wasinsmp = (*status == SMF__INSMP ? 1 : 0 );
/* Annul SMF__INSMP as it was probably due to a bad bolometer */
errAnnul( status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": ROW,COL (%zu,%zu) %s", status, j, i,
(wasinsmp ? "insufficient good samples" : "division by zero" ));
/* Flag entire bolo as bad if it isn't already */
if( qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
} else {
/* Store gain and offset in model */
if( model ) {
gainbuf[j] = gain;
offsetbuf[j] = offset;
corrbuf[j] = corr;
}
if( msgFlevok( MSG__DEBUG1, status ) ) {
msgSeti( "COL", i );
msgSeti( "ROW", j );
msgSetd( "GAI", gain );
msgSetd( "OFF", offset );
msgSetd( "CORR", corr );
msgOutif( MSG__DEBUG1, "", FUNC_NAME
": ROW,COL (^ROW,^COL) GAIN,OFFSET,CORR "
"(^GAI,^OFF,^CORR)", status );
}
}
}
}
}
}
/* Re-normalize the average dark-squid here at the end of pass 0 */
if( (pass==0) && (ngood) && (*status==SAI__OK) ) {
for( j=jt1; j<=jt2; j++ ) {
dkav[j] /= ngood;
}
}
/* Replace bad bolos in model with the average? */
if( replacebad && calcdk && needDA && model && (*status==SAI__OK) ) {
for( i=0; i<ncol; i++ ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
if( !dkgood[i] ) {
memcpy( dksquid, dkav, sizeof(*dksquid)*ntslice );
dkgood[i] = 1;
}
}
}
}
/* Report number of new bad bolos that were flagged */
if( !replacebad && nbad ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": %zu new bolos flagged bad due to dead DKS", status, nbad );
}
/* Free dksquid only if it was a local buffer */
if( !model && dksquid ) dksquid = astFree( dksquid );
dkgood = astFree( dkgood );
dkav = astFree( dkav );
}
