void smurf_sc2fft( int *status ) {
int avpspec=0; /* Flag for doing average power spectrum */
double avpspecthresh=0; /* Threshold noise for detectors in avpspec */
Grp * basegrp = NULL; /* Basis group for output filenames */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* dark frames */
int ensureflat; /* Flag for flatfielding data */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t gcount=0; /* Grp index counter */
size_t i; /* Loop counter */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
Grp *igrp = NULL; /* Input group of files */
int inverse=0; /* If set perform inverse transform */
int isfft=0; /* Are data fft or real space? */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Total number of NDF names in the output group */
int polar=0; /* Flag for FFT in polar coordinates */
int power=0; /* Flag for squaring amplitude coeffs */
size_t size; /* Number of files in input group */
smfData *tempdata=NULL; /* Temporary smfData pointer */
int weightavpspec=0; /* Flag for 1/noise^2 weighting */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
/* We now need to combine files from the same subarray and same sequence
to form a continuous time series */
smf_grp_related( igrp, size, 1, 0, 0, NULL, NULL, &maxconcat, NULL, &igroup,
&basegrp, NULL, status );
/* Get output file(s) */
size = grpGrpsz( basegrp, status );
if( size > 0 ) {
kpg1Wgndf( "OUT", basegrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ", TASK_NAME ": All supplied input frames were DARK,"
" nothing to do", status );
}
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
msgOutiff( MSG__NORM, "", "Found %zu continuous chunk%s", status, ncontchunks,
(ncontchunks > 1 ? "s" : "") );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Are we doing an inverse transform? */
parGet0l( "INVERSE", &inverse, status );
/* Are we using polar coordinates instead of cartesian for the FFT? */
parGet0l( "POLAR", &polar, status );
/* Are we going to assume amplitudes are squared? */
parGet0l( "POWER", &power, status );
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* Are we calculating the average power spectrum? */
parGet0l( "AVPSPEC", &avpspec, status );
if( avpspec ) {
power = 1;
parGet0d( "AVPSPECTHRESH", &avpspecthresh, status );
parGet0l( "WEIGHTAVPSPEC", &weightavpspec, status );
}
/* If power is true, we must be in polar form */
if( power && !polar) {
msgOutif( MSG__NORM, " ", TASK_NAME
": power spectrum requested so setting POLAR=TRUE", status );
polar = 1;
}
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
size_t idx;
/* Concatenate this continuous chunk but forcing a raw data read.
We will need quality. */
smf_concat_smfGroup( wf, NULL, igroup, darks, NULL, flatramps, heateffmap,
contchunk, ensureflat, 1, NULL, 0, NULL, NULL, 0, 0, 0,
&concat, NULL, status );
/* Now loop over each subarray */
/* Export concatenated data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
if( concat->sdata[idx] ) {
smfData * idata = concat->sdata[idx];
int provid = NDF__NOID;
dim_t nbolo; /* Number of detectors */
dim_t ndata; /* Number of data points */
/* Apply a mask to the quality array and data array */
smf_apply_mask( idata, bbms, SMF__BBM_QUAL|SMF__BBM_DATA, 0, status );
smf_get_dims( idata, NULL, NULL, &nbolo, NULL, &ndata, NULL, NULL,
status );
/* Check for double precision data */
if( idata->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": data are not double precision.", status );
}
/* Are we zeroing VAL__BADD? */
if( (*status==SAI__OK) && zerobad ) {
double *data= (double *) idata->pntr[0];
for( i=0; i<ndata; i++ ) {
if( data[i] == VAL__BADD ) {
data[i] = 0;
}
}
}
/* Check whether we need to transform the data at all */
isfft = smf_isfft(idata,NULL,NULL,NULL,NULL,NULL,status);
if( isfft && avpspec && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": to calculate average power spectrum input data cannot "
"be FFT", status );
}
if( (*status == SAI__OK) && (isfft == inverse) ) {
if( avpspec ) {
/* If calculating average power spectrum do the transforms with
smf_bolonoise so that we can also measure the noise of
each detector */
double *whitenoise=NULL;
smf_qual_t *bolomask=NULL;
double mean, sig, freqlo;
size_t ngood, newgood;
whitenoise = astCalloc( nbolo, sizeof(*whitenoise) );
bolomask = astCalloc( nbolo, sizeof(*bolomask) );
freqlo = 1. / (idata->hdr->steptime * idata->hdr->nframes);
smf_bolonoise( wf, idata, 1, freqlo, SMF__F_WHITELO,
SMF__F_WHITEHI, 1, 0, whitenoise, NULL, &odata,
status );
/* Initialize quality */
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] == VAL__BADD ) {
bolomask[i] = SMF__Q_BADB;
} else {
/* smf_bolonoise returns a variance, so take sqrt */
whitenoise[i] = sqrt(whitenoise[i]);
}
}
ngood=-1;
newgood=0;
/* Iteratively cut n-sigma noisy outlier detectors */
while( ngood != newgood ) {
ngood = newgood;
smf_stats1D( whitenoise, 1, nbolo, bolomask, 1, SMF__Q_BADB,
&mean, &sig, NULL, NULL, status );
msgOutiff( MSG__DEBUG, "", TASK_NAME
": mean=%lf sig=%lf ngood=%li\n", status,
mean, sig, ngood);
newgood=0;
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] != VAL__BADD ){
if( (whitenoise[i] - mean) > avpspecthresh *sig ) {
whitenoise[i] = VAL__BADD;
bolomask[i] = SMF__Q_BADB;
} else {
newgood++;
}
}
}
}
msgOutf( "", TASK_NAME
": Calculating average power spectrum of best %li "
" bolometers.", status, newgood);
/* If using 1/noise^2 weights, calculate 1/whitenoise^2 in-place
to avoid allocating another array */
if( weightavpspec ) {
msgOutif( MSG__VERB, "", TASK_NAME ": using 1/noise^2 weights",
status );
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] && (whitenoise[i] != VAL__BADD) ) {
whitenoise[i] = 1/(whitenoise[i]*whitenoise[i]);
}
}
}
/* Calculate the average power spectrum of good detectors */
tempdata = smf_fft_avpspec( odata, bolomask, 1, SMF__Q_BADB,
weightavpspec ? whitenoise : NULL,
status );
smf_close_file( &odata, status );
whitenoise = astFree( whitenoise );
bolomask = astFree( bolomask );
odata = tempdata;
tempdata = NULL;
/* Store the number of good bolometers */
parPut0i( "NGOOD", newgood, status );
} else {
/* Otherwise do forward/inverse transforms here as needed */
/* If inverse transform convert to cartesian representation first */
if( inverse && polar ) {
smf_fft_cart2pol( wf, idata, 1, power, status );
}
/* Tranform the data */
odata = smf_fft_data( wf, idata, NULL, inverse, 0, status );
smf_convert_bad( wf, odata, status );
if( inverse ) {
/* If output is time-domain, ensure that it is ICD bolo-ordered */
smf_dataOrder( odata, 1, status );
} else if( polar ) {
/* Store FFT of data in polar form */
smf_fft_cart2pol( wf, odata, 0, power, status );
}
}
/* open a reference input file for provenance propagation */
ndgNdfas( basegrp, gcount, "READ", &provid, status );
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, gcount, provid,
MSG__VERB, 0, status );
/* Free resources */
ndfAnnul( &provid, status );
smf_close_file( &odata, status );
} else {
msgOutif( MSG__NORM, " ",
"Data are already transformed. No output will be produced",
status );
}
}
/* Update index into group */
gcount++;
}
/* Close the smfArray */
smf_close_related( &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if (basegrp) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if (bbms) smf_close_related( &bbms, status );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_fit_pspec( const double *pspec, dim_t nf, size_t box, double df,
double minfreq, double whitefreq, double maxfreq,
double *a, double *b, double *w, int *status ) {
double A=VAL__BADD; /* Amplitude of 1/f component */
double B=VAL__BADD; /* Exponent of 1/f component */
int converged; /* Has white noise level calc converged? */
double fit[2]; /* fit coefficients */
size_t i; /* Loop counter */
size_t i_flo; /* Index of lowest frequency for calculating 1/f */
size_t i_whi; /* Index of high-freq. edge for white-noise level */
size_t i_wlo; /* Index of low-freq. edge for white-noise level */
size_t nbad; /* Number of outliers in white noise calc */
smf_qual_t *qual=NULL; /* Quality to go with pspec */
double sigma; /* Uncertainty in white noise level */
size_t thisnbad; /* Outliers in white noise calc, this iteration */
double white=VAL__BADD;/* White noise level */
double *x=NULL; /* Independent axis for 1/f fit */
double *y=NULL; /* Dependent axis for 1/f fit */
if (*status != SAI__OK) return;
if( (df<=0) || (nf<1) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": invalid frequency information supplied.", status );
return;
}
/* Convert frequencies to array indices (but only consider above DC) */
i_flo = smf_get_findex( minfreq, df, nf, status );
i_wlo = smf_get_findex( whitefreq, df, nf, status );
if( !maxfreq ) {
/* If maxfreq is 0 set to Nyquist */
i_whi = nf-1;
} else {
i_whi = smf_get_findex( maxfreq, df, nf, status );
}
if( i_flo+box*2 > nf ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": box too large compared to number of frequencies",
status );
return;
}
/* Create a local quality array to go with pspec */
qual = astCalloc( nf, sizeof(*qual) );
if( i_flo == 0 ) i_flo = 1;
if( (*status==SAI__OK) && ((i_flo > i_wlo) || (i_wlo > i_whi)) ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": must have i_maxfreq (%zu) > i_whitefreq (%zu) > "
"i_minfreq (%zu)", status, i_whi, i_wlo, i_flo );
}
/* Calculate the white noise level */
converged = 0;
nbad = 0;
while( (!converged) && (*status==SAI__OK) ) {
double thresh;
smf_stats1D( pspec+i_wlo, 1, i_whi-i_wlo+1, qual, 1, SMF__Q_SPIKE, &white,
&sigma, NULL, NULL, status );
if( *status==SAI__OK ) {
thresh = white + SMF__FPSPEC_WHITESNR*sigma;
thisnbad = 0;
for( i=i_wlo; i<=i_whi; i++ ) {
if( pspec[i] > thresh ) {
qual[i] = SMF__Q_SPIKE;
thisnbad++;
}
}
if( thisnbad==nbad ) {
converged = 1;
}
nbad = thisnbad;
}
}
/* Now identify and fit the 1/f part of the power spectrum. We use a
rolling box and increase the frequency of its centre until the fraction
of samples below some threshold above the white level is exceeded.
We then fit a straight line to the logarithm of the x- and y-axes. */
if( *status == SAI__OK ) {
size_t nfit;
size_t ngood = 0;
size_t nused;
double thresh = white + SMF__FPSPEC_KNEESNR*sigma;
/* Initialize ngood -- skip the DC term in the FFT */
for( i=i_flo; i<(i_flo+box); i++ ) {
if( pspec[i] > thresh ) {
ngood++;
}
}
/* Continuing from next element, and go until we hit the end or we reach
the threshold number of samples below the threshold SNR above
the white noise level. */
for( i=i_flo+1; (i<(nf-(box-1))) && (ngood >= SMF__FPSPEC_KNEETHRESH*box);
i++ ) {
/* If the previous first element was good remove it */
if( pspec[i-1] >= thresh ) {
ngood--;
}
/* If the new last element is good add it */
if( pspec[i+box-1] >= thresh ) {
ngood++;
}
}
/* We will fit the power-law from elements i_flo to nfit-1. We then
evaluate the fitted power law as
y = A * x^B
where x is the index in the frequency array
A = exp( fit[0] )
B = fit[1] */
nfit = i+box/2-i_flo;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": i_flow=%zu nfit=%zu i_wlo=%zu i_whi=%zu\n", status,
i_flo, nfit-1, i_wlo, i_whi);
/* If we've entered the white-noise band in order to fit the 1/f
noise give up */
if( i >= i_wlo ) {
*status = SMF__BADFIT;
errRep( "", FUNC_NAME
": unable to fit 1/f spectrum with provided frequency ranges",
status);
goto CLEANUP;
}
/* Now fit a straight line to the log of the two axes */
x = astMalloc( (nfit-1)*sizeof(*x) );
y = astMalloc( (nfit-1)*sizeof(*y) );
if( *status == SAI__OK ) {
for( i=0; i<(nfit-1); i++ ) {
x[i] = log((i+i_flo)*df);
y[i] = log(pspec[i+i_flo]);
}
}
smf_fit_poly1d( 1, nfit-1, 0, x, y, NULL, NULL, fit, NULL, NULL, &nused,
status );
if( *status == SAI__OK ) {
/* if the exponent is positive the fit is either garbage, or
else there just isn't very much low frequency noise compared
to the white noise level. Set B to 0 but generate a warning
message */
if( fit[1] >= 0 ) {
*status = SMF__BADFIT;
errRep( "", FUNC_NAME
": fit to 1/f component encountered rising spectrum!",
status);
goto CLEANUP;
} else {
B = fit[1];
}
A = (exp(fit[0]));
}
}
/* Return fit values */
if( a ) *a = A;
if( b ) *b = B;
if( w ) *w = white;
CLEANUP:
qual = astFree( qual );
x = astFree( x );
y = astFree( y );
}
void smf_bolonoise( ThrWorkForce *wf, smfData *data, double gfrac,
size_t window, double f_low,
double f_white1, double f_white2,
int nep, size_t len, double *whitenoise, double *fratio,
smfData **fftpow,int *status ) {
double *base=NULL; /* Pointer to base coordinates of array */
size_t bstride; /* bolometer index stride */
double df=1; /* Frequency step size in Hz */
size_t i; /* Loop counter */
size_t i_low; /* Index in power spectrum to f_low */
size_t i_w1; /* Index in power spectrum to f_white1 */
size_t i_w2; /* Index in power spectrum to f_white2 */
size_t j; /* Loop counter */
size_t mingood; /* Min. required no. of good values in bolometer */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t nf=0; /* Number of frequencies */
size_t ngood; /* Number of good samples */
dim_t ntslice; /* Number of time slices */
double p_low; /* Power at f_low */
double p_white; /* Average power from f_white1 to f_white2 */
smfData *pow=NULL; /* Pointer to power spectrum data */
smf_qual_t *qua=NULL; /* Pointer to quality component */
double steptime=1; /* Length of a sample in seconds */
size_t tstride; /* time index stride */
if (*status != SAI__OK) return;
/* Check inputs */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
if( !data->hdr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData has no header", status );
return;
}
/* Obtain dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( *status==SAI__OK ) {
steptime = data->hdr->steptime;
if( steptime < VAL__SMLD ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": FITS header error, STEPTIME must be > 0",
status);
} else {
/* Frequency steps in the FFT */
df = 1. / (steptime * (double) ntslice );
}
}
/* Initialize arrays */
if( whitenoise ) for(i=0; i<nbolo; i++) whitenoise[i] = VAL__BADD;
if( fratio ) for(i=0; i<nbolo; i++) fratio[i] = VAL__BADD;
/* FFT the data and convert to polar power spectral density form */
pow = smf_fft_data( wf, data, NULL, 0, len, status );
smf_convert_bad( wf, pow, status );
smf_fft_cart2pol( wf, pow, 0, 1, status );
{
dim_t fdims[2];
smf_isfft( pow, NULL, NULL, fdims, NULL, NULL, status );
if( *status == SAI__OK ) nf=fdims[0];
}
/* Check for reasonble frequencies, and integer offsets in the array */
i_low = smf_get_findex( f_low, df, nf, status );
i_w1 = smf_get_findex( f_white1, df, nf, status );
i_w2 = smf_get_findex( f_white2, df, nf, status );
/* Get the quality pointer from the smfData so that we can mask known
bad bolometer. */
qua = smf_select_qualpntr( data, NULL, status );
/* The minimum required number of good values in a bolometer. */
mingood = ( gfrac > 0.0 ) ? ntslice*gfrac : 0;
/* Loop over detectors */
for( i=0; (*status==SAI__OK)&&(i<nbolo); i++ )
if( !qua || !(qua[i*bstride]&SMF__Q_BADB) ) {
/* Pointer to start of power spectrum */
base = pow->pntr[0];
base += nf*i;
/* Smooth the power spectrum */
smf_boxcar1D( base, nf, 1, window, NULL, 0, 1, NULL, status );
/* Measure the power */
if( *status == SAI__OK ) {
p_low = base[i_low];
smf_stats1D( base+i_w1, 1, i_w2-i_w1+1, NULL, 0, 0, &p_white, NULL, NULL,
&ngood, status );
/* It's OK if bad status was generated as long as a mean was calculated */
if( *status==SMF__INSMP ) {
errAnnul( status );
/* if we had no good data there was probably a problem with SMF__Q_BADB
so we simply go to the next bolometer */
if (ngood == 0) continue;
}
/* Count the number of initially good values for the current
bolometer. */
if( (*status==SAI__OK) && qua ) {
ngood = 0;
for( j=0; j<ntslice; j++ ) {
if( qua[i*bstride + j*tstride] == 0 ) ngood++;
}
/* Set bolometer to bad if no power detected, or the number of good
values is too low. */
if( (p_low <= 0) || (p_white <= 0) || (ngood < mingood) ) {
for( j=0; j<ntslice; j++ ) {
qua[i*bstride + j*tstride] |= SMF__Q_BADB;
}
}
}
}
if( (*status==SAI__OK) && (!qua || !(qua[i*bstride]&SMF__Q_BADB)) ) {
/* Power ratio requested */
if ( fratio ) {
fratio[i] = p_low/p_white;
}
/* Store values */
if( whitenoise ) {
/* Integrate the PSD by multiplying the average white noise
level by total number of samples and the frequency spacing:
this calculates the time-domain variance (in 200 Hz SCUBA-2
samples for example) assuming this level holds at all
frequencies. */
whitenoise[i] = p_white * ntslice * df;
/* If NEP set, scale this to variance in a 1-second average by
dividing by the sampling frequency (equivalent to
multiplying by sample length). */
if( nep ) {
whitenoise[i] *= steptime;
}
}
}
}
/* Clean up if the caller does not want to take over the power spectrum */
if( pow ) {
if (fftpow) {
*fftpow = pow;
} else {
smf_close_file( &pow, status );
}
}
}
size_t smf_clean_pca( ThrWorkForce *wf, smfData *data, size_t t_first,
size_t t_last, double thresh, size_t ncomp,
smfData **components, smfData **amplitudes,
int flagbad, int sub, AstKeyMap *keymap,
smf_qual_t mask, int *status ){
double *amp=NULL; /* matrix of components amplitudes for each bolo */
size_t abstride; /* bolo stride in amp array */
size_t acompstride; /* component stride in amp array */
size_t bstride; /* bolo stride */
double *comp=NULL; /* data cube of components */
size_t ccompstride; /* component stride in comp array */
size_t ctstride; /* time stride in comp array */
gsl_matrix *cov=NULL; /* bolo-bolo covariance matrix */
size_t i; /* Loop counter */
int ii; /* Loop counter */
size_t j; /* Loop counter */
smfPCAData *job_data=NULL;/* job data */
size_t k; /* Loop counter */
size_t *goodbolo=NULL; /* Indices of the good bolometers for analysis */
dim_t nbolo; /* number of bolos */
dim_t ndata; /* number of samples in data */
size_t ngoodbolo; /* number good bolos = number principal components */
dim_t ntslice; /* number of time slices */
int nw; /* total available worker threads */
smfPCAData *pdata=NULL; /* Pointer to job data */
smf_qual_t *qua=NULL; /* Pointer to quality array */
gsl_vector *s=NULL; /* singular values for SVD */
size_t bstep; /* Bolo step size for job division */
size_t step; /* step size for job division */
size_t tlen; /* Length of the time-series used for PCA */
size_t tstride; /* time slice stride */
gsl_vector *work=NULL; /* workspace for SVD */
if (*status != SAI__OK) return 0;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check for NULL smfData pointer */
if( !data || !data->pntr[0]) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, NULL data supplied", status );
return 0;
}
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( data->ndims != 3 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be 3-dimensional",
status );
return 0;
}
if( data->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be double precision",
status );
return 0;
}
if( ntslice <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 time slices!", status );
goto CLEANUP;
}
/* If the range of time slices has not been specified, us the total
range excluding padding and apodizing. */
qua = smf_select_qualpntr( data, 0, status );
if( !t_last ) {
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, (SMF__Q_PAD | SMF__Q_APOD),
&t_first, &t_last, status );
} else {
t_last = ntslice-1;
}
}
if( t_last > (ntslice-1) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last is set past the last time slice!",
status );
goto CLEANUP;
}
if( (t_last < t_first) || ( (t_last - t_first) < 1 ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last - t_first must be > 1", status );
goto CLEANUP;
}
tlen = t_last - t_first + 1;
if( flagbad && (tlen != ntslice ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": flagbad unsupported if t_first/last do not span full data",
status );
goto CLEANUP;
}
if( qua ) {
/* If quality supplied, identify good bolometers */
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
ngoodbolo++;
}
}
/* Now remember which were the good bolometers */
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
goodbolo[ngoodbolo] = i;
ngoodbolo++;
}
}
} else {
/* Otherwise assume all bolometers are good */
ngoodbolo = nbolo;
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
for( i=0; i<ngoodbolo; i++ ) {
goodbolo[i] = i;
}
}
if( ngoodbolo <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 working bolometers!", status );
goto CLEANUP;
}
/* Fill bad values and values flagged via "mask" (except entirely bad
bolometers) with interpolated data values. */
mask &= ~SMF__Q_BADB;
smf_fillgaps( wf, data, mask, status );
/* Allocate arrays */
amp = astCalloc( nbolo*ngoodbolo, sizeof(*amp) );
comp = astCalloc( ngoodbolo*tlen, sizeof(*comp) );
cov = gsl_matrix_alloc( ngoodbolo, ngoodbolo );
s = gsl_vector_alloc( ngoodbolo );
work = gsl_vector_alloc( ngoodbolo );
/* These strides will make comp time-ordered */
ccompstride = 1;
ctstride = ngoodbolo;
/* These strides will also make amp look time-ordered (sort-of: the time
axis is now the component number */
abstride = 1;
acompstride = nbolo;
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
/* Set up the division of labour for threads: independent blocks of time */
if( nw > (int) tlen ) {
step = 1;
} else {
step = tlen/nw;
}
if( nw > (int) ngoodbolo ) {
bstep = 1;
} else {
bstep = ngoodbolo/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step + t_first;
pdata->t2 = (ii+1)*step + t_first - 1;
/* Blocks of bolometers. */
pdata->b1 = ii*bstep;
pdata->b2 = (ii+1)*bstep - 1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) ) {
pdata->t2 = t_first + tlen - 1;
pdata->b2 = ngoodbolo - 1;
}
/* initialize work data */
pdata->amp = NULL;
pdata->abstride = abstride;
pdata->acompstride = acompstride;
pdata->bstride = bstride;
pdata->comp = comp;
pdata->cov = NULL;
pdata->covwork = NULL;
pdata->ccompstride = ccompstride;
pdata->ctstride = ctstride;
pdata->data = data;
pdata->goodbolo = NULL;
pdata->ijob = -1;
pdata->nbolo = nbolo;
pdata->ngoodbolo = ngoodbolo;
pdata->t_first = t_first;
pdata->t_last = t_last;
pdata->tlen = tlen;
pdata->operation = 0;
pdata->tstride = tstride;
/* Each thread will accumulate the projection of its own portion of
the time-series. We'll add them to the master amp at the end */
pdata->amp = astCalloc( nbolo*ngoodbolo, sizeof(*(pdata->amp)) );
/* Each thread will accumulate sums of x, y, and x*y for each bolo when
calculating the covariance matrix */
pdata->covwork = astCalloc( ngoodbolo*ngoodbolo,
sizeof(*(pdata->covwork)) );
/* each thread gets its own copy of the goodbolo lookup table */
pdata->goodbolo = astCalloc( ngoodbolo, sizeof(*(pdata->goodbolo)) );
if( *status == SAI__OK ) {
memcpy( pdata->goodbolo, goodbolo,
ngoodbolo*sizeof(*(pdata->goodbolo)) );
}
}
if( *status == SAI__OK ) {
/* Remove the mean from each gap-filled bolometer time stream ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME ": removing bolometer means...",
status );
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = -1;
thrAddJob( wf, 0, pdata, smfPCAParallel, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Measure the covariance matrix using parallel code ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": measuring bolo-bolo covariance matrix...", status );
/* Set up the jobs to calculate sums for each time block and submit */
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 0;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* We now have to add together all of the sums from each thread and
normalize */
if( *status == SAI__OK ) {
for( i=0; i<ngoodbolo; i++ ) {
for( j=i; j<ngoodbolo; j++ ) {
double c;
double *covwork=NULL;
double sum_xy;
sum_xy = 0;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
covwork = pdata->covwork;
sum_xy += covwork[ i + j*ngoodbolo ];
}
c = sum_xy / ((double)tlen-1);
gsl_matrix_set( cov, i, j, c );
gsl_matrix_set( cov, j, i, c );
}
}
}
}
/* Factor cov = u s v^T, noting that the SVD routine calculates v^T in
in-place of cov. --------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": perfoming singular value decomposition...", status );
smf_svd( wf, ngoodbolo, cov->data, s->data, NULL, 10*VAL__EPSD,
1, status );
if( CHECK ) {
double check=0;
for( i=0; i<ngoodbolo; i++ ) {
for( j=0; j<ngoodbolo; j++ ) {
check += gsl_matrix_get( cov, j, i );
}
}
printf("--- check inverted: %lf\n", check);
}
/* Calculate normalized eigenvectors with parallel code --------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating statistically-independent components...", status );
/* The above calculation tells us what linear combinations of the original
bolometer time series will give us the statistically independent new
set of basis vectors (components), which we then normalize by their RMS. */
/* Set up the jobs to calculate sums for each time block and submit */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->cov = cov;
pdata->operation = 1;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Then normalize. Some of the components may have zero amplitude and
so cannot be used (i.e. we are trying to use more components than
there is evidence for in the data). So we check for zero sigma. In
fact, we check for silly small sigma, not just zero sigma. Any
component for which the sigma is less than 1E-10 of the log-mean
sigma is excluded. */
{
double *sigmas = astMalloc( ngoodbolo*sizeof( *sigmas ) );
double check = 0;
double s1 = 0.0;
int s2 = 0;
int nlow = 0;
for( i=0; (*status==SAI__OK)&&(i<ngoodbolo); i++ ) {
double sigma;
smf_stats1D( comp + i*ccompstride, ctstride, tlen, NULL, 0,
0, NULL, &sigma, NULL, NULL, status );
/* Apparently we need this to get the normalization right */
sigma *= sqrt((double) tlen);
if( *status == SAI__OK ) {
if( sigma > 0.0 ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] /= sigma;
sigmas[ i ] = sigma;
s1 += log10( sigma );
s2++;
}
} else {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
sigmas[ i ] = VAL__BADD;
}
nlow++;
}
}
}
/* Exclude any components that have a silly small standard deviation
(less that 1E-10 of the logmean of all components). Any with zero
standard deviation will already have been excluded. */
if( s2 > 0 ) {
double logmean = s1/s2;
for( i=0; i<ngoodbolo; i++ ) {
if( sigmas[ i ] != VAL__BADD && sigmas[ i ] < 1E-10*logmean ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
nlow++;
}
}
}
}
msgOutiff( MSG__DEBUG, "", FUNC_NAME ": rejecting %d (out of %zu) components"
" because they are too weak to normalise", status, nlow, ngoodbolo );
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
sigmas = astFree( sigmas );
//printf("--- check component: %lf\n", check);
}
/* Now project the data along each of these normalized basis vectors
to figure out the amplitudes of the components in each bolometer
time series. ------------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating component amplitudes in each bolo...", status );
/* Set up the jobs */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 2;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Add all of the amp arrays together from the threads */
if( *status == SAI__OK ) {
size_t index;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
for( i=0; i<ngoodbolo; i++ ) { /* Loop over good bolo */
for( j=0; j<ngoodbolo; j++ ) { /* Loop over component */
index = goodbolo[i]*abstride + j*acompstride;
amp[index] += pdata->amp[index];
}
}
}
}
if( CHECK ){
double check=0;
for( i=0; i<nbolo*ngoodbolo; i++ ) {
check += amp[i];
}
printf("--- check combined amp: %lf\n", check);
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component A: %lf\n", check);
}
/* Check to see if the amplitudes are mostly negative or positive. If
mostly negative, flip the sign of both the component and amplitudes */
if( *status == SAI__OK ) {
double total;
for( j=0; j<ngoodbolo; j++ ) { /* loop over component */
total = 0;
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
total += amp[goodbolo[i]*abstride + j*acompstride];
}
/* Are most amplitudes negative for this component? */
if( total < 0 ) {
/* Flip sign of the amplitude */
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
amp[goodbolo[i]*abstride + j*acompstride] =
-amp[goodbolo[i]*abstride + j*acompstride];
}
/* Flip sign of the component */
for( k=0; k<tlen; k++ ) {
if( comp[j*ccompstride + k*ctstride] != VAL__BADD ) {
comp[j*ccompstride + k*ctstride] *= -1;
}
}
}
}
}
/* Finally, copy the master amp array back into the workspace for
each thread */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
memcpy( pdata->amp, amp, sizeof(*(pdata->amp))*nbolo*ngoodbolo );
}
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component B: %lf\n", check);
}
/* Flag outlier bolometers if requested ------------------------------------*/
if( (*status==SAI__OK) && flagbad ) {
smfArray *data_array=NULL;
smfArray *gain_array=NULL;
smfGroup *gain_group=NULL;
AstKeyMap *kmap=NULL; /* Local keymap */
AstObject *obj=NULL; /* Used to avoid compiler warnings */
double *template=NULL;