void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_fillgaps( ThrWorkForce *wf, smfData *data,
smf_qual_t mask, int *status ) {
/* Local Variables */
const gsl_rng_type *type; /* GSL random number generator type */
dim_t bpt; /* Number of bolos per thread */
dim_t i; /* Bolometer index */
dim_t nbolo; /* Number of bolos */
dim_t ntslice; /* Number of time slices */
double *dat=NULL; /* Pointer to bolo data */
int fillpad; /* Fill PAD samples? */
size_t bstride; /* bolo stride */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
size_t tstride; /* time slice stride */
smfFillGapsData *job_data; /* Structures holding data for worker threads */
smfFillGapsData *pdata; /* Pointer to data for next worker thread */
smf_qual_t *qua=NULL; /* Pointer to quality array */
/* Main routine */
if (*status != SAI__OK) return;
/* Check we have double precision data floating point data. */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
/* Pointers to data and quality */
dat = data->pntr[0];
qua = smf_select_qualpntr( data, NULL, status );
if( !qua ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No valid QUALITY array was provided", status );
return;
}
if( !dat ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not contain a DATA component",status);
return;
}
/* obtain data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride, &tstride,
status );
/* Determine how many bolometers to process in each thread, and create
the structures used to pass data to the threads. */
if( wf ) {
bpt = nbolo/wf->nworker;
if( wf->nworker*bpt < nbolo ) bpt++;
job_data = astMalloc( sizeof( smfFillGapsData )*wf->nworker );
} else {
bpt = nbolo;
job_data = astMalloc( sizeof( smfFillGapsData ) );
}
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_PAD, &pstart, &pend,
status );
/* Report an error if it is too short. */
if( pend - pstart <= 2*BOX && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": length of data (%d samples) is too small "
"to fill gaps. Must have at least %d samples per bolometer.",
status, (int) ( pend - pstart ), 2*BOX + 1 );
}
/* If the supplied "mask" value includes SMF__Q_PAD, then we will be
replacing the zero-padded region at the start and end of each time series
with artificial noisey data that connects the first and last data values
smoothly. Remove SMF__Q_PAD from the mask. */
if( mask & SMF__Q_PAD ) {
mask &= ~SMF__Q_PAD;
fillpad = 1;
} else {
fillpad = 0;
}
/* Get the default GSL randim number generator type. A separate random
number generator is used for each worker thread so that the gap filling
process does not depend on the the order in which threads are
executed. */
type = gsl_rng_default;
/* Begin a job context. */
thrBeginJobContext( wf, status );
/* Loop over bolometer in groups of "bpt". */
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
/* Store information for this group in the next smfFillGapsData
structure. */
pdata->ntslice = ntslice;
pdata->dat = dat;
pdata->r = gsl_rng_alloc( type );
pdata->b1 = i;
pdata->b2 = i + bpt - 1;
pdata->pend = pend;
pdata->fillpad = fillpad;
pdata->pstart = pstart;
if( pdata->b2 >= nbolo ) pdata->b2 = nbolo - 1;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->qua = qua;
pdata->mask = mask;
/* Submit a job to the workforce to process this group of bolometers. */
(void) thrAddJob( wf, 0, pdata, smfFillGapsParallel, 0, NULL, status );
}
/* Wait until all jobs in the current job context have completed, and
then end the job context. */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free resources. */
if( job_data ) {
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
if( pdata->r ) gsl_rng_free( pdata->r );
}
job_data = astFree( job_data );
}
}
void smf_clean_dksquid( smfData *indata, smf_qual_t mask, size_t window, smfData *model,
int calcdk, int nofit, int replacebad, int *status ) {
dim_t b; /* Bolometer index */
size_t bstride; /* Bolometer index stride */
double corr; /* Linear correlation coefficient */
double *corrbuf=NULL; /* Array of correlation coeffs all bolos this col */
int needDA=0; /* Do we need dksquids from the DA? */
int *dkgood=NULL; /* Flag for non-constant dark squid */
double *dksquid=NULL; /* Buffer for smoothed dark squid */
double *dkav=NULL; /* Buffer for average dark squid */
double firstdk; /* First value in dksquid signal */
double gain; /* Gain parameter from template fit */
double *gainbuf=NULL; /* Array of gains for all bolos in this col */
size_t i; /* Loop counter */
size_t jt1;
size_t jt2;
size_t jf1; /* Starting tslice that should be fit */
size_t jf2; /* Final tslice that should be fit */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
size_t nbad=0; /* Number of new bad bolos due to bad dark squid */
dim_t nbolo; /* Number of bolometers */
dim_t ncol; /* Number of columns */
dim_t ndata; /* Number of data points */
size_t nfit; /* number of samples over good range to fit */
size_t ngood=0; /* number of good dark squids */
dim_t nrow; /* Number of rows */
size_t ntot;
dim_t ntslice; /* Number of time slices */
double offset; /* Offset parameter from template fit */
double *offsetbuf=NULL; /* Array of offsets for all bolos in this col */
int pass; /* two passes over data to get estimate of average */
smf_qual_t *qua=NULL;/* Pointer to quality array */
size_t tstride; /* Time slice index stride */
if (*status != SAI__OK) return;
/* Check for NULL smfData pointer */
if( !indata ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData pointer is NULL", status );
return;
}
/* Decide if we need the DA extension or not */
if( (!model) || (model && calcdk) ) {
needDA = 1;
if( !indata->da) {
/* Check for NULL smfDA */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no smfDA struct in smfData",
status);
return;
} else if( !indata->da->dksquid) {
/* Check for NULL dksquid */
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, no dksquid array in smfData",
status);
return;
}
/* Assert the correct data order here */
smf_dataOrder( indata->da->dksquid, 1, status );
}
/* Check for 3-d data and get dimensions */
smf_get_dims( indata, &nrow, &ncol, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
/* Identify the range of data that should be fit using SMF__Q_BOUND */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jf1, &jf2,
status );
} else {
jf1 = 0;
jf2 = ntslice-1;
}
nfit = jf2-jf1+1;
/* Total total range only using SMF__Q_PAD */
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_BOUND, &jt1, &jt2,
status );
} else {
jt1 = 0;
jt2 = ntslice-1;
}
ntot = jt2-jt1+1;
if( model ) {
/* Check for valid model dimensions if supplied */
if( model->dtype != SMF__DOUBLE ) {
msgSetc("DT", smf_dtype_str(model->dtype, status) );
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Data type ^DT for model not supported.",
status );
return;
}
if( (model->ndims != 2) ||
(model->dims[0] != ntslice+nrow*3) ||
(model->dims[1] != ncol) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": model has incorrect dimensions", status );
return;
}
} else {
/* Otherwise allocate space for local dksquid buffer */
dksquid = astCalloc( ntslice, sizeof(*dksquid) );
}
/* Pointer to quality */
qua = smf_select_qualpntr( indata, 0, status );
/* Two passes: in the first we calculate an average dark squid to use as
a surrogate for columns with dead dark squids. In the second we do the
actual cleaning etc. */
dkgood = astCalloc( ncol, sizeof(*dkgood) );
dkav = astCalloc( ntslice, sizeof(*dkav) );
for( pass=0; (*status==SAI__OK)&&(pass<2); pass++ ) {
/* Loop over columns */
for( i=0; (*status==SAI__OK)&&(i<ncol); i++ ) {
/* Point dksquid, gainbuf, offsetbuf and corrbuf to the right
place in model if supplied. */
if( model ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
gainbuf = dksquid + ntslice;
offsetbuf = gainbuf + nrow;
corrbuf = offsetbuf + nrow;
}
/* First pass is just to copy the dark squid over to the model and replace
dead dark squids with the average */
if( pass==0 ) {
/* Copy dark squids from the DA extension into dksquid */
if( needDA && calcdk && model ) {
double *ptr = indata->da->dksquid->pntr[0];
for( j=0; j<ntslice; j++ ) {
dksquid[j] = ptr[i+ncol*j];
}
}
/* Check for a good dark squid by seeing if it ever changes */
firstdk = VAL__BADD;
for( j=jt1; j<=jt2; j++ ) {
if( dksquid[j] != VAL__BADD ) {
if( firstdk == VAL__BADD ) {
firstdk = dksquid[j];
} else if( dksquid[j] != firstdk ) {
dkgood[i] = 1;
ngood++;
/* Add good squid to average dksquid */
for( k=jt1; k<=jt2; k++ ) {
dkav[k] += dksquid[k];
}
break;
}
}
}
}
/* Second pass actually do the fitting / replace with average dksquid if
dksquid was dead */
if( pass==1 ) {
/* Do some dksquid initialization if requested */
if( (*status==SAI__OK) && needDA && calcdk && model && dkgood[i] ) {
/* Smooth the dark squid template */
smf_boxcar1D( &dksquid[jt1], ntot, window, NULL, 0, status );
}
/* Initialize fit coeffs to VAL__BADD */
if( (*status == SAI__OK) && model ) {
for( j=0; j<nrow; j++ ) {
gainbuf[j] = VAL__BADD;
offsetbuf[j] = VAL__BADD;
corrbuf[j] = VAL__BADD;
}
}
/* Loop over rows, removing the fitted dksquid template. */
for( j=0; (!nofit) && (*status==SAI__OK) && (j<nrow); j++ ) {
/* Calculate bolometer index from row/col counters */
if( SC2STORE__COL_INDEX ) {
b = i*nrow + j;
} else {
b = i + j*ncol;
}
/* If dark squid is bad, flag entire bolo as bad if it isn't already */
if( !dkgood[i] && qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
nbad++;
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
/* Try to fit if we think we have a good dark squid and bolo, and only
the goodrange of data (excluding padding etc.) */
if((!qua && dkgood[i]) ||
(qua && dkgood[i] && !(qua[b*bstride]&SMF__Q_BADB))) {
double *d_d;
int *d_i;
switch( indata->dtype ) {
case SMF__DOUBLE:
d_d = (double *) indata->pntr[0];
smf_templateFit1D( &d_d[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL,
mask, mask, nfit, tstride, &dksquid[jf1], 1,
1, &gain, &offset, &corr, status );
break;
case SMF__INTEGER:
d_i = (int *) indata->pntr[0];
smf_templateFit1I( &d_i[b*bstride+jf1*tstride],
&qua[b*bstride+jf1*tstride], NULL, NULL, mask,
mask, nfit, tstride, &dksquid[jf1], 1, 1,
&gain, &offset, &corr, status );
break;
default:
msgSetc( "DT", smf_dtype_string( indata, status ));
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": Unsupported data type for dksquid cleaning (^DT)",
status );
}
if( *status == SMF__INSMP || *status == SMF__DIVBZ ) {
int wasinsmp = (*status == SMF__INSMP ? 1 : 0 );
/* Annul SMF__INSMP as it was probably due to a bad bolometer */
errAnnul( status );
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": ROW,COL (%zu,%zu) %s", status, j, i,
(wasinsmp ? "insufficient good samples" : "division by zero" ));
/* Flag entire bolo as bad if it isn't already */
if( qua && !(qua[b*bstride]&SMF__Q_BADB) ) {
for( k=0; k<ntslice; k++ ) {
qua[b*bstride+k*tstride] |= SMF__Q_BADB;
}
}
} else {
/* Store gain and offset in model */
if( model ) {
gainbuf[j] = gain;
offsetbuf[j] = offset;
corrbuf[j] = corr;
}
if( msgFlevok( MSG__DEBUG1, status ) ) {
msgSeti( "COL", i );
msgSeti( "ROW", j );
msgSetd( "GAI", gain );
msgSetd( "OFF", offset );
msgSetd( "CORR", corr );
msgOutif( MSG__DEBUG1, "", FUNC_NAME
": ROW,COL (^ROW,^COL) GAIN,OFFSET,CORR "
"(^GAI,^OFF,^CORR)", status );
}
}
}
}
}
}
/* Re-normalize the average dark-squid here at the end of pass 0 */
if( (pass==0) && (ngood) && (*status==SAI__OK) ) {
for( j=jt1; j<=jt2; j++ ) {
dkav[j] /= ngood;
}
}
/* Replace bad bolos in model with the average? */
if( replacebad && calcdk && needDA && model && (*status==SAI__OK) ) {
for( i=0; i<ncol; i++ ) {
dksquid = model->pntr[0];
dksquid += i*(ntslice+nrow*3);
if( !dkgood[i] ) {
memcpy( dksquid, dkav, sizeof(*dksquid)*ntslice );
dkgood[i] = 1;
}
}
}
}
/* Report number of new bad bolos that were flagged */
if( !replacebad && nbad ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": %zu new bolos flagged bad due to dead DKS", status, nbad );
}
/* Free dksquid only if it was a local buffer */
if( !model && dksquid ) dksquid = astFree( dksquid );
dkgood = astFree( dkgood );
dkav = astFree( dkav );
}
void smf_write_shortmap( ThrWorkForce *wf, int shortmap, smfArray *res,
smfArray *lut, smfArray *qua, smfDIMMData *dat,
dim_t msize, const Grp *shortrootgrp, size_t contchunk,
int varmapmethod, const int *lbnd_out,
const int *ubnd_out, AstFrameSet *outfset,
int *status ) {
dim_t dsize; /* Size of data arrays in containers */
size_t i; /* loop counter */
size_t idx=0; /* index within subgroup */
size_t istart; /* First useful timeslice */
size_t iend; /* Last useful timeslice */
int *lut_data=NULL; /* Pointer to DATA component of lut */
char name[GRP__SZNAM+1]; /* Buffer for storing names */
size_t nshort=0; /* Number of short maps */
dim_t ntslice; /* Number of time slices */
char *pname=NULL; /* Poiner to name */
smf_qual_t *qua_data=NULL; /* Pointer to DATA component of qua */
double *res_data=NULL; /* Pointer to DATA component of res */
size_t sc; /* Short map counter */
double *shortmapweight=NULL; /* buffer for shotmap weights */
double *shortmapweightsq=NULL;/* buffer for shotmap weights squared */
int *shorthitsmap=NULL; /* buffer for shotmap hits */
size_t shortstart; /* first time slice of short map */
size_t shortend; /* last time slice of short map */
size_t tstride; /* Time stride */
if( *status != SAI__OK ) return;
if( !res || !lut || !qua || !dat || !shortrootgrp ||
!lbnd_out || !ubnd_out || !outfset || !shortmap ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL inputs supplied", status );
return;
}
if( !res || !res->sdata || !res->sdata[idx] || !res->sdata[idx]->hdr ||
!res->sdata[idx]->hdr->allState ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": RES does not contain JCMTState", status );
return;
}
/* Allocate space for the arrays */
shortmapweight = astMalloc( msize*sizeof(*shortmapweight) );
shortmapweightsq = astMalloc( msize*sizeof(*shortmapweightsq) );
shorthitsmap = astMalloc( msize*sizeof(*shorthitsmap) );
/* Use first subarray to figure out time dimension. Get the
useful start and end points of the time series, and then
determine "nshort" -- the number of complete blocks of
shortmap time slices in the useful range. */
smf_get_dims( qua->sdata[0], NULL, NULL, NULL, &ntslice,
NULL, NULL, &tstride, status );
qua_data = (qua->sdata[0]->pntr)[0];
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_BOUND,
&istart, &iend, status );
shortstart = istart;
if( *status == SAI__OK ) {
if( shortmap == -1 ) {
nshort = res->sdata[idx]->hdr->allState[iend].tcs_index -
res->sdata[idx]->hdr->allState[istart].tcs_index + 1;
msgOutf( "", FUNC_NAME
": writing %zu short maps, once each time TCS_INDEX increments",
status, nshort );
} else {
nshort = (iend-istart+1)/shortmap;
if( nshort ) {
msgOutf( "", FUNC_NAME
": writing %zu short maps of length %i time slices.",
status, nshort, shortmap );
} else {
/* Generate warning message if requested short maps are too long*/
msgOutf( "", FUNC_NAME
": Warning! short maps of lengths %i requested, but "
"data only %zu time slices.", status, shortmap,
iend-istart+1 );
}
}
}
/* Loop over short maps */
for( sc=0; (sc<nshort)&&(*status==SAI__OK); sc++ ) {
Grp *mgrp=NULL; /* Temporary group for map names */
smfData *mapdata=NULL; /* smfData for new map */
char tempstr[20]; /* Temporary string */
char tmpname[GRP__SZNAM+1]; /* temp name buffer */
char thisshort[20]; /* name particular to this shortmap */
/* Create a name for the new map, take into account the
chunk number. Only required if we are using a single
output container. */
pname = tmpname;
grpGet( shortrootgrp, 1, 1, &pname, sizeof(tmpname), status );
one_strlcpy( name, tmpname, sizeof(name), status );
one_strlcat( name, ".", sizeof(name), status );
/* Continuous chunk number */
sprintf(tempstr, "CH%02zd", contchunk);
one_strlcat( name, tempstr, sizeof(name), status );
/* Shortmap number */
sprintf( thisshort, "SH%06zu", sc );
one_strlcat( name, thisshort, sizeof(name), status );
mgrp = grpNew( "shortmap", status );
grpPut1( mgrp, name, 0, status );
msgOutf( "", "*** Writing short map (%zu / %zu) %s", status,
sc+1, nshort, name );
smf_open_newfile ( wf, mgrp, 1, SMF__DOUBLE, 2, lbnd_out,
ubnd_out, SMF__MAP_VAR, &mapdata,
status);
/* Time slice indices for start and end of short map -- common to
all subarrays */
if( shortmap > 0) {
/* Evenly-spaced shortmaps in time */
shortstart = istart+sc*shortmap;
shortend = istart+(sc+1)*shortmap-1;
} else {
/* One map each time TCS_INDEX increments -- just uses header
for the first subarray */
for(i=shortstart+1; (i<=iend) &&
(res->sdata[0]->hdr->allState[i].tcs_index ==
res->sdata[0]->hdr->allState[shortstart].tcs_index);
i++ );
shortend = i-1;
}
/* Bad status if we have invalid shortmap ranges. This might
happen if there is ever a jump in TCS_INDEX for the shortmap=-1
case since the total number of shortmaps is calculated simply
as the difference between the first and final TCS indices. */
if( !nshort || (iend<istart) || (iend>=ntslice) ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": invalid shortmap range (%zu--%zu, ntslice=%zu)"
"encountered", status, istart, iend, ntslice );
break;
}
/* Loop over subgroup index (subarray) */
for( idx=0; (idx<res->ndat)&&(*status==SAI__OK); idx++ ) {
int rebinflag = 0;
/* Pointers to everything we need */
res_data = (res->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
smf_get_dims( res->sdata[idx], NULL, NULL, NULL, &ntslice,
&dsize, NULL, &tstride, status );
/* Rebin the data for this range of tslices. */
if( idx == 0 ) {
rebinflag |= AST__REBININIT;
}
if( idx == (res->ndat-1) ) {
rebinflag |= AST__REBINEND;
}
smf_rebinmap1( NULL, res->sdata[idx],
dat->noi ? dat->noi[0]->sdata[idx] : NULL,
lut_data, shortstart, shortend, 1, NULL, 0,
SMF__Q_GOOD, varmapmethod,
rebinflag,
mapdata->pntr[0],
shortmapweight, shortmapweightsq, shorthitsmap,
mapdata->pntr[1], msize, NULL, status );
/* Write out FITS header */
if( (*status == SAI__OK) && res->sdata[idx]->hdr &&
res->sdata[idx]->hdr->allState ) {
AstFitsChan *fitschan=NULL;
JCMTState *allState = res->sdata[idx]->hdr->allState;
size_t midpnt = (shortstart + shortend) / 2;
fitschan = astFitsChan ( NULL, NULL, " " );
atlPtfti( fitschan, "SEQSTART", allState[shortstart].rts_num,
"RTS index number of first frame", status );
atlPtfti( fitschan, "SEQEND", allState[shortend].rts_num,
"RTS index number of last frame", status);
atlPtftd( fitschan, "MJD-AVG", allState[midpnt].rts_end,
"Average MJD of this map", status );
atlPtfts( fitschan, "TIMESYS", "TAI", "Time system for MJD-AVG",
status );
atlPtfti( fitschan, "TCSINDST", allState[shortstart].tcs_index,
"TCS index of first frame", status );
atlPtfti( fitschan, "TCSINDEN", allState[shortend].tcs_index,
"TCS index of last frame", status );
kpgPtfts( mapdata->file->ndfid, fitschan, status );
if( fitschan ) fitschan = astAnnul( fitschan );
}
}
/* Update shortstart in case we are counting steps in TCS_INDEX */
shortstart = shortend+1;
/* Write WCS */
smf_set_moving( (AstFrame *) outfset, NULL, status );
ndfPtwcs( outfset, mapdata->file->ndfid, status );
/* Clean up */
if( mgrp ) grpDelet( &mgrp, status );
smf_close_file( wf, &mapdata, status );
}
/* Free up memory */
shortmapweight = astFree( shortmapweight );
shortmapweightsq = astFree( shortmapweightsq );
shorthitsmap = astFree( shorthitsmap );
}
void smf_filter_execute( ThrWorkForce *wf, smfData *data, smfFilter *filt,
int complement, int whiten, int *status ) {
/* Local Variables */
size_t apod_length=0; /* apodization length */
fftw_iodim dims; /* I/O dimensions for transformations */
size_t first; /* First sample apodization at start */
int i; /* Loop counter */
smfFilterExecuteData *job_data=NULL;/* Array of job data for each thread */
size_t last; /* Last sample apodization at end */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int nw; /* Number of worker threads */
dim_t ntslice=0; /* Number of time slices */
smf_qual_t *qua=NULL; /* Pointer to quality flags */
smfFilterExecuteData *pdata=NULL; /* Pointer to current job data */
size_t step; /* step size for dividing up work */
/* Main routine */
if (*status != SAI__OK) return;
/* Check for NULL pointers */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfData pointer", status );
return;
}
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter pointer", status );
return;
}
if( filt->ndims == 2 ) {
smf_filter2d_execute( wf, data, filt, complement, status );
return;
}
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Ensure that the smfData is ordered correctly (bolo ordered) */
smf_dataOrder( wf, data, 0, status );
/* Obtain the dimensions of the array being filtered */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, NULL, NULL, status);
if( *status != SAI__OK ) return;
/* Using complement of the filter? */
if( complement ) smf_filter_complement( filt, status );
/* Pointers to quality */
qua = smf_select_qualpntr( data, NULL, status );
/* Determine the first and last samples to apodize (after padding), if
any. Assumed to be the same for all bolometers. */
if( qua ) {
smf_get_goodrange( qua, ntslice, 1, SMF__Q_PAD, &first, &last, status );
} else {
first = 0;
last = ntslice - 1;
}
/* Can we apodize? */
apod_length = filt->apod_length;
if( *status == SAI__OK ) {
if( apod_length == SMF__MAXAPLEN ) {
apod_length = (last-first+1)/2;
msgOutiff( MSG__DEBUG, "", FUNC_NAME
": Using maximum apodization length, %zu samples.",
status, apod_length );
} else if( (last-first+1) < (2*apod_length) && apod_length != SMF__BADSZT ){
*status = SAI__ERROR;
errRepf("", FUNC_NAME
": Can't apodize, not enough samples (%zu < %zu).", status,
last-first+1, 2*apod_length);
}
}
/* If apodising is switched off, fill gaps in the data and re-create
the artifical data used for padding based on the current contents of
the smfData. */
if( apod_length == SMF__BADSZT ) {
smf_fillgaps( wf, data, SMF__Q_PAD | SMF__Q_GAP, status );
/* If apodising is switched on, fill the data (retaining the zero padding)
and apodise the data. */
} else {
smf_fillgaps( wf, data, SMF__Q_GAP, status );
if( apod_length > 0 ) smf_apodize( data, apod_length, 1, status );
}
/* Describe the input and output array dimensions for FFTW guru interface.
- dims describes the length and stepsize of time slices within a bolometer
*/
dims.n = ntslice;
dims.is = 1;
dims.os = 1;
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->data = data;
pdata->qua = qua;
pdata->data_fft_r = astMalloc(filt->fdims[0]*sizeof(*pdata->data_fft_r));
pdata->data_fft_i = astMalloc(filt->fdims[0]*sizeof(*pdata->data_fft_i));
pdata->filt = filt;
pdata->whiten = whiten;
pdata->complement = complement;
pdata->ijob = -1; /* Flag job as ready to start */
/* Setup forward FFT plan using guru interface. Requires protection
with a mutex */
thrMutexLock( &smf_filter_execute_mutex, status );
if( *status == SAI__OK ) {
/* Just use the data_fft_* arrays from the first chunk of job data since
the guru interface allows you to use the same plans for multiple
transforms. */
pdata->plan_forward = fftw_plan_guru_split_dft_r2c( 1, &dims, 0, NULL,
data->pntr[0],
pdata->data_fft_r,
pdata->data_fft_i,
FFTW_ESTIMATE |
FFTW_UNALIGNED );
}
thrMutexUnlock( &smf_filter_execute_mutex, status );
if( !pdata->plan_forward && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": FFTW3 could not create plan for forward transformation",
status);
}
/* Setup inverse FFT plan using guru interface */
thrMutexLock( &smf_filter_execute_mutex, status );
if( *status == SAI__OK ) {
pdata->plan_inverse = fftw_plan_guru_split_dft_c2r( 1, &dims, 0, NULL,
pdata->data_fft_r,
pdata->data_fft_i,
data->pntr[0],
FFTW_ESTIMATE |
FFTW_UNALIGNED);
}
thrMutexUnlock( &smf_filter_execute_mutex, status );
if( !pdata->plan_inverse && (*status==SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": FFTW3 could not create plan for inverse transformation",
status);
}
}
/* Execute the filter */
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFilterExecuteParallel, 0,
NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Clean up the job data array */
if( job_data ) {
for( i=0; i<nw; i++ ) {
pdata = job_data + i;
if( pdata->data_fft_r ) pdata->data_fft_r = astFree( pdata->data_fft_r );
if( pdata->data_fft_i ) pdata->data_fft_i = astFree( pdata->data_fft_i );
/* Destroy the plans */
thrMutexLock( &smf_filter_execute_mutex, status );
fftw_destroy_plan( pdata->plan_forward );
fftw_destroy_plan( pdata->plan_inverse );
thrMutexUnlock( &smf_filter_execute_mutex, status );
}
job_data = astFree( job_data );
}
/* Return the filter to its original state if required */
if( complement == -1 ) smf_filter_complement( filt, status );
/* Remove the effects of the apodisation from the filtered data. */
if( apod_length != SMF__BADSZT && apod_length > 0 ) {
smf_apodize( data, apod_length, 0, status );
}
}
size_t smf_clean_pca( ThrWorkForce *wf, smfData *data, size_t t_first,
size_t t_last, double thresh, size_t ncomp,
smfData **components, smfData **amplitudes,
int flagbad, int sub, AstKeyMap *keymap,
smf_qual_t mask, int *status ){
double *amp=NULL; /* matrix of components amplitudes for each bolo */
size_t abstride; /* bolo stride in amp array */
size_t acompstride; /* component stride in amp array */
size_t bstride; /* bolo stride */
double *comp=NULL; /* data cube of components */
size_t ccompstride; /* component stride in comp array */
size_t ctstride; /* time stride in comp array */
gsl_matrix *cov=NULL; /* bolo-bolo covariance matrix */
size_t i; /* Loop counter */
int ii; /* Loop counter */
size_t j; /* Loop counter */
smfPCAData *job_data=NULL;/* job data */
size_t k; /* Loop counter */
size_t *goodbolo=NULL; /* Indices of the good bolometers for analysis */
dim_t nbolo; /* number of bolos */
dim_t ndata; /* number of samples in data */
size_t ngoodbolo; /* number good bolos = number principal components */
dim_t ntslice; /* number of time slices */
int nw; /* total available worker threads */
smfPCAData *pdata=NULL; /* Pointer to job data */
smf_qual_t *qua=NULL; /* Pointer to quality array */
gsl_vector *s=NULL; /* singular values for SVD */
size_t bstep; /* Bolo step size for job division */
size_t step; /* step size for job division */
size_t tlen; /* Length of the time-series used for PCA */
size_t tstride; /* time slice stride */
gsl_vector *work=NULL; /* workspace for SVD */
if (*status != SAI__OK) return 0;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check for NULL smfData pointer */
if( !data || !data->pntr[0]) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, NULL data supplied", status );
return 0;
}
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( data->ndims != 3 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be 3-dimensional",
status );
return 0;
}
if( data->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": possible programming error, smfData should be double precision",
status );
return 0;
}
if( ntslice <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 time slices!", status );
goto CLEANUP;
}
/* If the range of time slices has not been specified, us the total
range excluding padding and apodizing. */
qua = smf_select_qualpntr( data, 0, status );
if( !t_last ) {
if( qua ) {
smf_get_goodrange( qua, ntslice, tstride, (SMF__Q_PAD | SMF__Q_APOD),
&t_first, &t_last, status );
} else {
t_last = ntslice-1;
}
}
if( t_last > (ntslice-1) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last is set past the last time slice!",
status );
goto CLEANUP;
}
if( (t_last < t_first) || ( (t_last - t_first) < 1 ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": t_last - t_first must be > 1", status );
goto CLEANUP;
}
tlen = t_last - t_first + 1;
if( flagbad && (tlen != ntslice ) ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME
": flagbad unsupported if t_first/last do not span full data",
status );
goto CLEANUP;
}
if( qua ) {
/* If quality supplied, identify good bolometers */
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
ngoodbolo++;
}
}
/* Now remember which were the good bolometers */
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
ngoodbolo = 0;
for( i=0; i<nbolo; i++ ) {
if( !(qua[i*bstride]&SMF__Q_BADB) ) {
goodbolo[ngoodbolo] = i;
ngoodbolo++;
}
}
} else {
/* Otherwise assume all bolometers are good */
ngoodbolo = nbolo;
goodbolo = astCalloc( ngoodbolo, sizeof(*goodbolo) );
for( i=0; i<ngoodbolo; i++ ) {
goodbolo[i] = i;
}
}
if( ngoodbolo <= 2 ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": fewer than 2 working bolometers!", status );
goto CLEANUP;
}
/* Fill bad values and values flagged via "mask" (except entirely bad
bolometers) with interpolated data values. */
mask &= ~SMF__Q_BADB;
smf_fillgaps( wf, data, mask, status );
/* Allocate arrays */
amp = astCalloc( nbolo*ngoodbolo, sizeof(*amp) );
comp = astCalloc( ngoodbolo*tlen, sizeof(*comp) );
cov = gsl_matrix_alloc( ngoodbolo, ngoodbolo );
s = gsl_vector_alloc( ngoodbolo );
work = gsl_vector_alloc( ngoodbolo );
/* These strides will make comp time-ordered */
ccompstride = 1;
ctstride = ngoodbolo;
/* These strides will also make amp look time-ordered (sort-of: the time
axis is now the component number */
abstride = 1;
acompstride = nbolo;
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
/* Set up the division of labour for threads: independent blocks of time */
if( nw > (int) tlen ) {
step = 1;
} else {
step = tlen/nw;
}
if( nw > (int) ngoodbolo ) {
bstep = 1;
} else {
bstep = ngoodbolo/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step + t_first;
pdata->t2 = (ii+1)*step + t_first - 1;
/* Blocks of bolometers. */
pdata->b1 = ii*bstep;
pdata->b2 = (ii+1)*bstep - 1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) ) {
pdata->t2 = t_first + tlen - 1;
pdata->b2 = ngoodbolo - 1;
}
/* initialize work data */
pdata->amp = NULL;
pdata->abstride = abstride;
pdata->acompstride = acompstride;
pdata->bstride = bstride;
pdata->comp = comp;
pdata->cov = NULL;
pdata->covwork = NULL;
pdata->ccompstride = ccompstride;
pdata->ctstride = ctstride;
pdata->data = data;
pdata->goodbolo = NULL;
pdata->ijob = -1;
pdata->nbolo = nbolo;
pdata->ngoodbolo = ngoodbolo;
pdata->t_first = t_first;
pdata->t_last = t_last;
pdata->tlen = tlen;
pdata->operation = 0;
pdata->tstride = tstride;
/* Each thread will accumulate the projection of its own portion of
the time-series. We'll add them to the master amp at the end */
pdata->amp = astCalloc( nbolo*ngoodbolo, sizeof(*(pdata->amp)) );
/* Each thread will accumulate sums of x, y, and x*y for each bolo when
calculating the covariance matrix */
pdata->covwork = astCalloc( ngoodbolo*ngoodbolo,
sizeof(*(pdata->covwork)) );
/* each thread gets its own copy of the goodbolo lookup table */
pdata->goodbolo = astCalloc( ngoodbolo, sizeof(*(pdata->goodbolo)) );
if( *status == SAI__OK ) {
memcpy( pdata->goodbolo, goodbolo,
ngoodbolo*sizeof(*(pdata->goodbolo)) );
}
}
if( *status == SAI__OK ) {
/* Remove the mean from each gap-filled bolometer time stream ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME ": removing bolometer means...",
status );
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = -1;
thrAddJob( wf, 0, pdata, smfPCAParallel, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Measure the covariance matrix using parallel code ---------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": measuring bolo-bolo covariance matrix...", status );
/* Set up the jobs to calculate sums for each time block and submit */
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 0;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* We now have to add together all of the sums from each thread and
normalize */
if( *status == SAI__OK ) {
for( i=0; i<ngoodbolo; i++ ) {
for( j=i; j<ngoodbolo; j++ ) {
double c;
double *covwork=NULL;
double sum_xy;
sum_xy = 0;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
covwork = pdata->covwork;
sum_xy += covwork[ i + j*ngoodbolo ];
}
c = sum_xy / ((double)tlen-1);
gsl_matrix_set( cov, i, j, c );
gsl_matrix_set( cov, j, i, c );
}
}
}
}
/* Factor cov = u s v^T, noting that the SVD routine calculates v^T in
in-place of cov. --------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": perfoming singular value decomposition...", status );
smf_svd( wf, ngoodbolo, cov->data, s->data, NULL, 10*VAL__EPSD,
1, status );
if( CHECK ) {
double check=0;
for( i=0; i<ngoodbolo; i++ ) {
for( j=0; j<ngoodbolo; j++ ) {
check += gsl_matrix_get( cov, j, i );
}
}
printf("--- check inverted: %lf\n", check);
}
/* Calculate normalized eigenvectors with parallel code --------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating statistically-independent components...", status );
/* The above calculation tells us what linear combinations of the original
bolometer time series will give us the statistically independent new
set of basis vectors (components), which we then normalize by their RMS. */
/* Set up the jobs to calculate sums for each time block and submit */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->cov = cov;
pdata->operation = 1;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Then normalize. Some of the components may have zero amplitude and
so cannot be used (i.e. we are trying to use more components than
there is evidence for in the data). So we check for zero sigma. In
fact, we check for silly small sigma, not just zero sigma. Any
component for which the sigma is less than 1E-10 of the log-mean
sigma is excluded. */
{
double *sigmas = astMalloc( ngoodbolo*sizeof( *sigmas ) );
double check = 0;
double s1 = 0.0;
int s2 = 0;
int nlow = 0;
for( i=0; (*status==SAI__OK)&&(i<ngoodbolo); i++ ) {
double sigma;
smf_stats1D( comp + i*ccompstride, ctstride, tlen, NULL, 0,
0, NULL, &sigma, NULL, NULL, status );
/* Apparently we need this to get the normalization right */
sigma *= sqrt((double) tlen);
if( *status == SAI__OK ) {
if( sigma > 0.0 ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] /= sigma;
sigmas[ i ] = sigma;
s1 += log10( sigma );
s2++;
}
} else {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
sigmas[ i ] = VAL__BADD;
}
nlow++;
}
}
}
/* Exclude any components that have a silly small standard deviation
(less that 1E-10 of the logmean of all components). Any with zero
standard deviation will already have been excluded. */
if( s2 > 0 ) {
double logmean = s1/s2;
for( i=0; i<ngoodbolo; i++ ) {
if( sigmas[ i ] != VAL__BADD && sigmas[ i ] < 1E-10*logmean ) {
for( k=0; k<tlen; k++ ) {
comp[i*ccompstride + k*ctstride] = VAL__BADD;
nlow++;
}
}
}
}
msgOutiff( MSG__DEBUG, "", FUNC_NAME ": rejecting %d (out of %zu) components"
" because they are too weak to normalise", status, nlow, ngoodbolo );
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
sigmas = astFree( sigmas );
//printf("--- check component: %lf\n", check);
}
/* Now project the data along each of these normalized basis vectors
to figure out the amplitudes of the components in each bolometer
time series. ------------------------------------------------------------*/
msgOutif( MSG__VERB, "", FUNC_NAME
": calculating component amplitudes in each bolo...", status );
/* Set up the jobs */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
pdata->operation = 2;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfPCAParallel,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Add all of the amp arrays together from the threads */
if( *status == SAI__OK ) {
size_t index;
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
for( i=0; i<ngoodbolo; i++ ) { /* Loop over good bolo */
for( j=0; j<ngoodbolo; j++ ) { /* Loop over component */
index = goodbolo[i]*abstride + j*acompstride;
amp[index] += pdata->amp[index];
}
}
}
}
if( CHECK ){
double check=0;
for( i=0; i<nbolo*ngoodbolo; i++ ) {
check += amp[i];
}
printf("--- check combined amp: %lf\n", check);
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component A: %lf\n", check);
}
/* Check to see if the amplitudes are mostly negative or positive. If
mostly negative, flip the sign of both the component and amplitudes */
if( *status == SAI__OK ) {
double total;
for( j=0; j<ngoodbolo; j++ ) { /* loop over component */
total = 0;
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
total += amp[goodbolo[i]*abstride + j*acompstride];
}
/* Are most amplitudes negative for this component? */
if( total < 0 ) {
/* Flip sign of the amplitude */
for( i=0; i<ngoodbolo; i++ ) { /* loop over bolometer */
amp[goodbolo[i]*abstride + j*acompstride] =
-amp[goodbolo[i]*abstride + j*acompstride];
}
/* Flip sign of the component */
for( k=0; k<tlen; k++ ) {
if( comp[j*ccompstride + k*ctstride] != VAL__BADD ) {
comp[j*ccompstride + k*ctstride] *= -1;
}
}
}
}
}
/* Finally, copy the master amp array back into the workspace for
each thread */
if( *status == SAI__OK ) {
for( ii=0; ii<nw; ii++ ) {
pdata = job_data + ii;
memcpy( pdata->amp, amp, sizeof(*(pdata->amp))*nbolo*ngoodbolo );
}
}
if( CHECK ){
double check=0;
for( i=0; i<ngoodbolo*tlen; i++ ) {
if( comp[i] != VAL__BADD ) check += comp[i];
}
printf("--- check component B: %lf\n", check);
}
/* Flag outlier bolometers if requested ------------------------------------*/
if( (*status==SAI__OK) && flagbad ) {
smfArray *data_array=NULL;
smfArray *gain_array=NULL;
smfGroup *gain_group=NULL;
AstKeyMap *kmap=NULL; /* Local keymap */
AstObject *obj=NULL; /* Used to avoid compiler warnings */
double *template=NULL;
