static void
smf__fill_smfData ( smfData * data, double dval, int *status ) {
dim_t ndata;
if (*status != SAI__OK) return;
if (!data) return;
/* And fill the flatfield with bad values */
if ( data->ndims == 1) {
ndata = data->dims[0];
} else {
smf_get_dims( data, NULL, NULL, NULL, NULL, &ndata,
NULL, NULL, status );
}
if (*status == SAI__OK) {
double *dpntr = (data->pntr)[0];
size_t i;
for (i=0; i<ndata; i++) {
dpntr[i] = dval;
}
}
}
void smfParallelTime( void *job_data_ptr, int *status ) {
smfArray **array=NULL; /* array of smfArrays that we're working on */
smfTimeChunkData *data=NULL; /* Pointer to job data */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
dim_t nbolo; /* Number of bolos in smfData */
dim_t ndata; /* Size of DATA component in smfData */
dim_t ntslice; /* Number of time slices in smfData */
dim_t nsub; /* Number of subarrays */
double *val=NULL; /* Pointer to DATA component of smfData */
if( *status != SAI__OK ) return;
/* Pointer to the data that this thread will process */
data = job_data_ptr;
/* Check for valid inputs */
if( !data ) {
*status = SAI__ERROR;
errRep( "", "smfParallelTime: job_data_ptr is NULL.", status );
return;
}
if( !(array=data->data) ) {
*status = SAI__ERROR;
errRep( "", "smfParallelTime: data array is NULL.", status );
return;
}
/* Message indicating the thread started */
msgSeti( "C1", data->chunk1);
msgSeti( "C2", data->chunk2);
msgOutif( MSG__DEBUG, "",
"-- parallel time: thread starting on chunks ^C1 -- ^C2",
status );
/* Loop over time chunk. Some chunks may be flagged to skip if
chunk1=nchunks */
for( i=data->chunk1; (i<=data->chunk2)&&(i<data->nchunks)&&(*status==SAI__OK);
i++ ) {
nsub = array[i]->ndat;
/* Initialize filter -- assume same data dimensions for all subarrays */
if( data->type == 2 ) {
filt = smf_create_smfFilter( array[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
}
/* Loop over subarray */
for( j=0; (j<nsub)&&(*status==SAI__OK); j++ ) {
/* Change to bolo-ordered data */
if( data->type == 0 ) {
/* --- Re-order the data --- */
smf_dataOrder( array[i]->sdata[j], 0, status );
}
smf_get_dims( array[i]->sdata[j], NULL, NULL, &nbolo, &ntslice, &ndata,
NULL, NULL, status );
if( data->type == 1 ) {
/* --- Boxcar smooth the data --- */
for( k=0; (*status==SAI__OK)&&(k<nbolo); k++ ) {
val = array[i]->sdata[j]->pntr[0];
val += k*ntslice;
/* Boxcar smooth each bolometer by 500 samples */
smf_boxcar1D( val, ntslice, 500, NULL, 0, status );
}
}
if( data->type == 2 ) {
/* --- FFT filter the data --- */
smf_filter_execute( NULL, array[i]->sdata[j], filt, 0, 0, status );
}
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/* Message indicating the thread started */
msgSeti( "C1", data->chunk1);
msgSeti( "C2", data->chunk2);
msgOutif( MSG__DEBUG, "",
"-- parallel time: thread finished chunks ^C1 -- ^C2",
status );
/* Try setting some status in thread to test merging mechanism */
*status = SMF__INSMP;
errRepf( " ", "Set SMF__INSMP in thread chunks %zu -- %zu", status,
data->chunk1, data->chunk2 );
}
void smf_mask_noisy( ThrWorkForce *wf, smfData *data, smfData **noise,
double sigcliphigh, double sigcliplow, int cliplog,
int zeropad, int * status ) {
size_t i;
dim_t nbolo = 0; /* Number of bolometers */
double *noisedata = NULL; /* Pointer to the noise data array */
smfData * noisemap = NULL; /* Somewhere to receive the result */
double *work = NULL; /* Temporary work array */
if (*status != SAI__OK) return;
if( (sigcliphigh <= 0.0) && (sigcliplow <= 0.0) ) return;
/* Work out how many bolometers we have */
smf_get_dims( data, NULL, NULL, &nbolo, NULL, NULL,
NULL, NULL, status );
/* Create some space for the result */
smf_create_bolfile( wf, NULL, 1, data, "Noise", "blahs s**0.5",
SMF__MAP_VAR, &noisemap, status );
if (noisemap) noisedata = (noisemap->pntr)[0];
/* Calculate the noise on each bolometer */
smf_bolonoise( wf, data, -1.0, 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
(noisemap->pntr)[0], NULL, NULL, status );
/* Now need to convert this to noise by square rooting */
if (*status == SAI__OK) {
for (i=0; i<nbolo; i++) {
if (noisedata[i] != VAL__BADD) {
noisedata[i] = sqrt( noisedata[i] );
}
}
}
/* Now create a mask and then mask the quality. We only mask bolometers
that are too noisy. We also do not mask bolometers that were already
masked. We want the statistics to reflect what we actually masked
and not any previous masking. This will miss any bolometers masked
out by smf_bolonoise because they had zero power. */
msgOutif( MSG__VERB, "", FUNC_NAME
": Checking for bolometers with outlier noise values. ", status );
msgOutiff( MSG__VERB, "", FUNC_NAME
": Units are (%s s**0.5): ", status, data->hdr->units );
work = astCalloc( nbolo, sizeof(*work) );
if( *status == SAI__OK ) memcpy( work, noisedata, nbolo*sizeof(*work) );
smf_clipnoise( noisedata, nbolo, cliplog, sigcliplow, sigcliphigh, NULL,
status );
/* The only bad values that should appear in noisedata are new bolometers
that were clipped by smf_clipnoise, not bolometers that were bad before
for other reasons. */
if( *status == SAI__OK ) {
for( i=0; i<nbolo; i++ ) {
if( (noisedata[i]==VAL__BADD) && (work[i] == VAL__BADD) ) {
noisedata[i] = 0;
}
}
}
if( work ) work = astFree( work );
/* The mask has to be inside a smfArray */
if (*status == SAI__OK) {
smfArray *masks = smf_create_smfArray( status );
if (masks) masks->owndata = 0; /* someone else owns the smfData */
smf_addto_smfArray( masks, noisemap, status );
smf_apply_mask( wf, data, masks, SMF__BBM_QUAL, SMF__Q_NOISE,
status );
smf_close_related( wf, &masks, status );
}
/* Give noisemap back to caller if requested, or close it */
if( noise ) {
*noise = noisemap;
} else {
smf_close_file( wf, &noisemap, status );
}
}
void smf_calcmodel_gai( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
size_t bstride; /* bolometer stride */
dim_t gain_box=0; /* No. of time slices in a block */
size_t gbstride; /* GAIn bolo stride */
size_t gcstride; /* GAIn coeff stride */
int gflat=0; /* correct flatfield using GAI */
dim_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local GAIn keymap */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nblock; /* No. of time slice blocks */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
smfArray *noi=NULL; /* Pointer to NOI at chunk */
double *noi_data=NULL; /* Pointer to DATA component of model */
size_t noibstride; /* bolo stride for noise */
dim_t nointslice; /* number of time slices for noise */
size_t noitstride; /* Time stride for noise */
dim_t npar; /* No. of parameters per bolometer */
dim_t ntslice; /* Number of time slices */
int oldalg = 1; /* Is the old COM algorithm being used? */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DAT */
double *scale; /* Pointer to scale factor */
size_t tstride; /* time slice stride */
double *wg; /* Workspace holding time slice gains */
double *woff; /* Workspace holding time slice offsets */
/* Main routine */
if( *status != SAI__OK ) return;
if( !(flags&SMF__DIMM_INVERT) ) return;
/* See if the new sigma-clipping COM algorithm is being used. */
astMapGet0A( keymap, "COM", &kmap );
astMapGet0I( kmap, "OLDALG", &oldalg );
kmap = astAnnul( kmap );
/* Obtain pointer to sub-keymap containing GAI parameters */
if( !astMapHasKey( keymap, "GAI" ) ) return;
astMapGet0A( keymap, "GAI", &kmap );
astMapGet0I( kmap, "FLATFIELD", &gflat );
if( kmap ) kmap = astAnnul( kmap );
/* Report an error if gai.flatfield is used with the new COM algorithm. */
if( !oldalg && gflat && *status == SAI__OK ) {
errRep( "", "Cannot use GAI.FLATFIELD with new COM algorithm.", status );
}
/* Only have to do something if gai.flatfield set */
if( !gflat || *status != SAI__OK ) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
if(dat->noi) noi = dat->noi[chunk];
/* Get the number of blocks into which to split each time series. Each box
(except possibly the last one contains "gain_box" time slices. */
astMapGet0A( keymap, "COM", &kmap );
smf_get_nsamp( kmap, "GAIN_BOX", res->sdata[0], &gain_box, status );
if (kmap) kmap = astAnnul( kmap );
if (*status != SAI__OK) return;
/* Ensure everything is in bolo-order */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__NOI, 0, status );
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( noi ) {
smf_get_dims( noi->sdata[idx], NULL, NULL, NULL, &nointslice,
NULL, &noibstride, &noitstride, status);
noi_data = (double *)(noi->sdata[idx]->pntr)[0];
}
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status);
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &npar, NULL,
&gbstride, &gcstride, status);
/* If com.gain_box is zero, use a value of ntslice, so that a single
box will be used covering the whoel time stream. */
if( gain_box == 0 ) gain_box = ntslice;
/* Allocate work space for the gain and offset for each time slice. */
woff = astMalloc( ntslice*sizeof( *woff ) );
wg = astMalloc( ntslice*sizeof( *wg ) );
/* Get the number of blocks into which the time stream is divided.
Each block has a separate gain, offset and correlation factor
for each bolometer. */
nblock = npar/3;
/* Undo the gain correction stored in GAI (the gain is applied to
the signal and noise in smf_calcmodel_com) */
for( i=0; i<nbolo; i++ ) {
if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Get the gain and offset for each time slice of this bolometer. */
smf_gandoff( i, 0, ntslice - 1, ntslice, gbstride, gcstride,
model_data, nblock, gain_box, wg, woff, NULL, status );
/* First undo the flatfield correction to the signal */
scale = wg;
for( j=0; j<ntslice; j++,scale++ ) {
if( !(qua_data[i*bstride + j*tstride]&SMF__Q_MOD) &&
*scale != VAL__BADD && *scale > 0.0 ) {
res_data[i*bstride + j*tstride] *= *scale;
}
}
/* Then scale the noise. */
if( noi ) {
scale = wg;
for( j=0; j<nointslice; j++,scale++ ) {
if( noi_data[i*noibstride + j*noitstride] != VAL__BADD &&
*scale != VAL__BADD && *scale > 0.0 ) {
noi_data[i*noibstride + j*noitstride] *= (*scale) * (*scale);
}
}
}
}
}
/* Free work space. */
woff = astFree( woff );
wg = astFree( wg );
}
}
}
void smfCalcMapcoordPar( void *job_data_ptr, int *status ) {
AstSkyFrame *abskyfrm=NULL;
smfData *data=NULL;
fts2Port fts_port;
int *lbnd_out=NULL;
int *lut=NULL;
int moving;
int *ubnd_out=NULL;
dim_t nbolo; /* number of bolometers */
dim_t ntslice; /* number of time slices */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
AstMapping *sky2map=NULL;
double *theta = NULL;
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if( *status != SAI__OK ) return;
/* Pointer to the data that this thread will process */
pdata = job_data_ptr;
/* Check for valid inputs */
if( !pdata ) {
*status = SAI__ERROR;
errRep( "", "smfCalcMapcoordPar: No job data supplied", status );
return;
}
/* Extract values from pdata */
abskyfrm = pdata->abskyfrm;
data = pdata->data;
lut = pdata->lut;
theta = pdata->theta;
lbnd_out = pdata->lbnd_out;
moving = pdata->moving;
sky2map = pdata->sky2map;
ubnd_out = pdata->ubnd_out;
fts_port = pdata->fts_port;
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* if t1 past end of the work, nothing to do so we return */
if( pdata->t1 >= ntslice ) {
msgOutif( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: nothing for thread to do, returning",
status);
return;
}
/* Debugging message indicating thread started work */
msgOutiff( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: thread starting on tslices %zu -- %zu",
status, pdata->t1, pdata->t2 );
smf_timerinit( &tv1, &tv2, status );
/* Lock the supplied AST object pointers for exclusive use by this
thread. The invoking thread should have unlocked them before
starting this job. */
astLock( abskyfrm, 0 );
astLock( sky2map, 0 );
smf_lock_data( data, 1, status );
/* Calculate and store the LUT values for the range of time slices
being processed by this thread. A generic algorithm is used for
moving targets, but a faster algorithm can be used for stationary
targets. */
smf_coords_lut( data, pdata->tstep, pdata->t1, pdata->t2,
abskyfrm, sky2map, moving, lbnd_out, ubnd_out, fts_port,
lut + pdata->t1*nbolo, theta + pdata->t1,
pdata->lon_ptr, pdata->lat_ptr, status );
/* Unlock the supplied AST object pointers so that other threads can use
them. */
smf_lock_data( data, 0, status );
astUnlock( abskyfrm, 1 );
astUnlock( sky2map, 1 );
msgOutiff( SMF__TIMER_MSG, "",
"smfCalcMapcoordPar: thread finishing tslices %zu -- "
"%zu (%.3f sec)", status, pdata->t1, pdata->t2,
smf_timerupdate(&tv1, &tv2, status) );
}
void smf_rebinmap1( ThrWorkForce *wf, smfData *data, smfData *variance, int *lut,
size_t tslice1, size_t tslice2, int trange,
int *whichmap, dim_t nmap, smf_qual_t mask, int sampvar,
int flags, double *map, double *mapweight,
double *mapweightsq, int *hitsmap,
double *mapvar, dim_t msize, double *scalevariance,
int *status ) {
/* Local Variables */
SmfRebinMap1Data *job_data = NULL;
SmfRebinMap1Data *pdata;
double *dat=NULL; /* Pointer to data array */
size_t dbstride; /* bolo stride of data */
size_t dtstride; /* tstride of data */
int iw; /* Thread index */
dim_t mbufsize; /* Size of full (multi-map) map buffers */
dim_t nbolo; /* number of bolos */
dim_t ntslice; /* number of time slices */
int nw; /* Number of worker threads */
size_t pixstep; /* Number of map pixels per thread */
dim_t bolostep; /* Number of bolos per thread */
smf_qual_t * qual = NULL; /* Quality pointer */
double scalevar; /* variance scale factor */
double scaleweight; /* weights for calculating scalevar */
size_t t1, t2; /* range of time slices to re-grid */
double *var=NULL; /* Pointer to variance array */
size_t vbstride; /* bolo stride of variance */
dim_t vnbolo; /* number of bolos in variance */
dim_t vntslice; /* number of bolos in variance */
size_t vtstride; /* tstride of variance */
/* Main routine */
if (*status != SAI__OK) return;
/* Check inputs */
if( !data || !map || !lut || !mapweight || !mapweightsq || !mapvar ||
!hitsmap ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": Null inputs", status );
return;
}
if( !data->pntr[0] ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": supplied data is empty", status );
return;
}
dat = data->pntr[0];
qual = smf_select_qualpntr( data, NULL, status );
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &dbstride,
&dtstride, status );
/* Size of full map buffers */
if( whichmap ) {
mbufsize = nmap * msize;
} else {
mbufsize = msize;
}
if( variance ) {
var = variance->pntr[0];
smf_get_dims( variance, NULL, NULL, &vnbolo, &vntslice, NULL, &vbstride,
&vtstride, status );
/* Check that the variance dimensions are compatible with data */
if( (*status==SAI__OK) &&
((vnbolo != nbolo) || ((vntslice>1)&&(vntslice!=ntslice))) ) {
*status = SAI__ERROR;
errRep(" ", FUNC_NAME ": variance dimensions incompatible with data",
status );
return;
}
}
/* Range of time slices to regrid */
if( trange ) {
if( tslice2 >= ntslice ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice2 (%zu) can't be >= ntslice (%zu)",
status, tslice2, ntslice );
return;
}
if( tslice1 > tslice2 ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": tslice1 (%zu) > tslice2 (%zu)",
status, tslice1, tslice2 );
return;
}
t1 = tslice1;
t2 = tslice2;
} else {
t1 = 0;
t2 = ntslice-1;
}
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* If this is the first data to be accumulated zero the arrays */
if( flags & AST__REBININIT ) {
memset( map, 0, nw*mbufsize*sizeof(*map) );
memset( mapweight, 0, nw*mbufsize*sizeof(*mapweight) );
memset( mapweightsq, 0, nw*mbufsize*sizeof(*mapweightsq) );
memset( mapvar, 0, nw*mbufsize*sizeof(*mapvar) );
memset( hitsmap, 0, nw*mbufsize*sizeof(*hitsmap) );
}
/* Find how many bolos to process in each worker thread. */
bolostep = nbolo/nw;
if ( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + bolostep - 1;
} else {
pdata->p2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->msize = msize;
pdata->nbolo = nbolo;
pdata->t1 = t1;
pdata->t2 = t2;
pdata->vntslice = vntslice;
pdata->dat = dat;
pdata->map = map;
pdata->mapvar = mapvar;
pdata->mapweightsq = mapweightsq;
pdata->mapweight = mapweight;
pdata->var = var;
pdata->hitsmap = hitsmap;
pdata->lut = lut;
pdata->whichmap = whichmap;
pdata->dbstride = dbstride;
pdata->dtstride = dtstride;
pdata->vbstride = vbstride;
pdata->vtstride = vtstride;
pdata->mask = mask;
pdata->qual = qual;
pdata->mbufsize = mbufsize;
pdata->nw = nw; /* used for final summing/rescaling */
pdata->iw = iw; /* so the thread knows with chunk it's working on */
}
}
if( var ) {
/* Accumulate data and weights in the case that variances are given*/
if( sampvar ) {
/* Measure weighted sample variance for varmap */
if( qual ) { /* QUALITY checking version */
/* Set up jobs to add the previous estimate of COM back on to the
residuals, and then wait for the jobs to complete. These jobs also
clear any SMF__Q_COM flags set by previous iterations. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 1;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 2;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
} else {
/* Otherwise use simple error propagation for varmap */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 3;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 4;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
} else {
/* Accumulate data and weights when no variances are given. In this case
the variance map is always estimated from the sample variance */
if( qual ) { /* QUALITY checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 5;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
} else { /* VAL__BADD checking version */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 6;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
/* If this is the last data to be accumulated re-normalize */
if( flags & AST__REBINEND ) {
/* Find how many buffer pixels to process in each worker thread. */
pixstep = mbufsize/nw;
if( pixstep == 0 ) pixstep = 1;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->p1 = iw*pixstep;
if( iw < nw - 1 ) {
pdata->p2 = pdata->p1 + pixstep - 1;
} else {
pdata->p2 = mbufsize - 1 ;
}
}
if( sampvar || !var ) {
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 7;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
scaleweight=0;
scalevar=0;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
scaleweight += pdata->scaleweight;
scalevar += pdata->scalevar;
}
/* Re-normalize scalevar */
if( scaleweight ) {
scalevar /= scaleweight;
if( scalevariance ) {
*scalevariance = scalevar;
}
}
} else {
/* Re-normalization for error propagation case */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->operation = 8;
thrAddJob( wf, 0, pdata, smf1_rebinmap1, 0, NULL, status );
}
thrWait( wf, status );
}
}
job_data = astFree( job_data );
}
void smf_filter2d_execute( ThrWorkForce *wf, smfData *data, smfFilter *filt,
int complement, int *status ) {
double *data_i=NULL; /* Imaginary part of the transformed data */
double *data_r=NULL; /* Real part of the transformed data */
smfData *fdata=NULL; /* Transform of data */
dim_t fdims[2]={0,0}; /* Frequency dimensions */
size_t i; /* loop counter */
size_t ndims=0; /* Number of real-space dimensions */
size_t nfdata; /* Total number of frequency data points */
smfData *varfilt=NULL; /* real-space square of supplied filter for var */
AstFrameSet *wcs=NULL; /* Copy of real-space WCS */
/* Main routine */
if (*status != SAI__OK) return;
/* Check for NULL pointers */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfData pointer", status );
return;
}
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter pointer", status );
return;
}
if( filt->ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Filter must be 2-dimensional", status );
return;
}
if( smf_isfft( data, NULL, NULL, fdims, NULL, &ndims, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": FFT'd data supplied!", status );
return;
}
if( ndims != 2 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": supplied data are not a 2-d map", status );
return;
}
/* Check that the filter dimensions are appropriate for the data */
for( i=0; i<ndims; i++ ) {
if( fdims[i] != filt->fdims[i] ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": Filter axis %zu has length %zu, doesn't match data %zu",
status, i, filt->fdims[i], fdims[i] );
return;
}
}
/* Dimensions of the transformed data */
nfdata = 1;
for( i=0; i<ndims; i++ ) {
if( filt->fdims[i] != fdims[i] ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": Filter axis %zu has dim %zu doesn't match data dim %zu",
status, i, filt->fdims[i], fdims[i]);
return;
}
nfdata *= fdims[i];
}
/* Using complement of the filter? */
if( complement ) smf_filter_complement( filt, status );
/* Get a copy of the wcs of the input map */
if( data->hdr && data->hdr->wcs ) {
wcs = astCopy( data->hdr->wcs );
}
/* Transform the data */
fdata = smf_fft_data( wf, data, NULL, 0, 0, status );
/* Copy the FFT of the data if we will also be filtering the VARIANCE
since this will get us a useful container of the correct dimensions
for the squared filter */
if( data->pntr[1] ) {
varfilt = smf_deepcopy_smfData( wf, fdata, 0, SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA, 0, 0, status );
}
/* Apply the frequency-domain filter. */
if( *status == SAI__OK ) {
data_r = fdata->pntr[0];
data_i = data_r + nfdata;
if( filt->isComplex ) {
double ac, bd, aPb, cPd;
for( i=0; i<nfdata; i++ ) {
ac = data_r[i] * filt->real[i];
bd = data_i[i] * filt->imag[i];
aPb = data_r[i] + data_i[i];
cPd = filt->real[i] + filt->imag[i];
}
} else {
for( i=0; i<nfdata; i++ ) {
data_r[i] *= filt->real[i];
data_i[i] *= filt->real[i];
}
}
}
/* Transform back */
smf_fft_data( wf, fdata, data, 1, 0, status );
/* Insert the copy of original real-space WCS into the smfHead since
smf_fft_data does not currently calculate it for inverse
transforms. */
if( data->hdr ) {
/* Annul current wcs if it exists */
if( data->hdr->wcs ) {
data->hdr->wcs = astAnnul( data->hdr->wcs );
}
data->hdr->wcs = wcs;
}
/* If we have a VARIANCE component we also need to smooth it. This
is slightly complicated because we have to do the equivalent of a
real-space convolution between the variance map and the
element-wise square of the real-space filter. So we first stuff
the supplied filter into a smfData (frequency space), take its
inverse to real space and square it. We then transform back to
frequency space, and run it through smf_filter2d_execute to apply
it to the VARIANCE map (which is also stuffed into its own
smfData and then copied into the correct location of the supplied
smfData when finished. */
if( (data->pntr[1]) && (*status==SAI__OK) ) {
dim_t ndata;
double *ptr=NULL;
smfData *realfilter=NULL; /* Real space smfData container for var filter */
smfData *vardata=NULL; /* smfData container for variance only */
smfFilter *vfilt=NULL; /* The var filter */
/* Copy the filter into the smfData container and transform into the
time domain. */
ptr = varfilt->pntr[0];
memcpy(ptr, filt->real, nfdata*sizeof(*ptr));
if( filt->imag) {
memcpy(ptr+nfdata, filt->imag, nfdata*sizeof(*ptr));
} else {
memset(ptr+nfdata, 0, nfdata*sizeof(*ptr));
}
realfilter = smf_fft_data( wf, varfilt, NULL, 1, 0, status );
smf_close_file( wf, &varfilt, status );
/* Square each element of the real-space filter and then transform
back to the frequency domain and stuff into a smfFilter
(vfilt). We just point the real and imaginary parts of the
smfFilter to the respective regions of the smfData to save
memory/time, but we need to be careful when freeing at the
end. */
if( *status == SAI__OK ) {
ptr = realfilter->pntr[0];
smf_get_dims( realfilter, NULL, NULL, NULL, NULL, &ndata, NULL, NULL,
status );
if( *status == SAI__OK ) {
double norm = 1. / (double) ndata;
for(i=0; i<ndata; i++) {
/* Note that we need an additional normalization of N samples */
ptr[i] *= ptr[i] * norm;
}
}
}
varfilt = smf_fft_data( wf, realfilter, NULL, 0, 0, status );
if( *status == SAI__OK ) {
ptr = varfilt->pntr[0];
vfilt = smf_create_smfFilter( data, status );
vfilt->real = ptr;
if( filt->isComplex ) {
/* Only worry about imaginary part if the original filter was
complex. */
vfilt->isComplex = 1;
vfilt->imag = ptr + nfdata;
}
}
/* Now stuff the variance array into a smfData and filter it. */
vardata = smf_deepcopy_smfData( wf, data, 0, SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY |
SMF__NOCREATE_FILE |
SMF__NOCREATE_DA, 0, 0, status );
if( *status == SAI__OK ) {
ptr = vardata->pntr[0];
memcpy( ptr, data->pntr[1], ndata*sizeof(*ptr) );
smf_filter2d_execute( wf, vardata, vfilt, 0, status );
}
/* Finally, copy the filtered variance into our output filtered smfData */
if( *status == SAI__OK ) {
ptr = data->pntr[1];
memcpy( ptr, vardata->pntr[0], ndata*sizeof(*ptr) );
}
/* Clean up */
if( realfilter ) smf_close_file( wf, &realfilter, status );
if( vardata ) smf_close_file( wf, &vardata, status );
if( vfilt ) {
vfilt->real = NULL;
vfilt->imag = NULL;
vfilt = smf_free_smfFilter( vfilt, status );
}
}
/* Return the filter to its original state if required */
if( complement == -1 ) smf_filter_complement( filt, status );
/* Clean up */
if( varfilt ) smf_close_file( wf, &varfilt, status );
if( fdata ) smf_close_file( wf, &fdata, status );
}
void smf_bolonoise( ThrWorkForce *wf, smfData *data, double gfrac,
size_t window, double f_low,
double f_white1, double f_white2,
int nep, size_t len, double *whitenoise, double *fratio,
smfData **fftpow,int *status ) {
double *base=NULL; /* Pointer to base coordinates of array */
size_t bstride; /* bolometer index stride */
double df=1; /* Frequency step size in Hz */
size_t i; /* Loop counter */
size_t i_low; /* Index in power spectrum to f_low */
size_t i_w1; /* Index in power spectrum to f_white1 */
size_t i_w2; /* Index in power spectrum to f_white2 */
size_t j; /* Loop counter */
size_t mingood; /* Min. required no. of good values in bolometer */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t nf=0; /* Number of frequencies */
size_t ngood; /* Number of good samples */
dim_t ntslice; /* Number of time slices */
double p_low; /* Power at f_low */
double p_white; /* Average power from f_white1 to f_white2 */
smfData *pow=NULL; /* Pointer to power spectrum data */
smf_qual_t *qua=NULL; /* Pointer to quality component */
double steptime=1; /* Length of a sample in seconds */
size_t tstride; /* time index stride */
if (*status != SAI__OK) return;
/* Check inputs */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
if( !data->hdr ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData has no header", status );
return;
}
/* Obtain dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride, &tstride,
status );
if( *status==SAI__OK ) {
steptime = data->hdr->steptime;
if( steptime < VAL__SMLD ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": FITS header error, STEPTIME must be > 0",
status);
} else {
/* Frequency steps in the FFT */
df = 1. / (steptime * (double) ntslice );
}
}
/* Initialize arrays */
if( whitenoise ) for(i=0; i<nbolo; i++) whitenoise[i] = VAL__BADD;
if( fratio ) for(i=0; i<nbolo; i++) fratio[i] = VAL__BADD;
/* FFT the data and convert to polar power spectral density form */
pow = smf_fft_data( wf, data, NULL, 0, len, status );
smf_convert_bad( wf, pow, status );
smf_fft_cart2pol( wf, pow, 0, 1, status );
{
dim_t fdims[2];
smf_isfft( pow, NULL, NULL, fdims, NULL, NULL, status );
if( *status == SAI__OK ) nf=fdims[0];
}
/* Check for reasonble frequencies, and integer offsets in the array */
i_low = smf_get_findex( f_low, df, nf, status );
i_w1 = smf_get_findex( f_white1, df, nf, status );
i_w2 = smf_get_findex( f_white2, df, nf, status );
/* Get the quality pointer from the smfData so that we can mask known
bad bolometer. */
qua = smf_select_qualpntr( data, NULL, status );
/* The minimum required number of good values in a bolometer. */
mingood = ( gfrac > 0.0 ) ? ntslice*gfrac : 0;
/* Loop over detectors */
for( i=0; (*status==SAI__OK)&&(i<nbolo); i++ )
if( !qua || !(qua[i*bstride]&SMF__Q_BADB) ) {
/* Pointer to start of power spectrum */
base = pow->pntr[0];
base += nf*i;
/* Smooth the power spectrum */
smf_boxcar1D( base, nf, 1, window, NULL, 0, 1, NULL, status );
/* Measure the power */
if( *status == SAI__OK ) {
p_low = base[i_low];
smf_stats1D( base+i_w1, 1, i_w2-i_w1+1, NULL, 0, 0, &p_white, NULL, NULL,
&ngood, status );
/* It's OK if bad status was generated as long as a mean was calculated */
if( *status==SMF__INSMP ) {
errAnnul( status );
/* if we had no good data there was probably a problem with SMF__Q_BADB
so we simply go to the next bolometer */
if (ngood == 0) continue;
}
/* Count the number of initially good values for the current
bolometer. */
if( (*status==SAI__OK) && qua ) {
ngood = 0;
for( j=0; j<ntslice; j++ ) {
if( qua[i*bstride + j*tstride] == 0 ) ngood++;
}
/* Set bolometer to bad if no power detected, or the number of good
values is too low. */
if( (p_low <= 0) || (p_white <= 0) || (ngood < mingood) ) {
for( j=0; j<ntslice; j++ ) {
qua[i*bstride + j*tstride] |= SMF__Q_BADB;
}
}
}
}
if( (*status==SAI__OK) && (!qua || !(qua[i*bstride]&SMF__Q_BADB)) ) {
/* Power ratio requested */
if ( fratio ) {
fratio[i] = p_low/p_white;
}
/* Store values */
if( whitenoise ) {
/* Integrate the PSD by multiplying the average white noise
level by total number of samples and the frequency spacing:
this calculates the time-domain variance (in 200 Hz SCUBA-2
samples for example) assuming this level holds at all
frequencies. */
whitenoise[i] = p_white * ntslice * df;
/* If NEP set, scale this to variance in a 1-second average by
dividing by the sampling frequency (equivalent to
multiplying by sample length). */
if( nep ) {
whitenoise[i] *= steptime;
}
}
}
}
/* Clean up if the caller does not want to take over the power spectrum */
if( pow ) {
if (fftpow) {
*fftpow = pow;
} else {
smf_close_file( &pow, status );
}
}
}
void smf_calcmodel_pln( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
int *lut_data = NULL; /* Lut data for one subarray */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
smfArray *lut=NULL; /* Pointer to LUT at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
size_t tstride; /* Time slice stride in data array */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "PLN", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping PLN this iteration", status );
return;
}
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
lut = dat->lut[chunk];
/* Assert ICD-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__LUT,
1, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, NULL,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
if( *status == SAI__OK ) {
/* Place last iteration of plane signal back into residual */
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
/* Copy the residual+old model into model_data where it will be
fitted again in this iteration. */
memcpy( model_data, res_data,
ndata*smf_dtype_size(res->sdata[idx],status) );
}
/* Calculate the fit and subtract it*/
smf_subtract_plane3( wf, res->sdata[idx], dat->mdims, lut_data, status );
/* Store the difference between the plane-subtracted signal and the residual
in the model container */
if( *status == SAI__OK ) {
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
model_data[i] -= res_data[i];
} else {
model_data[i] = VAL__BADD;
}
}
}
}
}
if( kmap ) kmap = astAnnul( kmap );
model_data_copy = astFree( model_data_copy );
}
void smf_flag_spikes( ThrWorkForce *wf, smfData *data, smf_qual_t mask,
double thresh, size_t box, size_t *nflagged,
int *status ){
/* Local Variables */
int i; /* Loop counter */
smfFlagSpikesData *job_data=NULL;/* Array of job data for each thread */
dim_t nbolo; /* Number of bolometers */
dim_t ntime; /* Number of time-slices */
double *dat = NULL; /* Pointer to bolo data */
smfFlagSpikesData *pdata=NULL;/* Pointer to job data */
int nw; /* Number of worker threads */
size_t bstride; /* Vector stride between bolometer samples */
size_t nflag; /* Number of samples flagged */
size_t tstride; /* Vector stride between time samples */
smf_qual_t *qua = NULL; /* Pointer to quality flags */
size_t step; /* step size for dividing up work */
int njobs=0; /* Number of jobs to be processed */
/* Check inherited status. Also return immediately if no spike flagging
is to be performed. */
if( *status != SAI__OK || thresh == 0.0 ) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Check we have double precision data. */
smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status );
/* Get a pointer to the quality array to use. */
qua = smf_select_qualpntr( data, NULL, status );
/* Report an error if we have no quality array. */
if( !qua && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": No valid QUALITY array was provided", status );
}
/* Get a pointer to the data array to use. Report an error if we have
no data array. */
dat = data->pntr[0];
if( !dat && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": smfData does not contain a DATA component",
status);
}
/* Check the supplied thresh value is valid. */
if( thresh <= 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetd( "THRESH", thresh );
errRep( " ", FUNC_NAME ": Can't find spikes: thresh=^THRESH, must be > 0",
status);
}
/* Check the supplied box value is valid. */
if( box <= 2 && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX, must be > 2",
status);
}
/* Obtain data dimensions, and the stride between adjacent elements on
each axis (bolometer and time). Use the existing data order to avoid
the cost of re-ordering. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntime, NULL, &bstride, &tstride,
status );
/* Check we have room for at least 3 boxes along the time axis. */
if( 3*box > ntime && *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "BOX", box );
msgSeti( "MAX", ntime/3 - 1 );
errRep( " ", FUNC_NAME ": Can't find spikes: box=^BOX is too large, "
" must be < ^MAX.", status);
}
/* Set up the job data */
if( nw > (int) nbolo ) {
step = 1;
} else {
step = nbolo/nw;
if( !step ) {
step = 1;
}
}
job_data = astCalloc( nw, sizeof(*job_data) );
for( i=0; (*status==SAI__OK)&&i<nw; i++ ) {
pdata = job_data + i;
pdata->b1 = i*step;
pdata->b2 = (i+1)*step-1;
/* if b1 is greater than the number of bolometers, we've run out of jobs... */
if( pdata->b1 >= nbolo ) {
break;
}
/* increase the jobs counter */
njobs++;
/* Ensure that the last thread picks up any left-over bolometers */
if( (i==(nw-1)) && (pdata->b1<(nbolo-1)) ) {
pdata->b2=nbolo-1;
}
pdata->ijob = -1; /* Flag job as ready to start */
pdata->box = box;
pdata->bstride = bstride;
pdata->dat = dat;
pdata->mask = mask;
pdata->thresh = thresh;
pdata->nbolo = nbolo;
pdata->ntime = ntime;
pdata->qua = qua;
pdata->tstride = tstride;
}
/* Submit jobs to find spikes in each block of bolos */
thrBeginJobContext( wf, status );
for( i=0; (*status==SAI__OK)&&i<njobs; i++ ) {
pdata = job_data + i;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfFlagSpikesPar, 0, NULL, status );
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Count flagged samples from all of the jobs and free resources */
nflag=0;
if( job_data ) {
for( i=0; i<njobs; i++ ) {
pdata = job_data + i;
nflag += pdata->nflag;
}
job_data = astFree( job_data );
}
/* Return the number of flagged samples, if requested */
if( nflagged ) *nflagged = nflag;
}
void smf_fillgaps( ThrWorkForce *wf, smfData *data,
smf_qual_t mask, int *status ) {
/* Local Variables */
const gsl_rng_type *type; /* GSL random number generator type */
dim_t bpt; /* Number of bolos per thread */
dim_t i; /* Bolometer index */
dim_t nbolo; /* Number of bolos */
dim_t ntslice; /* Number of time slices */
double *dat=NULL; /* Pointer to bolo data */
int fillpad; /* Fill PAD samples? */
size_t bstride; /* bolo stride */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
size_t tstride; /* time slice stride */
smfFillGapsData *job_data; /* Structures holding data for worker threads */
smfFillGapsData *pdata; /* Pointer to data for next worker thread */
smf_qual_t *qua=NULL; /* Pointer to quality array */
/* Main routine */
if (*status != SAI__OK) return;
/* Check we have double precision data floating point data. */
if (!smf_dtype_check_fatal( data, NULL, SMF__DOUBLE, status )) return;
/* Pointers to data and quality */
dat = data->pntr[0];
qua = smf_select_qualpntr( data, NULL, status );
if( !qua ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No valid QUALITY array was provided", status );
return;
}
if( !dat ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": smfData does not contain a DATA component",status);
return;
}
/* obtain data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride, &tstride,
status );
/* Determine how many bolometers to process in each thread, and create
the structures used to pass data to the threads. */
if( wf ) {
bpt = nbolo/wf->nworker;
if( wf->nworker*bpt < nbolo ) bpt++;
job_data = astMalloc( sizeof( smfFillGapsData )*wf->nworker );
} else {
bpt = nbolo;
job_data = astMalloc( sizeof( smfFillGapsData ) );
}
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua, ntslice, tstride, SMF__Q_PAD, &pstart, &pend,
status );
/* Report an error if it is too short. */
if( pend - pstart <= 2*BOX && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": length of data (%d samples) is too small "
"to fill gaps. Must have at least %d samples per bolometer.",
status, (int) ( pend - pstart ), 2*BOX + 1 );
}
/* If the supplied "mask" value includes SMF__Q_PAD, then we will be
replacing the zero-padded region at the start and end of each time series
with artificial noisey data that connects the first and last data values
smoothly. Remove SMF__Q_PAD from the mask. */
if( mask & SMF__Q_PAD ) {
mask &= ~SMF__Q_PAD;
fillpad = 1;
} else {
fillpad = 0;
}
/* Get the default GSL randim number generator type. A separate random
number generator is used for each worker thread so that the gap filling
process does not depend on the the order in which threads are
executed. */
type = gsl_rng_default;
/* Begin a job context. */
thrBeginJobContext( wf, status );
/* Loop over bolometer in groups of "bpt". */
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
/* Store information for this group in the next smfFillGapsData
structure. */
pdata->ntslice = ntslice;
pdata->dat = dat;
pdata->r = gsl_rng_alloc( type );
pdata->b1 = i;
pdata->b2 = i + bpt - 1;
pdata->pend = pend;
pdata->fillpad = fillpad;
pdata->pstart = pstart;
if( pdata->b2 >= nbolo ) pdata->b2 = nbolo - 1;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->qua = qua;
pdata->mask = mask;
/* Submit a job to the workforce to process this group of bolometers. */
(void) thrAddJob( wf, 0, pdata, smfFillGapsParallel, 0, NULL, status );
}
/* Wait until all jobs in the current job context have completed, and
then end the job context. */
thrWait( wf, status );
thrEndJobContext( wf, status );
/* Free resources. */
if( job_data ) {
pdata = job_data;
for( i = 0; i < nbolo; i += bpt, pdata++ ) {
if( pdata->r ) gsl_rng_free( pdata->r );
}
job_data = astFree( job_data );
}
}
void smf_write_bolomap( ThrWorkForce *wf, smfArray *res, smfArray *lut,
smfArray *qua, smfDIMMData *dat, dim_t msize,
const Grp *bolrootgrp, int varmapmethod,
const int *lbnd_out, const int *ubnd_out,
AstFrameSet *outfset, int *status ) {
int addtomap=0; /* Set if adding to existing map */
size_t bstride; /* Bolometer stride */
double *curmap=NULL; /* Pointer to current map being rebinned */
double *curvar=NULL; /* Pointer to variance associate with curmap */
dim_t dsize; /* Size of data arrays in containers */
size_t idx=0; /* index within subgroup */
size_t k; /* loop counter */
int *lut_data=NULL; /* Pointer to DATA component of lut */
char name[GRP__SZNAM+1]; /* Buffer for storing names */
dim_t nbolo; /* Number of bolometers */
size_t nbolomaps = 0; /* Number of bolomaps written */
char *pname=NULL; /* Poiner to name */
smf_qual_t *qua_data=NULL; /* Pointer to DATA component of qua */
double *res_data=NULL; /* Pointer to DATA component of res */
if( *status != SAI__OK ) return;
if( !res || !lut || !qua || !dat || !bolrootgrp ||
!lbnd_out || !ubnd_out || !outfset ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL inputs supplied", status );
return;
}
/* Loop over subgroup index (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
smf_qual_t *bolomask = NULL;
double *bmapweight = NULL;
double *bmapweightsq = NULL;
int *bhitsmap = NULL;
/* Pointers to everything we need */
res_data = res->sdata[idx]->pntr[0];
lut_data = lut->sdata[idx]->pntr[0];
qua_data = qua->sdata[idx]->pntr[0];
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, NULL,
&dsize, &bstride, NULL, status );
/* Make a copy of the quality at first time slice as a good
bolo mask, and then set quality to SMF__Q_BADB. Later we
will unset BADB for one bolo at a time to make individual
maps. */
bolomask = astMalloc( nbolo*sizeof(*bolomask) );
bmapweight = astMalloc( msize*sizeof(*bmapweight) );
bmapweightsq = astMalloc( msize*sizeof(*bmapweightsq) );
bhitsmap = astMalloc( msize*sizeof(*bhitsmap) );
if( *status == SAI__OK ) {
for( k=0; k<nbolo; k++ ) {
bolomask[k] = qua_data[k*bstride];
qua_data[k*bstride] = SMF__Q_BADB;
}
/* Identify good bolos in the copied mask and produce a map */
for( k=0; (k<nbolo)&&(*status==SAI__OK); k++ ) {
if( !(bolomask[k]&SMF__Q_BADB) ) {
Grp *mgrp=NULL; /* Temporary group to hold map names */
smfData *mapdata=NULL;/* smfData for new map */
char tmpname[GRP__SZNAM+1]; /* temp name buffer */
char thisbol[20]; /* name particular to this bolometer */
size_t col, row;
char subarray[10];
nbolomaps++;
/* Set the quality back to good for this single bolometer */
qua_data[k*bstride] = bolomask[k];
/* Create a name for the new map, take into account the
chunk number and subarray. Only required if we are using a single
output container. */
pname = tmpname;
grpGet( bolrootgrp, 1, 1, &pname, sizeof(tmpname), status );
one_strlcpy( name, tmpname, sizeof(name), status );
one_strlcat( name, ".", sizeof(name), status );
/* Subarray, column and row. HDS does not care about case but we
convert to upper case anyhow. */
smf_find_subarray( res->sdata[idx]->hdr, subarray, sizeof(subarray),
NULL, status );
if (*status == SAI__OK) {
size_t len = strlen(subarray);
size_t n = 0;
for (n=0; n<len; n++) {
subarray[n] = toupper(subarray[n]);
}
}
col = (k % res->sdata[idx]->dims[1])+1;
row = (k / res->sdata[idx]->dims[1])+1;
sprintf( thisbol, "%3sC%02zuR%02zu",
subarray,
col, /* x-coord */
row ); /* y-coord */
one_strlcat( name, thisbol, sizeof(name), status );
mgrp = grpNew( "bolomap", status );
grpPut1( mgrp, name, 0, status );
msgOutf( "", "*** Writing single bolo map %s", status,
name );
/* Try to open an existing extention first. Create a new map
array, and then later we'll add it to the existing
one. If it isn't there, create it. */
smf_open_file( mgrp, 1, "UPDATE", 0, &mapdata, status );
if( *status == SAI__OK ) {
/* Allocate memory for the new rebinned data */
curmap = astCalloc( msize, sizeof(*curmap) );
curvar = astCalloc( msize, sizeof(*curvar) );
addtomap = 1;
} else if( *status == DAT__NAMIN ) {
/* Create a new extension */
errAnnul( status );
smf_open_newfile ( mgrp, 1, SMF__DOUBLE, 2, lbnd_out,
ubnd_out, SMF__MAP_VAR, &mapdata, status);
/* Rebin directly into the newly mapped space */
if( *status == SAI__OK ) {
curmap = mapdata->pntr[0];
curvar = mapdata->pntr[1];
addtomap = 0;
}
}
/* Rebin the data for this single bolometer. Don't care
about variance weighting because all samples from
same detector are about the same. */
smf_rebinmap1( wf, res->sdata[idx],
dat->noi ? dat->noi[0]->sdata[idx] : NULL,
lut_data, 0, 0, 0, NULL, 0,
SMF__Q_GOOD, varmapmethod,
AST__REBININIT | AST__REBINEND,
curmap, bmapweight, bmapweightsq, bhitsmap,
curvar, msize, NULL, status );
/* If required, add this new map to the existing one */
if( addtomap ) {
size_t i;
double *oldmap=NULL;
double *oldvar=NULL;
double weight;
if( *status == SAI__OK ) {
oldmap = mapdata->pntr[0];
oldvar = mapdata->pntr[1];
for( i=0; i<msize; i++ ) {
if( oldmap[i]==VAL__BADD ) {
/* No data in this pixel in the old map, just copy */
oldmap[i] = curmap[i];
oldvar[i] = curvar[i];
} else if( curmap[i]!=VAL__BADD &&
oldvar[i]!=VAL__BADD && oldvar[i]!=0 &&
curvar[i]!=VAL__BADD && curvar[i]!=0 ) {
/* Both old and new values available */
weight = 1/oldvar[i] + 1/curvar[i];
oldmap[i] = (oldmap[i]/oldvar[i] + curmap[i]/curvar[i]) /
weight;
oldvar[i] = 1/weight;
}
}
}
/* Free up temporary arrays */
curmap = astFree( curmap );
curvar = astFree( curvar );
}
/* Write out COLNUM and ROWNUM to FITS header */
if( *status == SAI__OK ) {
AstFitsChan *fitschan=NULL;
fitschan = astFitsChan ( NULL, NULL, " " );
atlPtfti( fitschan, "COLNUM", col, "bolometer column", status);
atlPtfti( fitschan, "ROWNUM", row, "bolometer row", status );
atlPtfts( fitschan, "SUBARRAY", subarray, "Subarray identifier",
status );
kpgPtfts( mapdata->file->ndfid, fitschan, status );
if( fitschan ) fitschan = astAnnul( fitschan );
/* Set the bolo to bad quality again */
qua_data[k*bstride] = SMF__Q_BADB;
/* Write WCS */
smf_set_moving(outfset,NULL,status);
ndfPtwcs( outfset, mapdata->file->ndfid, status );
}
/* Clean up */
if( mgrp ) grpDelet( &mgrp, status );
if( mapdata ) smf_close_file( &mapdata, status );
}
}
/* Set quality back to its original state */
for( k=0; k<nbolo; k++ ) {
qua_data[k*bstride] = bolomask[k];
}
}
/* Free up memory */
bolomask = astFree( bolomask );
bmapweight = astFree( bmapweight );
bmapweightsq = astFree( bmapweightsq );
bhitsmap = astFree( bhitsmap );
}
msgOutf( "", "*** Wrote %zu bolo maps", status, nbolomaps );
}
int smf_dataOrder( ThrWorkForce *wf, smfData *data, int isTordered,
int *status ) {
/* Local Variables */
size_t bstr1; /* bolometer index stride input */
size_t bstr2; /* bolometer index stride output */
dim_t i; /* loop counter */
int inPlace=0; /* If set change array in-place */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
dim_t newdims[3]; /* Size of each dimension new buffer */
int newlbnd[3]; /* New pixel origin */
dim_t ntslice; /* Number of time slices */
size_t tstr1; /* time index stride input */
size_t tstr2; /* time index stride output */
int waschanged = 0; /* did we chagne the order? */
int writable; /* Is the NDF writable? */
int freeold; /* Free the old array if change is not in place? */
/* Main routine */
if (*status != SAI__OK) return waschanged;
/* Check for valid isTordered */
if( (isTordered != 0) && (isTordered != 1) ) {
*status = SAI__ERROR;
msgSeti("ISTORDERED",isTordered);
errRep( "", FUNC_NAME ": Invalid isTordered (0/1): ^ISTORDERED", status);
return waschanged;
}
/* Check for a valid data */
if( !data ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL data supplied", status);
return waschanged;
}
/* If value of isTordered matches current value in smfData return */
if( data->isTordered == isTordered ) return waschanged;
/* Make sure we're looking at 3-dimensions of bolo data */
if( data->ndims != 3 ) {
*status = SMF__WDIM;
msgSeti("NDIMS",data->ndims);
errRep( "", FUNC_NAME
": Don't know how to handle ^NDIMS dimensions, should be 3.",
status);
return waschanged;
}
/* We shouldn't be trying to change the order of FFT'd data */
if( smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status ) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": Possible programming error, attempting to re-order FFT of a "
"2D map!", status );
return waschanged;
}
/* we are going to change */
waschanged = 1;
/* inPlace=1 if smfData was mapped! Free the old array if it was not
stored in a file. */
freeold = 1;
if( data->file ){
if (data->file->fd ){
inPlace = 1;
freeold = 0;
/* Only change NDF values in-place if write access is abvaialble for
the NDF. */
} else if( data->file->ndfid ) {
freeold = 0;
ndfIsacc( data->file->ndfid, "WRITE", &writable, status );
if( writable ) inPlace = 1;
}
}
/* Calculate input data dimensions (before changing order) */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstr1, &tstr1,
status);
/* What will the dimensions/strides be in the newly-ordered array? */
if( isTordered ) {
newdims[0] = (data->dims)[1];
newdims[1] = (data->dims)[2];
newdims[2] = (data->dims)[0];
newlbnd[0] = (data->lbnd)[1];
newlbnd[1] = (data->lbnd)[2];
newlbnd[2] = (data->lbnd)[0];
bstr2 = 1;
tstr2 = nbolo;
} else {
newdims[0] = (data->dims)[2];
newdims[1] = (data->dims)[0];
newdims[2] = (data->dims)[1];
newlbnd[0] = (data->lbnd)[2];
newlbnd[1] = (data->lbnd)[0];
newlbnd[2] = (data->lbnd)[1];
bstr2 = ntslice;
tstr2 = 1;
}
/* Loop over elements of data->ptr and re-form arrays */
for( i=0; i<2; i++ ) {
data->pntr[i] = smf_dataOrder_array( wf, data->pntr[i], data->dtype,
data->dtype, ndata,
ntslice, nbolo, tstr1, bstr1, tstr2,
bstr2, inPlace, freeold, status );
}
/* And Quality */
data->qual = smf_dataOrder_array( wf, data->qual, SMF__QUALTYPE, SMF__QUALTYPE,
ndata, ntslice, nbolo, tstr1, bstr1, tstr2,
bstr2, inPlace, freeold, status );
/* If NDF associated with data, modify dimensions of the data */
if( data->file && (data->file->ndfid != NDF__NOID) ) {
msgOutif(MSG__DEBUG, " ", FUNC_NAME
": Warning - current implementation does not modify NDF "
"dimensions to match re-ordered data array", status);
}
/* If there is a LUT re-order it here */
data->lut = smf_dataOrder_array( wf, data->lut, SMF__INTEGER, SMF__INTEGER, ndata,
ntslice, nbolo, tstr1, bstr1, tstr2, bstr2,
inPlace, 1, status );
/* Set the new dimensions in the smfData */
if( *status == SAI__OK ) {
memcpy( data->dims, newdims, 3*sizeof(*newdims) );
memcpy( data->lbnd, newlbnd, 3*sizeof(*newlbnd) );
data->isTordered = isTordered;
}
/* Force any external quality to same ordering */
if (data->sidequal) {
int qchanged = 0;
qchanged = smf_dataOrder( wf, data->sidequal, isTordered, status );
/* and indicate if we changed anything (but not if we did not) */
if (qchanged) waschanged = qchanged;
}
/* Re-order the axes in the time-series WCS FrameSet */
if( data->hdr && data->hdr->tswcs ) {
smf_tswcsOrder( &(data->hdr->tswcs), isTordered, status );
}
/* If the re-ordering was not done in-place, then the new buffer must
have been allocated here. Set a flag so that smf_close_file knows to
deallocate the memory. */
if( ! inPlace ) data->isdyn = 1;
return waschanged;
}
void smurf_unmakemap( int *status ) {
/* Local Variables */
AstFrameSet *wcsin = NULL; /* WCS Frameset for input cube */
AstMapping *skymap; /* GRID->SkyFrame Mapping from input WCS */
AstSkyFrame *abskyfrm; /* Input SkyFrame (always absolute) */
AstSkyFrame *skyfrm = NULL;/* SkyFrame from the input WCS Frameset */
Grp *igrp1 = NULL; /* Group of input sky files */
Grp *igrp2 = NULL; /* Group of input template files */
Grp *igrpc = NULL; /* Group of input COM files */
Grp *igrpg = NULL; /* Group of input GAI files */
Grp *igrpq = NULL; /* Group of input Q sky files */
Grp *igrpu = NULL; /* Group of input U sky files */
Grp *ogrp = NULL; /* Group containing output file */
HDSLoc *cloc = NULL; /* HDS locator for component ipdata structure */
HDSLoc *iploc = NULL; /* HDS locator for top level ipdata structure */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
char ipdata[ 200 ]; /* Text buffer for IPDATA value */
char pabuf[ 10 ]; /* Text buffer for parameter value */
char subarray[ 5 ]; /* Name of SCUBA-2 subarray (s8a,s8b,etc) */
dim_t iel; /* Index of next element */
dim_t ndata; /* Number of elements in array */
dim_t ntslice; /* Number of time slices in array */
double *ang_data = NULL; /* Pointer to the FP orientation angles */
double *angc_data = NULL; /* Pointer to the instrumental ANGC data */
double *c0_data = NULL; /* Pointer to the instrumental C0 data */
double *gai_data = NULL; /* Pointer to the input GAI map */
double *in_data = NULL; /* Pointer to the input I sky map */
double *inc_data = NULL; /* Pointer to the input COM data */
double *inq_data = NULL; /* Pointer to the input Q sky map */
double *inu_data = NULL; /* Pointer to the input U sky map */
double *outq_data = NULL; /* Pointer to the Q time series data */
double *outu_data = NULL; /* Pointer to the U time series data */
double *p0_data = NULL; /* Pointer to the instrumental P0 data */
double *p1_data = NULL; /* Pointer to the instrumental P1 data */
double *pd; /* Pointer to next element */
double *pq = NULL; /* Pointer to next Q time series value */
double *pu = NULL; /* Pointer to next U time series value */
double *qinst_data = NULL; /* Pointer to the instrumental Q data */
double *uinst_data = NULL; /* Pointer to the instrumental U data */
double amp16; /* Amplitude of 16 Hz signal */
double amp2; /* Amplitude of 2 Hz signal */
double amp4; /* Amplitude of 4 Hz signal */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
double params[ 4 ]; /* astResample parameters */
double phase16; /* Phase of 16 Hz signal */
double phase2; /* Phase of 2 Hz signal */
double phase4; /* Phase of 4 Hz signal */
double sigma; /* Standard deviation of noise to add to output */
int alignsys; /* Align data in the map's system? */
int cdims[ 3 ]; /* Common-mode NDF dimensions */
int dims[ NDF__MXDIM ]; /* NDF dimensions */
int flag; /* Was the group expression flagged? */
int gdims[ 3 ]; /* GAI model NDF dimensions */
int harmonic; /* The requested harmonic */
int ifile; /* Input file index */
int indf; /* Input sky map NDF identifier */
int indfangc; /* IP ANGC values NDF identifier */
int indfc0; /* IP C0 values NDF identifier */
int indfc; /* Input COM NDF identifier */
int indfcs; /* NDF identifier for matching section of COM */
int indfg; /* Input GAI NDF identifier */
int indfin; /* Input template cube NDF identifier */
int indfiq; /* Input instrumental Q NDF */
int indfiu; /* Input instrumental U NDF */
int indfout; /* Output cube NDF identifier */
int indfp0; /* IP P0 values NDF identifier */
int indfp1; /* IP P1 values NDF identifier */
int indfq; /* Input Q map NDF identifier */
int indfu; /* Input U map NDF identifier */
int interp = 0; /* Pixel interpolation method */
int lbndc[ 3 ]; /* Array of lower bounds of COM NDF */
int moving; /* Is the telescope base position changing? */
int ndim; /* Number of pixel axes in NDF */
int ndimc; /* Number of pixel axes in common-mode NDF */
int ndimg; /* Number of pixel axes in GAI NDF */
int nel; /* Number of elements in array */
int nelc; /* Number of elements in COM array */
int nelg; /* Number of elements in GAI array */
int nelqu; /* Number of elements in Q or U array */
int ngood; /* No. of good values in putput cube */
int nparam = 0; /* No. of parameters required for interpolation scheme */
int pasign; /* Indicates sense of POL_ANG value */
int sdim[ 2 ]; /* Array of significant pixel axes */
int slbnd[ 2 ]; /* Array of lower bounds of input map */
int subnd[ 2 ]; /* Array of upper bounds of input map */
int ubndc[ 3 ]; /* Array of upper bounds of COM NDF */
size_t ncom; /* Number of com files */
size_t ngai; /* Number of gai files */
size_t nskymap; /* Number of supplied sky cubes */
size_t outsize; /* Number of files in output group */
size_t size; /* Number of files in input group */
smfData *odata = NULL; /* Pointer to output data struct */
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Begin an AST context */
astBegin;
/* Begin an NDF context. */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get an identifier for the input NDF. We use NDG (via kpg1Rgndf)
instead of calling ndfAssoc directly since NDF/HDS has problems with
file names containing spaces, which NDG does not have. */
kpg1Rgndf( "IN", 1, 1, "", &igrp1, &nskymap, status );
ndgNdfas( igrp1, 1, "READ", &indf, status );
/* Map the data array in the input sky map. */
ndfMap( indf, "DATA", "_DOUBLE", "READ", (void **) &in_data, &nel,
status );
/* Get the WCS FrameSet from the sky map, together with its pixel index
bounds. */
kpg1Asget( indf, 2, 0, 1, 1, sdim, slbnd, subnd, &wcsin, status );
/* Check the current Frame is a SKY frame. */
skyfrm = astGetFrame( wcsin, AST__CURRENT );
if( !astIsASkyFrame( skyfrm ) && *status == SAI__OK ) {
ndfMsg( "N", indf );
*status = SAI__ERROR;
errRep( " ", " Current Frame in ^N is not a SKY Frame.", status );
}
/* Get a copy of the current frame that represents absolute coords rather
than offsets. We assume the target is moving if the map represents
offsets. */
moving = ( *status == SAI__OK &&
!strcmp( astGetC( skyfrm, "SkyRefIs" ), "Origin" ) ) ? 1 : 0;
abskyfrm = astCopy( skyfrm );
astClear( abskyfrm, "SkyRefIs" );
/* If the ALIGNSYS parameter is TRUE then we align the raw data with the
map in the current system of the map, rather than the default ICRS. */
parGet0l( "ALIGNSYS", &alignsys, status );
if( alignsys ) astSetC( abskyfrm, "AlignSystem", astGetC( abskyfrm,
"System" ) );
/* Get the Mapping from the Sky Frame to grid axis in the iput map. */
skymap = astGetMapping( wcsin, AST__CURRENT, AST__BASE );
/* Get the pixel interpolation scheme to use. */
parChoic( "INTERP", "NEAREST", "NEAREST,LINEAR,SINC,"
"SINCSINC,SINCCOS,SINCGAUSS,SOMB,SOMBCOS",
1, pabuf, 10, status );
if( !strcmp( pabuf, "NEAREST" ) ) {
interp = AST__NEAREST;
nparam = 0;
} else if( !strcmp( pabuf, "LINEAR" ) ) {
interp = AST__LINEAR;
nparam = 0;
} else if( !strcmp( pabuf, "SINC" ) ) {
interp = AST__SINC;
nparam = 1;
} else if( !strcmp( pabuf, "SINCSINC" ) ) {
interp = AST__SINCSINC;
nparam = 2;
} else if( !strcmp( pabuf, "SINCCOS" ) ) {
interp = AST__SINCCOS;
nparam = 2;
} else if( !strcmp( pabuf, "SINCGAUSS" ) ) {
interp = AST__SINCGAUSS;
nparam = 2;
} else if( !strcmp( pabuf, "SOMB" ) ) {
interp = AST__SOMB;
nparam = 1;
} else if( !strcmp( pabuf, "SOMBCOS" ) ) {
interp = AST__SOMBCOS;
nparam = 2;
} else if( *status == SAI__OK ) {
nparam = 0;
*status = SAI__ERROR;
msgSetc( "V", pabuf );
errRep( "", "Support not available for INTERP = ^V (programming "
"error)", status );
}
/* Get an additional parameter vector if required. */
if( nparam > 0 ) parExacd( "PARAMS", nparam, params, status );
/* Get a group of reference time series files to use as templates for
the output time series files.*/
ndgAssoc( "REF", 1, &igrp2, &size, &flag, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp2, size, size, "More output files required...",
&ogrp, &outsize, status );
/* Get he noise level to add to the output data. */
parGet0d( "SIGMA", &sigma, status );
/* Get any Q and U input maps. */
if( *status == SAI__OK ) {
kpg1Rgndf( "QIN", 1, 1, "", &igrpq, &nskymap, status );
ndgNdfas( igrpq, 1, "READ", &indfq, status );
ndfMap( indfq, "DATA", "_DOUBLE", "READ", (void **) &inq_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "Q", indfq );
*status = SAI__ERROR;
errRep( "", "Q image '^Q' is not the same size as the I image.",
status );
}
kpg1Rgndf( "UIN", 1, 1, "", &igrpu, &nskymap, status );
ndgNdfas( igrpu, 1, "READ", &indfu, status );
ndfMap( indfu, "DATA", "_DOUBLE", "READ", (void **) &inu_data, &nelqu,
status );
if( nelqu != nel && *status == SAI__OK ) {
ndfMsg( "U", indfu );
*status = SAI__ERROR;
errRep( "", "U image '^U' is not the same size as the I image.",
status );
}
if( *status == PAR__NULL ) {
ndfAnnul( &indfq, status );
ndfAnnul( &indfu, status );
inq_data = NULL;
inu_data = NULL;
errAnnul( status );
} else {
parGet0d( "ANGROT", &angrot, status );
parGet0d( "PAOFF", &paoff, status );
parGet0l( "PASIGN", &pasign, status );
}
}
/* Get any common-mode files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "COM", size, size, "", &igrpc, &ncom, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ncom = 0;
}
}
/* Get any GAI files. */
if( *status == SAI__OK ) {
kpg1Rgndf( "GAI", size, size, "", &igrpg, &ngai, status );
if( *status == PAR__NULL ) {
errAnnul( status );
ngai = 0;
}
}
/* Get any instrumental polarisation files. */
if( *status == SAI__OK ) {
/* First see if the user wants to use the "INSTQ/INSTU" scheme for
specifying instrumental polarisation. */
ndfAssoc( "INSTQ", "Read", &indfiq, status );
ndfAssoc( "INSTU", "Read", &indfiu, status );
if( *status == PAR__NULL ) {
ndfAnnul( &indfiq, status );
ndfAnnul( &indfiu, status );
errAnnul( status );
} else {
msgOut( " ", "Using user-defined IP model", status );
ndfDim( indfiq, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiq );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiq, "DATA", "_DOUBLE", "READ", (void **) &qinst_data,
&nel, status );
}
ndfDim( indfiu, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfiu );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfiu, "DATA", "_DOUBLE", "READ", (void **) &uinst_data,
&nel, status );
}
}
/* If not, see if the user wants to use the Johnstone/Kennedy instrumental
polarisation model. The IPDATA parameter gives the path to an HDS
container file contining NDFs holding the required IP data for all
subarrays. */
if( !qinst_data ) {
parGet0c( "IPDATA", ipdata, sizeof(ipdata), status );
if( *status == PAR__NULL ) {
errAnnul( status );
} else {
msgOutf( " ", "Using Johnstone/Kennedy IP model in %s",
status, ipdata );
hdsOpen( ipdata, "READ", &iploc, status );
}
}
}
/* Loop round all the template time series files. */
for( ifile = 1; ifile <= (int) size && *status == SAI__OK; ifile++ ) {
/* Start a new NDF context. */
ndfBegin();
/* Create the output NDF by propagating everything from the input, except
for quality and variance. */
ndgNdfas( igrp2, ifile, "READ", &indfin, status );
ndfMsg( "FILE", indfin );
msgSeti( "THISFILE", ifile );
msgSeti( "NUMFILES", size );
msgOutif( MSG__NORM, " ", "Simulating ^THISFILE/^NUMFILES ^FILE",
status );
ndgNdfpr( indfin, "DATA,HISTORY,LABEL,TITLE,WCS,UNITS,EXTENSION(*)",
ogrp, ifile, &indfout, status );
ndfAnnul( &indfin, status );
ndfAnnul( &indfout, status );
/* We now re-open the output NDF and then modify its data values. */
smf_open_file( wf, ogrp, ifile, "UPDATE", 0, &odata, status );
/* Issue a suitable message and abort if anything went wrong. */
if( *status != SAI__OK ) {
errRep( FUNC_NAME, "Could not open input template file.", status );
break;
} else {
if( odata->file == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfFile associated with smfData.",
status );
break;
} else if( odata->hdr == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME, "No smfHead associated with smfData.",
status );
break;
}
}
/* Check the reference time series contains double precision values. */
smf_dtype_check_fatal( odata, NULL, SMF__DOUBLE, status );
/* Get the total number of data elements, and the number of time slices. */
smf_get_dims( odata, NULL, NULL, NULL, &ntslice, &ndata, NULL,
NULL, status );
/* Get the subarray name */
smf_fits_getS( odata->hdr, "SUBARRAY", subarray, sizeof(subarray),
status );
/* If we are using the Johnstone/Kennedy IP model, open and map the
relevant parameter NDFs within the IPDATA container file. */
if( iploc ) {
datFind( iploc, subarray, &cloc, status );
ndfFind( cloc, "C0", &indfc0, status );
ndfDim( indfc0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfc0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfc0, "DATA", "_DOUBLE", "READ", (void **) &c0_data,
&nel, status );
}
ndfFind( cloc, "P0", &indfp0, status );
ndfDim( indfp0, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp0 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp0, "DATA", "_DOUBLE", "READ", (void **) &p0_data,
&nel, status );
}
ndfFind( cloc, "P1", &indfp1, status );
ndfDim( indfp1, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfp1 );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfp1, "DATA", "_DOUBLE", "READ", (void **) &p1_data,
&nel, status );
}
ndfFind( cloc, "ANGC", &indfangc, status );
ndfDim( indfangc, 2, dims, &ndim, status );
if( dims[ 0 ] != 32 || dims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "N", indfangc );
errRep( " ", "Instrumental polarisation file ^N has bad "
"dimensions - should be 32x40.", status );
} else {
ndfMap( indfangc, "DATA", "_DOUBLE", "READ", (void **) &angc_data,
&nel, status );
}
}
/* Open any COM file. */
if( ncom ) {
ndgNdfas( igrpc, ifile, "READ", &indfc, status );
ndfDim( indfc, 3, cdims, &ndimc, status );
/* Check its dimensions. */
if( *status == SAI__OK ) {
if( ndimc == 1 ) {
if( cdims[ 0 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 0 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 0 ] = lbndc[ 0 ] + ntslice - 1;
ndfSect( indfc, 1, lbndc, ubndc, &indfcs, status );
}
} else if( ndimc == 3 ) {
if( cdims[ 0 ] != 1 || cdims[ 1 ] != 1 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
errRep( " ", "Supplied 3D COM file (^C) has bad "
"dimensions for axis 1 and/or 2 (should "
"both be 1 pixel long).", status );
} else if( cdims[ 2 ] < (int) ntslice ) {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
ndfMsg( "R", indfin );
msgSeti( "N", cdims[ 2 ] );
msgSeti( "M", ntslice );
errRep( " ", "Supplied COM file (^C) has ^N time-slices, but "
"the reference NDF (^R) has ^M time-slices.", status );
} else {
ndfBound( indfc, 3, lbndc, ubndc, &ndimc, status );
ubndc[ 2 ] = lbndc[ 2 ] + ntslice - 1;
ndfSect( indfc, 3, lbndc, ubndc, &indfcs, status );
}
} else {
*status = SAI__ERROR;
ndfMsg( "C", indfc );
msgSeti( "N", ndimc );
errRep( " ", "Supplied COM file (^C) has ^N dimensions - "
"must be 3.", status );
}
}
ndfMap( indfcs, "DATA", "_DOUBLE", "READ", (void **) &inc_data,
&nelc, status );
} else {
indfcs = NDF__NOID;
inc_data = NULL;
}
/* Open any GAI files. */
if( ngai ) {
ndgNdfas( igrpg, ifile, "READ", &indfg, status );
ndfDim( indfg, 3, gdims, &ndimg, status );
/* Check its dimensions, and map it if OK. */
if( *status == SAI__OK ) {
if( ndimg != 2 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
msgSeti( "N", ndimg );
errRep( " ", "Supplied GAI file (^C) has ^N dimensions - "
"must be 2.", status );
} else if( gdims[ 0 ] != 32 || gdims[ 1 ] != 40 ) {
*status = SAI__ERROR;
ndfMsg( "C", indfg );
errRep( " ", "Supplied GAI file (^C) has has bad "
"dimensions - should be 32x40.", status );
}
}
ndfMap( indfg, "DATA", "_DOUBLE", "READ", (void **) &gai_data,
&nelg, status );
} else {
indfg = NDF__NOID;
gai_data = NULL;
}
/* Fill the output with bad values. */
if( *status == SAI__OK ) {
pd = odata->pntr[ 0 ];
for( iel = 0; iel < ndata; iel++ ) *(pd++) = VAL__BADD;
}
/* Resample the sky map data into the output time series. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, in_data, odata->pntr[ 0 ],
NULL, &ngood, status );
/* Add on any COM data. */
smf_addcom( wf, odata, inc_data, status );
/* Issue a wrning if there is no good data in the output cube. */
if( ngood == 0 ) msgOutif( MSG__NORM, " ", " Output contains no "
"good data values.", status );
/* If Q and U maps have been given, allocate room to hold resampled Q and
U values, and fill them with bad values. */
if( inq_data && inu_data ) {
pq = outq_data = astMalloc( ndata*sizeof( *outq_data ) );
pu = outu_data = astMalloc( ndata*sizeof( *outu_data ) );
if( *status == SAI__OK ) {
for( iel = 0; iel < ndata; iel++ ) {
*(pu++) = VAL__BADD;
*(pq++) = VAL__BADD;
}
}
/* Determine the harmonic to use. */
parGet0i( "HARMONIC", &harmonic, status );
/* If producing the normal 8 Hz harmonic, get the amplitude and phase of a
other signals to add onto the 8 Hz signal. */
if( harmonic == 4 ) {
parGet0d( "AMP2", &2, status );
parGet0d( "PHASE2", &phase2, status );
parGet0d( "AMP4", &4, status );
parGet0d( "PHASE4", &phase4, status );
parGet0d( "AMP16", &16, status );
parGet0d( "PHASE16", &phase16, status );
} else {
amp2 = 0.0;
phase2 = 0.0;
amp4 = 0.0;
phase4 = 0.0;
amp16 = 0.0;
phase16 = 0.0;
}
/* Allocate room for an array to hold the angle from the Y pixel axis
in the sky map to the focal plane Y axis, in radians, at each time
slice. Positive rotation is in the same sense as rotation from
focal plane X to focal plane Y. */
ang_data = astMalloc( ntslice*sizeof( *ang_data ) );
/* Resample them both into 3D time series. These Q/U values arw with
respect to the sky image Y axis. */
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inq_data, outq_data,
ang_data, &ngood, status );
smf_resampmap( wf, odata, abskyfrm, skymap, moving, slbnd, subnd,
interp, params, sigma, inu_data, outu_data,
NULL, &ngood, status );
/* Combine these time series with the main output time series so that the
main output is analysed intensity. */
smf_uncalc_iqu( wf, odata, odata->pntr[ 0 ], outq_data, outu_data,
ang_data, pasign, AST__DD2R*paoff, AST__DD2R*angrot,
amp2, AST__DD2R*phase2, amp4, AST__DD2R*phase4,
amp16, AST__DD2R*phase16, qinst_data, uinst_data,
c0_data, p0_data, p1_data, angc_data, harmonic,
status );
/* Release work space. */
outq_data = astFree( outq_data );
outu_data = astFree( outu_data );
ang_data = astFree( ang_data );
}
/* Factor in any GAI data. */
smf_addgai( wf, odata, gai_data, status );
/* Close the output time series file. */
smf_close_file( wf, &odata, status );
/* Close the IP data container for the current subarray, if it is open. */
if( cloc ) datAnnul( &cloc, status );
/* End the NDF context. */
ndfEnd( status );
}
/* Close any input data file that is still open due to an early exit from
the above loop. */
if( odata != NULL ) {
smf_close_file( wf, &odata, status );
odata = NULL;
}
/* Free remaining resources. */
if( igrp1 != NULL) grpDelet( &igrp1, status);
if( igrp2 != NULL) grpDelet( &igrp2, status);
if( igrpq != NULL) grpDelet( &igrpq, status);
if( igrpu != NULL) grpDelet( &igrpu, status);
if( igrpc != NULL) grpDelet( &igrpc, status);
if( igrpg != NULL) grpDelet( &igrpg, status);
if( ogrp != NULL) grpDelet( &ogrp, status);
if( iploc ) datAnnul( &iploc, status );
/* End the NDF context. */
ndfEnd( status );
/* End the tile's AST context. */
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif(MSG__VERB," ",TASK_NAME " succeeded, time series written.", status);
} else {
msgOutif(MSG__VERB," ",TASK_NAME " failed.", status);
}
}
void smurf_sc2threadtest( int *status ) {
/* Local Variables */
smfArray **res=NULL; /* array of smfArrays of test data */
smfData *data=NULL; /* Pointer to SCUBA2 data struct */
dim_t datalen; /* Number of data points */
smfFilter *filt=NULL; /* Frequency domain filter */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
smfTimeChunkData *job_data=NULL; /* Array of pointers for job data */
size_t joblen; /* Number of chunks per job */
size_t k; /* Loop counter */
size_t nchunks; /* Number of chunks */
size_t nsub; /* Number of subarrays */
int nthread; /* Number of threads */
smfTimeChunkData *pdata=NULL; /* Pointer to data for single job */
int temp; /* Temporary integer */
size_t tsteps; /* How many time steps in chunk */
struct timeval tv1, tv2; /* Timers */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
double *dat=NULL;
dim_t nbolo;
dim_t ntslice;
dim_t ndata;
size_t bstride;
size_t tstride;
dim_t offset;
if (*status != SAI__OK) return;
/* Get input parameters */
parGdr0i( "NTHREAD", 1, 1, NUM__MAXI, 1, &nthread, status );
parGdr0i( "TSTEPS", 6000, 0, NUM__MAXI, 1, &temp, status );
tsteps = (size_t) temp;
parGdr0i( "NCHUNKS", 1, 1, NUM__MAXI, 1, &temp, status );
nchunks = (size_t) temp;
parGdr0i( "NSUB", 1, 1, 4, 1, &temp, status );
nsub = (size_t) temp;
msgSeti("N",nthread);
msgOut( "", TASK_NAME ": Running test with ^N threads", status );
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Create some fake test data in the form of an array of smfArrays */
msgSeti("T",tsteps);
msgSeti("C",nchunks);
msgSeti("NS",nsub);
msgOut( "", TASK_NAME
": Creating ^NS subarrays of data with ^C chunks * ^T samples",
status );
res = astCalloc( nchunks, sizeof(*res) );
for( k=0; (*status==SAI__OK)&&(k<nchunks); k++ ) {
res[k] = smf_create_smfArray( status );
for( i=0; (*status==SAI__OK)&&(i<nsub); i++ ) {
/* Create individual smfDatas and add to array */
data = smf_create_smfData( SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS, status );
if( *status==SAI__OK ) {
data->dtype=SMF__DOUBLE;
data->ndims=3;
data->dims[0]=40;
data->dims[1]=32;
data->dims[2]=(dim_t) tsteps;
datalen=1;
data->isFFT=-1;
for( j=0; j<data->ndims; j++ ) datalen *= data->dims[j];
data->hdr->steptime = 0.005;
data->pntr[0] = astCalloc( datalen, smf_dtype_sz(data->dtype,status) );
data->qual = astCalloc( datalen, sizeof(*data->qual) );
}
smf_addto_smfArray( res[k], data, status );
}
}
/*** TIMER ***/
msgOutf( "", "** %f seconds generating data", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 1 __parallel time: dataOrder__", status );
/* Create a pool of threads. */
wf = thrGetWorkforce( nthread, status );
/* Work out number of chunks per thread */
joblen = nchunks/nthread;
if( joblen == 0 ) joblen = 1; /* At least one chunk per thread */
/* The first test will process separate time chunks of data in
parallel, re-ordering each to bolo-ordered format. All subarrays
and an integer number of input file chunks all go into a single
thread. Start by allocating and initializing a number of
smfTimeChunkData's that hold the information required for each
thread */
job_data = astCalloc( nthread, sizeof(*job_data) );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 0; /* Start with a data re-order */
pdata->data = res; /* Pointer to main data array */
pdata->chunk1 = i*joblen; /* Index of first chunk for job */
pdata->nchunks = nchunks; /* Total number of time chunks in data */
pdata->ijob = -1; /* Flag job as available to do work */
/* The last thread has to pick up the remainder of chunks */
if( i==(size_t)(nthread-1) ) pdata->chunk2=nchunks-1;
else pdata->chunk2 = (i+1)*joblen-1; /* Index of last chunk for job */
/* Ensure a valid chunk range, or set to a length that we know to ignore */
if( pdata->chunk1 >= nchunks ) {
pdata->chunk1 = nchunks;
pdata->chunk2 = nchunks;
} else if( pdata->chunk2 >= nchunks ) {
pdata->chunk2 = nchunks-1;
}
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/* Annul the bad status that we set in smfParallelTime */
if( *status == SMF__INSMP ) {
errAnnul( status );
msgOut( "", " *** Annulled SMF__INSMP set in smfParallelTime *** ",
status );
} else {
msgOut( "", " *** Flushing good status *** ", status );
errFlush( status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* The second test will boxcar smooth bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 2 __parallel time: boxcar smooth__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 1; /* Boxcar smooth */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": *** Next 2 tests will be done twice due to FFTW planning *****",
status );
for( k=0; k<2; k++ ) {
/* The third test will FFT filter bolometers from time chunks in
parallel */
msgOut( "", TASK_NAME
": Starting test 3 __parallel time: FFT filter__", status );
for( i=0; (i<(size_t)nthread) && (*status==SAI__OK); i++ ) {
pdata = job_data + i;
pdata->type = 2; /* FFT filter */
if( pdata->chunk1 >= nchunks ) {
/* Nothing for this thread to do */
msgSeti( "W", i+1);
msgOutif( MSG__DEBUG, "",
"-- parallel time: skipping thread ^W, nothing to do",
status);
} else {
/* Since we know there is one job_data per thread, just submit jobs
immediately */
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata, smfParallelTime,
0, NULL, status );
}
}
/* Wait until all of the submitted jobs have completed */
thrWait( wf, status );
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut( "", TASK_NAME
": Starting test 4 __FFTW filter using internal threading__",
status );
for( i=0; (*status==SAI__OK)&&(i<nchunks); i++ ) {
filt = smf_create_smfFilter( res[i]->sdata[0], status );
smf_filter_ident( filt, 1, status );
for( j=0; (*status==SAI__OK)&&(j<nsub); j++ ) {
msgOutiff( MSG__DEBUG, "", " filter chunk %zu/%zu, bolo %zu/%zu",
status, i+1, nchunks, j+1, nsub );
smf_filter_execute( wf, res[i]->sdata[j], filt, 0, 0, status );
}
if( filt ) filt = smf_free_smfFilter( filt, status );
}
/*** TIMER ***/
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
}
msgOut( "", TASK_NAME
": **************************************************************",
status );
/* Series of short single-thread array index tests */
data = res[0]->sdata[0];
dat = data->pntr[0];
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, &ndata, &bstride,
&tstride, status );
msgOut("","Array index test #1: two multiplies in inner loop",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo; i++ ) {
for( j=0; j<ntslice; j++ ) {
dat[i*bstride + j*tstride] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #2: only index increments",status);
smf_timerinit( &tv1, &tv2, status );
for( i=0; i<nbolo*bstride; i+=bstride ) {
for( j=i; j<(i+ntslice*tstride); j+=tstride ) {
dat[j] += 5;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
msgOut("","Array index test #3: one multiply in outer loop",status);
smf_timerinit( &tv1, &tv2, status );
offset = 0;
for( i=0; i<nbolo; i++ ) {
offset = i*bstride;
for( j=0; j<ntslice; j++ ) {
dat[offset] += 5;
offset += tstride;
}
}
msgOutf( "", "** %f seconds to complete test", status,
smf_timerupdate(&tv1,&tv2,status) );
/* Clean up */
if( res ) {
for( i=0; i<nchunks; i++ ) {
if( res[i] ) {
smf_close_related( &res[i], status );
}
}
res = astFree( res );
}
job_data = astFree( job_data );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
int smf_block_end( smfData *data, int block_start, int ipolcrd, float arcerror,
int *status ){
/* Local Variables: */
const JCMTState *state; /* JCMTState info for current time slice */
dim_t ntslice; /* Number of time-slices in data */
double ac1_start; /* Tracking longitude at start of block (rads) */
double ac2_start; /* Tracking latitude at start of block (rads) */
double ang_start; /* Tracking orientaion at start of block (rads) */
double angle; /* Rotation that gives arcerror shift at corners */
double end_wang; /* Half-waveplate angle at end of block */
double start_wang; /* Half-waveplate angle at start of block */
double wang; /* Half-waveplate angle */
int ifail; /* Index of last time slice to fail the test */
int inc; /* No. of time slices between tests */
int ipass; /* Index of last time slice to pass the test */
int itime; /* Time slice index at next test */
int result; /* The returned time slice index at block end */
smfHead *hdr; /* Pointer to data header this time slice */
/* Initialise */
result = -1;
/* Check the inherited status. */
if( *status != SAI__OK ) return result;
/* Convenience pointers. */
hdr = data->hdr;
/* Obtain number of time slices - will also check for 3d-ness. */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Check that we have not used all time slices. */
if( block_start >= 0 && block_start < (int) ntslice ) {
/* Determine the critical angle (in radians) that produces a shift of
"arcerror" arc-seconds at a distance of 4 arc-minutes (the rough radius
of the SCUBA-2 field of view) from the focal plane centre. */
angle = arcerror/( 4.0 * 60 );
/* Convert arcerror from arc-seconds to radians. */
arcerror *= AST__DD2R/3600.0;
/* Note the actual boresight position and focal plane orientation within
the tracking system at the block start, all three in radians. */
state = (hdr->allState) + block_start;
ac1_start = state->tcs_tr_ac1;
ac2_start = state->tcs_tr_ac2;
ang_start = state->tcs_tr_ang;
/* For speed, we do not test every time slice. Instead we first check the
time slice following the start, but then accelarate through subsequent
time slices, doubling the gap between tests each time. When we find
a test that fails (i.e. the focal plane has moved further than
"arcerror" from the starting time slice), we do a binary chop between
that test and the previous successful test. */
ipass = 0;
ifail = -1;
inc = 1;
itime = block_start + inc;
while( itime < (int) ntslice ) {
state = (hdr->allState) + itime;
/* The test fails if the actual telescope position in the tracking system
has moved by more than arcerror arc-seconds since the block start, or
if the focal plane has rotated on the sky (in the tracking system) by
more than the critical angle. Leave the loop if the test fails. */
if( FPMOVED ) {
ifail = itime;
break;
}
/* The focal plane has not yet moved significantly away from its position
at the starting time slice. Record the index of the last known unmoved
time slice. */
ipass = itime;
/* Get the index of the time slice to test next. We double the gap
between tests. */
inc *= 2;
itime += inc;
}
/* If no test failed, test the last time slice. If the last time slice
passes the test then all time slices are in the block. So set the result
to the index of the last time slice. */
if( ifail == -1 ) {
itime = ntslice - 1;
state = (hdr->allState) + itime;
if( FPMOVED ) {
ifail = itime;
} else {
result = itime;
}
}
/* If we have not yet found the result, do a binary chop between the last
successful test and the first unsuccessful test. */
if( result == -1 ) {
/* Loop until the failed and successful tests are adjacent to each other. */
while( ifail > ipass + 1 ) {
/* Perform a test at the central time slice between the failed and
successful tests. If the time slice fails the test, use it to replace
"ifail". otherwise use it to replace "ipass". */
itime = ( ifail + ipass )/2;
state = (hdr->allState) + itime;
if( FPMOVED ) {
ifail = itime;
} else {
ipass = itime;
}
}
/* The returned index is the index of the last time slice to pass the test. */
result = ipass;
}
/* In order to reduce inaccuracies when finding the required Fourier
component of the time series, we now shorten the block until it spans an
integral number of quarter revolutions of the half-waveplate relative to
tracking north. We use quarter revolutions rather than whole
revolutions because analysed intensity varies four times faster than
the half-waveplate position. First find the half-waveplate angle with
respect to tracking north, at the start of the block. */
state = (hdr->allState) + block_start;
start_wang = state->pol_ang;
if( ipolcrd == 0 ) {
start_wang -= state->tcs_tr_ang;
} else if( ipolcrd == 1 ) {
start_wang -= state->tcs_tr_ang - state->tcs_az_ang ;
}
/* Now find the half-waveplate angle with respect to tracking north, at
the current end of the block. */
state = (hdr->allState) + result;
end_wang = state->pol_ang;
if( ipolcrd == 0 ) {
end_wang -= state->tcs_tr_ang;
} else if( ipolcrd == 1 ) {
end_wang -= state->tcs_tr_ang - state->tcs_az_ang ;
}
/* On the assumption that POL_ANG increases with time, if the
half-waveplate angle at the end of the block is less than at the start of
the block, it must have reached 2*PI and wrapped back round to zero. So
add on 2*PI to the end value. */
if( end_wang < start_wang ) end_wang += 2*AST__DPI;
/* Reduce the end angle so that it is an integral number of quarter
revolutions in front of the start angle. We are assuming here that
POL_ANG increases (rather than decreasing) with time. */
end_wang = start_wang +
AST__DPIBY2*( (int) ( ( end_wang - start_wang )/AST__DPIBY2 ) );
/* If the end angle is greater than 2*PI, reduce it by 2.PI. */
if( end_wang > 2*AST__DPI ) end_wang -= 2*AST__DPI;
/* Work backwards through the time slices, starting at the current end
time slice, until a time slice is found which has an angle less than the
end angle found above. */
for( ; result >= block_start; result-- ) {
state = (hdr->allState) + result;
wang = state->pol_ang;
if( ipolcrd == 0 ) {
wang -= state->tcs_tr_ang;
} else if( ipolcrd == 1 ) {
wang -= state->tcs_tr_ang - state->tcs_az_ang ;
}
if( wang < end_wang ) break;
}
}
/* Return the index of the last time slice in the block, or -1 if an
error has occurred. */
return ( *status == SAI__OK && result >= block_start ) ? result : -1 ;
}
void smf_expmodel_dks( const smfData *indata, smfData **outdata, int *status) {
/* Local Variables */
size_t bolo; /* bolo number */
size_t bstride; /* bolo stride */
double *d=NULL; /* pointer to data array */
smfData *data=NULL; /* newly created smfData */
double *dksquid=NULL; /* Pointer to current dark squid */
double *gainbuf=NULL; /* Array of gains for all bolos in this col */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
dim_t nbolo; /* Number of bolometers */
dim_t ncol; /* Number of columns */
dim_t nrow; /* Number of rows */
dim_t ntslice; /* Number of time slices */
double *offsetbuf=NULL; /* Array of offsets for all bolos in this col */
size_t tstride; /* time stride */
/* Main routine */
if (*status != SAI__OK) return;
/* Create an empty container initialized to VAL__BADD */
smf_expmodel_init( indata, &data, status );
if( *status != SAI__OK ) return;
if( !indata->pntr[0] || !data->pntr[0] ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL DATA pointer encountered", status );
}
/* Pointer to output data buffer */
d = data->pntr[0];
/* Get array dimensions*/
smf_get_dims( data, &nrow, &ncol, &nbolo, &ntslice, NULL, &bstride, &tstride,
status);
/* Loop over column */
for( i=0; (*status==SAI__OK)&&(i<ncol); i++ ) {
/* get pointer to the dark squid and gain/offset buffers this column */
dksquid = indata->pntr[0];
dksquid += i*(ntslice+nrow*3);
gainbuf = dksquid + ntslice;
offsetbuf = gainbuf + nrow;
/* Loop over rows */
for( j=0; (*status==SAI__OK)&&(j<nrow); j++ ) {
/* Calculate the bolo number */
if( SC2STORE__COL_INDEX ) {
bolo = i*nrow + j;
} else {
bolo = i + j*ncol;
}
/* Scale the dark squid into the expanded buffer*/
if( (gainbuf[j] != VAL__BADD) && (offsetbuf[j] != VAL__BADD) ) {
for( k=0; k<ntslice; k++ ) {
if( dksquid[k] != VAL__BADD) {
d[bolo*bstride+k*tstride] = dksquid[k]*gainbuf[j] + offsetbuf[j];
}
}
}
}
}
/* Return pointer to new data */
if( outdata ) *outdata = data;
}
int smf_fix_data ( msglev_t msglev, smfData * data, int * status ) {
int have_fixed = 0; /* Did we fix anything? */
smfHead *hdr = NULL; /* Data header struct */
size_t i; /* Loop counter */
size_t j; /* Loop counter */
size_t k; /* Loop counter */
size_t l; /* Loop counter */
sc2ast_subarray_t subnum; /* subarray number */
int was_fixed = 0; /* Were these data fixed before? */
if (*status != SAI__OK) return have_fixed;
/* Validate arguments - need smfData with smfHead including FITS
and state */
smf_validate_smfData( data, 1, 0, status );
if (*status != SAI__OK) return have_fixed;
hdr = data->hdr;
smf_validate_smfHead( hdr, 1, 1, status );
if (*status != SAI__OK) return have_fixed;
/* Only works on SCUBA-2 time-series data */
if( (hdr->instrument != INST__SCUBA2) ||
(data->ndims != 3) ) {
return have_fixed;
}
/* Need to at least have a DATA component. It might be a good idea
to check if there is a file associated with the data, and check
to see whether the caller mapped the DATA, VARIANCE, QUALITY. If
they only mapped a subset of those, we could generate a warning that
this routine may be introducing an inconsistency */
if( !data->pntr[0] ) {
return have_fixed;
}
/* Check to see if a FIXDATA FITS keyword exists -- if so, smf_fix_data
has already been run on the data and we return. */
smf_getfitsi( hdr, SMFFIXDATA, &was_fixed, status );
if( *status == SMF__NOKWRD ) {
errAnnul( status );
} else {
return have_fixed;
}
/* Fix the May/June 2011 s4a row readout order error. Check the
subarray first, then get the MJD for the start of the observation
to decide if we need to proceed. */
smf_find_subarray( hdr, NULL, 0, &subnum, status );
if( (*status == SAI__OK) && (subnum == S4A) ) {
double dateendfix; /* UTC MJD end date when fix needed */
double dateobs; /* UTC MJD observation start */
double datestartfix; /* UTS MJD start date when fix needed */
AstTimeFrame *tf = NULL; /* time frame for date conversion */
/* Get the MJD for start of the observation */
smf_find_dateobs( hdr, &dateobs, NULL, status );
/* Get the MJD for the first and last dates bracketing the period
during which the row order fix needs to be done */
tf = astTimeFrame( " " );
astSet( tf, "TimeScale=UTC" );
/* From Mike Macintosh Thu, 7 Jul 2011 15:07:51 +0000:
"I did a scan of the engineering MCE status files and it looks
the 'missing' row on s4a originated between 09:30 and 10:30 HST
on 26 May 2011. I haven't checked in detail whether the
'missing' row was consistently missing after that date but the
assumption is that it was."
*/
astSet( tf, "TimeOrigin=%s", "2011-05-26T19:00:00" );
datestartfix = astGetD( tf, "TimeOrigin" );
/* From Dan Bintley Wed, 06 Jul 2011 00:23:18 -1000:
"I have changed the source code in
/jac_sw/itsroot/scuba2_src/scuba2Da/setup/SG450_M1004D1000/mce_zero.txt"
*/
astSet( tf, "TimeOrigin=%s", "2011-07-06T10:23:00" );
dateendfix = astGetD( tf, "TimeOrigin" );
tf = astAnnul( tf );
/* Proceed with fix if within date range */
if( (*status == SAI__OK) && (dateobs >= datestartfix) &&
(dateobs <= dateendfix) ) {
size_t targetbolo;
size_t sourcebolo;
size_t bstride;
dim_t ncols;
dim_t nrows;
dim_t ntslice;
size_t tstride;
msgOutif( msglev, "", INDENT "Reparing row order readout error", status );
smf_get_dims( data, &nrows, &ncols, NULL, &ntslice, NULL, &bstride,
&tstride, status );
if( (*status==SAI__OK) && (nrows != 40) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": looks like we need to fix the data, but we "
"don't have 40 rows.", status );
}
/* Rows 0--13 are OK, row 14 was not read out
(FIXDATA_S4A_MISSROW), so we need to move all of rows 14--38
up to the range 15--39. After doing the shift, overwrite the
missing row 14 with 0 (or should this be VAL__BAD? in some
situations?) */
if( *status==SAI__OK ) {
/* Row counter */
for( i=nrows-1; (*status==SAI__OK)&&(i>=(FIXDATA_S4A_MISSROW+1)); i-- ){
for( j=0; (*status==SAI__OK)&&(j<ncols); j++ ) {
smf_qual_t *qual = data->qual;
/* Calculate the bolo index */
if( SC2STORE__ROW_INDEX ) {
/* Fastest changing index is column number */
sourcebolo = (i-1)*ncols + j;
targetbolo = i*ncols + j;
} else {
/* Fastest changing index is row number */
sourcebolo = (i-1) + j*nrows;
targetbolo = i + j*nrows;
}
/* Loop for DATA / VARIANCE components */
for( l=0; (*status==SAI__OK)&&(l<2); l++ ) {
void *buf = data->pntr[l];
if( buf ) {
switch( data->dtype ) {
case SMF__INTEGER:
for( k=0; k<ntslice; k++ ) {
((int *)buf)[targetbolo*bstride + k*tstride] =
((int *)buf)[sourcebolo*bstride + k*tstride];
}
if( i==(FIXDATA_S4A_MISSROW+1) ) {
for( k=0; k<ntslice; k++ ) {
((int *)buf)[sourcebolo*bstride + k*tstride] = 0;
}
}
break;
case SMF__USHORT:
for( k=0; k<ntslice; k++ ) {
((unsigned short *)buf)[targetbolo*bstride + k*tstride] =
((unsigned short *)buf)[sourcebolo*bstride + k*tstride];
}
if( i==(FIXDATA_S4A_MISSROW+1) ) {
for( k=0; k<ntslice; k++ ) {
((unsigned short *)buf)[sourcebolo*bstride + k*tstride] =
0;
}
}
break;
case SMF__DOUBLE:
for( k=0; k<ntslice; k++ ) {
((double *)buf)[targetbolo*bstride + k*tstride] =
((double *)buf)[sourcebolo*bstride + k*tstride];
}
if( i==(FIXDATA_S4A_MISSROW+1) ) {
for( k=0; k<ntslice; k++ ) {
((double *)buf)[sourcebolo*bstride + k*tstride] =
0;
}
}
break;
default:
*status = SAI__ERROR;
errRepf( "", FUNC_NAME
": Don't know how to handle %s type.", status,
smf_dtype_str(data->dtype,status) );
}
}
}
/* Now do quality: Just set SMF__Q_BADDA for the dead
row. Not sure if I should do something with sidecar
quality as well? */
if( qual ) {
for( k=0; k<ntslice; k++ ) {
qual[targetbolo*bstride + k*tstride] =
qual[sourcebolo*bstride + k*tstride];
}
if( i==(FIXDATA_S4A_MISSROW+1) ) {
for( k=0; k<ntslice; k++ ) {
qual[sourcebolo*bstride + k*tstride] = SMF__Q_BADDA;
}
}
}
}
}
}
/* If we get here and status is OK, set return value */
if( *status == SAI__OK ) {
have_fixed |= SMF__FIXED_ROWORDER;
}
}
}
/* Set FITS header so that we know smf_fix_data was run */
smf_fits_updateL( hdr, SMFFIXDATA, have_fixed,
(have_fixed ? "Data have been fixed" : "Data do not require fixing"),
status );
return have_fixed;
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_subip( ThrWorkForce *wf, smfArray *res, smfArray *lut, int *lbnd_out,
int *ubnd_out, AstKeyMap *keymap, AstFrameSet *outfs, int *status ) {
/* Local Variables: */
HDSLoc *loc = NULL;
HDSLoc *sloc = NULL;
SmfSubIPData *job_data = NULL;
SmfSubIPData *pdata;
char ipref[200];
char subname[10];
const char *ipdata;
const char *qu;
dim_t bolostep;
dim_t nbolo;
dim_t ntslice;
double *angcdata;
double *c0data;
double *imapdata;
double *ipang;
double *p0data;
double *p1data;
int angcndf;
int c0ndf;
int imapndf;
int iw;
int nmap;
int nw;
int p0ndf;
int p1ndf;
size_t bstride;
size_t idx;
size_t tstride;
smfData *data;
smf_qual_t *qua_data;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Check if we have pol2 data, and see if it is Q or U. */
qu = NULL;
for( idx = 0; idx < res->ndat; idx++ ) {
data = res->sdata[idx];
if( !strcmp( data->hdr->dlabel, "Q" ) ){
if( !qu ) {
qu = "Q";
} else if( strcmp( qu, "Q" ) ) {
*status = SAI__ERROR;
break;
}
} else if( !strcmp( data->hdr->dlabel, "U" ) ) {
if( !qu ) {
qu = "U";
} else if( strcmp( qu, "U" ) ) {
*status = SAI__ERROR;
break;
}
} else if( qu ) {
*status = SAI__ERROR;
qu = NULL;
break;
}
}
/* Report an error if there is a mix of pol2 and non-pol2, or a mix of Q
and U. */
if( *status != SAI__OK ) {
if( qu ) {
errRep( "", "smf_subip: Input data contains mix of Q and U "
"data", status );
} else {
errRep( "", "smf_subip: Input data contains mix of POL2 and "
"non-POL2 data", status );
}
/* If we have pol2 data, get the path to the total intensity image that
is to be used to define the level of IP correction required. If no
value is supplied, annul the error and set "qu" NULL to indicate we should
leave immediately. */
} else if( qu && *status == SAI__OK ) {
parGet0c( "IPREF", ipref, sizeof(ipref), status );
if( *status == PAR__NULL ) {
errAnnul( status );
qu = NULL;
}
}
/* If we are applying IP correction... */
if( qu && *status == SAI__OK ) {
msgOutf( "", "smf_subip: applying instrumental polarisation %s "
"correction based on total intensity map `%s'", status,
qu, ipref );
/* Get an identifier for the IPREF NDF. */
ndfFind( NULL, ipref, &imapndf, status );
/* Resample the NDFs data values onto the output map grid. */
imapdata = smf_alignndf( imapndf, outfs, lbnd_out, ubnd_out,
status );
/* Annul the NDF identifier. */
ndfAnnul( &imapndf, status );
/* Create structures used to pass information to the worker threads. */
nw = wf ? wf->nworker : 1;
job_data = astMalloc( nw*sizeof( *job_data ) );
/* Get the path to the container file holding the IP model parameters. */
ipdata = "$STARLINK_DIR/share/smurf/ipdata.sdf";
astMapGet0C( keymap, "IPDATA", &ipdata );
/* Open the container file. */
hdsOpen( ipdata, "READ", &loc, status );
/* Do the IP correction for each subarray (s8a, s8b, etc) in turn. */
for( idx = 0; idx < res->ndat && *status == SAI__OK; idx++ ) {
data = res->sdata[idx];
/* Get an array holding the angle (rad.s) from north to focal plane Y,
measured positive in the sense of rotation from focal plane Y to focal
plane X, for every bolometer sample in the smfData. The values are bolo
ordered so that "bstride" is 1 and "tstsride" is nbolo. */
ipang = smf1_calcang( data, status );
/* Get the number of bolometers and time slices for the current subarray,
together with the strides between adjacent bolometers and adjacent
time slices. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Get a locator for the structure holding the IP parameters for the
current subarray */
smf_find_subarray( data->hdr, subname, sizeof( subname ), NULL,
status );
datFind( loc, subname, &sloc, status );
/* Begin an NDF context. */
ndfBegin();
/* Get identifiers for the NDFs holding the individual parameters. Each
NDF holds a parameter value for each bolometer. */
ndfFind( sloc, "P0", &p0ndf, status );
ndfFind( sloc, "P1", &p1ndf, status );
ndfFind( sloc, "C0", &c0ndf, status );
ndfFind( sloc, "ANGC", &angcndf, status );
/* Map them. Check each one has the expected number of elements. */
ndfMap( p0ndf, "DATA", "_DOUBLE", "READ", (void **) &p0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( p1ndf, "DATA", "_DOUBLE", "READ", (void **) &p1data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", p1ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( c0ndf, "DATA", "_DOUBLE", "READ", (void **) &c0data,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", c0ndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
ndfMap( angcndf, "DATA", "_DOUBLE", "READ", (void **) &angcdata,
&nmap, status );
if( nmap != (int) nbolo && *status == SAI__OK ) {
*status = SAI__ERROR;
ndfMsg( "N", angcndf );
errRep( "", "smf_subip: Bad dimensions for ^N - should be 32x40.", status );
}
/* Get a pointer to the quality array for the residuals. */
qua_data = smf_select_qualpntr( data, NULL, status );
/* See how many bolometers to process in each thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Create jobs to apply the IP correction to a range of bolometers. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
/* Set the range of bolometers (b1 to b2) to be processed by the current
job. */
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store the other info needed by the worker thread. */
pdata->ntslice = ntslice;
pdata->nbolo = nbolo;
pdata->res_data = res->sdata[idx]->pntr[0];
pdata->lut_data = lut->sdata[idx]->pntr[0];
pdata->qua_data = qua_data;
pdata->ipang = ipang;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->imapdata = imapdata;
pdata->qu = qu;
pdata->p0data = p0data;
pdata->p1data = p1data;
pdata->c0data = c0data;
pdata->angcdata = angcdata;
pdata->allstate = data->hdr->allState;
/* Submit the job for execution by the next available thread. */
thrAddJob( wf, 0, pdata, smf1_subip, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* End the NDF context, thus unmapping and freeing all NDF identifiers
created since the context was started. */
ndfEnd( status );
/* Free locator for subarray IP parameters. */
datAnnul( &sloc, status );
ipang = astFree( ipang );
}
/* Free resources. */
datAnnul( &loc, status );
imapdata = astFree( imapdata );
job_data = astFree( job_data );
}
}
void smf_addcom( ThrWorkForce *wf, smfData *data, const double *com,
int *status ) {
/* Local Variables */
dim_t nbolo;
dim_t ntslice;
dim_t step;
int iw;
int nw;
smfAddComData *job_data;
smfAddComData *pdata;
/* Check the inherited status. */
if( *status != SAI__OK || !com ) return;
/* Check supplied smfData is time ordered (i.e. bstride=1, tstride=nbolo). */
if( !data->isTordered ) {
*status = SAI__ERROR;
errRep( " ", "smf_addcom: Supplied smfData is not time-ordered.",
status );
}
/* Note the number of time slices and bolometers. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL,
status );
/* Store the number of workers in the work force. */
nw = wf ? wf->nworker : 1;
/* Allocate job data for threads */
job_data = astMalloc( nw*sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Get the number of time slices to process in each thread. */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
/* Set up the job data for each thread. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
/* The first and last time slice to be processed by the thread. */
pdata->t1 = iw*step;
if( iw < nw - 1 ) {
pdata->t2 = ( iw + 1 )*step - 1;
} else {
pdata->t2 = ntslice - 1;
}
/* Pointer to the first output data element for the first time slice. */
pdata->out = ( (double *) data->pntr[0] ) + pdata->t1*nbolo;
/* Pointer to the first COM data element. */
pdata->com = com + pdata->t1;
/* Number of bolometers. */
pdata->nbolo = nbolo;
}
/* Add each job to the job queue. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
(void) thrAddJob( wf, 0, pdata, smf1AddCom, 0, NULL, status );
}
/* Wait until all of the jobs have completed */
thrWait( wf, status );
}
/* Free resources. */
job_data = astFree( job_data );
}
/* Returns an array holding the angle (rad.s) from north to focal plane Y,
measured positive in the sense of rotation from focal plane Y to focal
plane X, for every bolometer sample in a smfData. The values are bolo
ordered so that "bstride" is 1 and "tstsride" is nbolo. The returned
array should be freed using astFre when no longer needed. */
static double *smf1_calcang( smfData *data, int *status ){
/* Local Variables: */
AstFrameSet *fpfset;
AstFrameSet *wcs;
AstMapping *g2s;
AstMapping *s2f;
const char *usesys;
dim_t ibolo;
dim_t itime;
dim_t nbolo;
dim_t ncol;
dim_t ntslice;
double *fx2;
double *fx;
double *fy2;
double *fy;
double *gx;
double *gy;
double *pr;
double *result;
double *sx;
double *sy;
int subsysnum;
/* Check the inherited status. */
if( *status != SAI__OK ) return NULL;
/* Get the number of bolometers and time slices, together with the strides
between adjacent bolometers and adjacent time slices. */
smf_get_dims( data, NULL, &ncol, &nbolo, &ntslice, NULL, NULL, NULL,
status );
/* Allocate the returned array. */
result = astMalloc( nbolo*ntslice*sizeof( *result ) );
/* Allocate arrays to hold the grid coords for every bolometer. */
gx = astMalloc( nbolo*sizeof( *gx ) );
gy = astMalloc( nbolo*sizeof( *gy ) );
/* Allocate arrays to hold the sky coords for every bolometer. */
sx = astMalloc( nbolo*sizeof( *sx ) );
sy = astMalloc( nbolo*sizeof( *sy ) );
/* Allocate arrays to hold the focal plane coords for every bolometer. */
fx = astMalloc( nbolo*sizeof( *fx ) );
fy = astMalloc( nbolo*sizeof( *fy ) );
/* Allocate arrays to hold the focal plane coords of a point slightly to
the north of every bolometer. */
fx2 = astMalloc( nbolo*sizeof( *fx2 ) );
fy2 = astMalloc( nbolo*sizeof( *fy2 ) );
/* Get the AST code equivalent to the tracking system. */
usesys = sc2ast_convert_system( (data->hdr->allState)[0].tcs_tr_sys, status );
if( *status == SAI__OK ) {
/* Initialise the arrays holding the grid coords for every bolometer. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
gx[ ibolo ] = ibolo % ncol + 1;
gy[ ibolo ] = ibolo / ncol + 1;
}
/* Get the GRID->focal plane FrameSet (the same for every time slice). */
smf_find_subarray( data->hdr, NULL, 0, &subsysnum, status );
sc2ast_createwcs( subsysnum, NULL, data->hdr->instap, data->hdr->telpos,
NO_FTS, &fpfset, status);
/* Use this to transform the bolometrer GRID coords to focal plane. */
astTran2( fpfset, nbolo, gx, gy, 1, fx, fy );
/* Loop over all time slices. */
pr = result;
for( itime = 0; itime < ntslice; itime++ ) {
/* Get the WCS FrameSet for the time slice, and set its current Frame to the tracking
frame. */
smf_tslice_ast( data, itime, 1, NO_FTS, status );
wcs = data->hdr->wcs;
if( wcs ) {
astSetC( wcs, "System", usesys );
/* Get the mapping from GRID to SKY. */
astBegin;
g2s = astSimplify( astGetMapping( wcs, AST__BASE, AST__CURRENT ));
/* Get the mapping from SKY to focal plane (x,y) (the index of the FPLANE
Frame is fixed at 3 by file sc2ast.c). */
s2f = astSimplify( astGetMapping( wcs, AST__CURRENT, 3 ) );
/* Transform the grid coords of all bolometers to SKY coordinates using the FrameSet. */
astTran2( g2s, nbolo, gx, gy, 1, sx, sy );
/* Increment the sky positions slightly to the north. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) sy[ ibolo ] += 1.0E-6;
/* Transform these modified sky coordinates to focal plane. */
astTran2( s2f, nbolo, sx, sy, 1, fx2, fy2 );
astEnd;
/* Loop round all bolometers. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
/* Get the angle from north to focal plane Y, measured positive in the
sense of rotation from focal plane Y to focal plane X. */
if( fx[ibolo] != VAL__BADD && fy[ibolo] != VAL__BADD &&
fx2[ibolo] != VAL__BADD && fy2[ibolo] != VAL__BADD ) {
*(pr++) = atan2( fx[ibolo] - fx2[ibolo], fy2[ibolo] - fy[ibolo] );
} else {
*(pr++) = VAL__BADD;
}
}
} else {
for( ibolo = 0; ibolo < nbolo; ibolo++ ) *(pr++) = VAL__BADD;
}
}
}
/* Free resources. */
fx = astFree( fx );
fy = astFree( fy );
fx2 = astFree( fx2 );
fy2 = astFree( fy2 );
sx = astFree( sx );
sy = astFree( sy );
gx = astFree( gx );
gy = astFree( gy );
/* Return the array of angle values. */
return result;
}
void smf_uncalc_iqu( ThrWorkForce *wf, smfData *data,
double *idata, double *qdata, double *udata,
int *status ){
/* Local Variables: */
const JCMTState *state; /* JCMTState info for current time slice */
dim_t nbolo; /* No. of bolometers */
dim_t ntslice; /* Number of time-slices in data */
int bstep; /* Bolometer step between threads */
int itime; /* Time slice index */
int iworker; /* Index of a worker thread */
int ntime; /* Time slices to check */
int nworker; /* No. of worker threads */
int old; /* Data has old-style POL_ANG values? */
size_t bstride; /* Stride between adjacent bolometer values */
size_t tstride; /* Stride between adjacent time slice values */
smfHead *hdr; /* Pointer to data header this time slice */
smfUncalcIQUJobData *job_data = NULL; /* Pointer to all job data */
smfUncalcIQUJobData *pdata = NULL;/* Pointer to next job data */
char headval[ 81 ]; /* FITS header value */
int ipolcrd; /* Reference direction for waveplate angles */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Convenience pointer. */
hdr = data->hdr;
/* Check the half-waveplate and analyser were in the beam. */
headval[ 0 ] = 0;
smf_getfitss( hdr, "POLWAVIN", headval, sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
smf_smfFile_msg( data->file, "N", 0, "" );
*status = SAI__ERROR;
errRep( " ", "Half-waveplate was not in the beam for "
"input NDF ^N.", status );
}
headval[ 0 ] = 0;
smf_getfitss( hdr, "POLANLIN", headval, sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
smf_smfFile_msg( data->file, "N", 0, "" );
*status = SAI__ERROR;
errRep( " ", "Analyser was not in the beam for input "
"NDF ^N.", status );
}
/* Get the reference direction for JCMTSTATE:POL_ANG values. */
smf_getfitss( hdr, "POL_CRD", headval, sizeof(headval), status );
if( !strcmp( headval, "FPLANE" ) ) {
ipolcrd = 0;
} else if( !strcmp( headval, "AZEL" ) ) {
ipolcrd = 1;
} else if( !strcmp( headval, "TRACKING" ) ) {
ipolcrd = 2;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "N", 0, "" );
msgSetc( "V", headval );
errRep( " ", "Input NDF ^N contains unknown value "
"'^V' for FITS header 'POL_CRD'.", status );
}
/* Obtain number of time slices - will also check for 3d-ness. Also get
the dimensions of the bolometer array and the strides between adjacent
bolometer values. */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, &bstride,
&tstride, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Check the above pointers can be used safely. */
if( *status == SAI__OK ) {
/* Go through the first thousand POL_ANG values to see if they are in
units of radians (new data) or arbitrary encoder units (old data).
They are assumed to be in radians if no POL_ANG value is larger than
20. */
old = 0;
state = hdr->allState;
ntime = ( ntslice > 1000 ) ? 1000 : ntslice;
for( itime = 0; itime < ntime; itime++,state++ ) {
if( state->pol_ang > 20 ) {
old = 1;
msgOutif( MSG__VERB, ""," POL2 data contains POL_ANG values "
"in encoder units - converting to radians.", status );
break;
}
}
/* Determine which bolometers are to be processed by which threads. */
bstep = nbolo/nworker;
if( bstep < 1 ) bstep = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*bstep;
pdata->b2 = pdata->b1 + bstep - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to calculate the analysed intensity values in each bolo, and then
wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->bstride = bstride;
pdata->nbolo = nbolo;
pdata->tstride = tstride;
pdata->allstates = hdr->allState;
pdata->ipi = idata;
pdata->ipq = qdata;
pdata->ipu = udata;
pdata->ipolcrd = ipolcrd;
pdata->old = old;
pdata->ntslice = ntslice;
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_uncalc_iqu_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
}
/* Free resources. */
job_data = astFree( job_data );
}
void smf_write_flagmap( ThrWorkForce *wf, smf_qual_t mask, smfArray *lut, smfArray *qua,
smfDIMMData *dat, const Grp *flagrootgrp,
size_t contchunk, const int *lbnd_out,
const int *ubnd_out, AstFrameSet *outfset,
int *status ) {
AstFrameSet *tfset; /* Temporary FrameSet pointer */
Grp *mgrp=NULL; /* Temporary group for map names */
char *pname=NULL; /* Poiner to name */
char name[GRP__SZNAM+1]; /* Buffer for storing names */
char tempstr[20]; /* Temporary string */
char tmpname[GRP__SZNAM+1]; /* temp name buffer */
dim_t nbolo; /* Number of bolometers */
dim_t ntslice; /* Number of time slices */
double shift[ 1 ]; /* Shift from GRID to bit number */
int *flagmap=NULL; /* pointer to flagmap data */
int *lut_data=NULL; /* Pointer to DATA component of lut */
int ibit; /* Quality bit number */
int lbnd3d[3]; /* Lower bounds for 3D output */
int npix; /* Number of pixels per plane */
int target; /* Target value for incrementing pixel count */
int ubnd3d[3]; /* Upper bounds for 3D output */
size_t bstride; /* Bolometer stride */
size_t i; /* loop counter */
size_t idx=0; /* index within subgroup */
size_t ii; /* array offset index */
size_t j; /* loop counter */
size_t tstride; /* Time stride */
smfData *mapdata=NULL; /* smfData for new map */
smf_qual_t *qua_data=NULL; /* Pointer to DATA component of qua */
if( *status != SAI__OK ) return;
if( !lut || !qua || !dat || !flagrootgrp || !lbnd_out || !ubnd_out ||
!outfset ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL inputs supplied", status );
return;
}
/* Create a name for the flagmap, taking into account the chunk
number. Only required if we are using a single output
container. */
pname = tmpname;
grpGet( flagrootgrp, 1, 1, &pname, sizeof(tmpname), status );
one_strlcpy( name, tmpname, sizeof(name), status );
one_strlcat( name, ".", sizeof(name), status );
sprintf(tempstr, "CH%02zd", contchunk);
one_strlcat( name, tempstr, sizeof(name), status );
mgrp = grpNew( "flagmap", status );
grpPut1( mgrp, name, 0, status );
msgOutf( "", "*** Writing flagmap %s", status, name );
/* If a non-zero mask value wassupplied, the flagmap is 2-dimensional
and each pixel value counts the number of samples flagged by any of
the qualities included in the mask. */
if( mask ) {
smf_open_newfile( wf, mgrp, 1, SMF__INTEGER, 2, lbnd_out, ubnd_out, 0, &mapdata,
status);
flagmap = mapdata->pntr[0];
/* Loop over subgroup index (subarray) */
for( idx=0; (idx<qua->ndat)&&(*status==SAI__OK); idx++ ) {
smf_get_dims( qua->sdata[idx], NULL, NULL, &nbolo, &ntslice,
NULL, &bstride, &tstride, status );
qua_data = (qua->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
/* Loop over bolometer and time slice and create map */
for( i=0; i<nbolo; i++ ) {
/* Skip bolometers only if SMF__Q_BADB is set both in the
data and the mask */
if( !(qua_data[i*bstride] & mask & SMF__Q_BADB) ) {
for( j=0; j<ntslice; j++ ) {
ii = i*bstride + j*tstride;
if( (qua_data[ii] & mask) && (lut_data[ii] != VAL__BADI) ) {
flagmap[lut_data[ii]]++;
}
}
}
}
}
/* Write WCS */
smf_set_moving( (AstFrame *) outfset, NULL, status );
ndfPtwcs( outfset, mapdata->file->ndfid, status );
/* If the mask is zero, the flagmap is 3-dimensional and contains a
plane for each quality bit, plus an additional plane (plane 1)
containing the number of unflagged samples in each pixel. */
} else {
lbnd3d[ 0 ] = lbnd_out[ 0 ];
lbnd3d[ 1 ] = lbnd_out[ 1 ];
lbnd3d[ 2 ] = -1;
ubnd3d[ 0 ] = ubnd_out[ 0 ];
ubnd3d[ 1 ] = ubnd_out[ 1 ];
ubnd3d[ 2 ] = SMF__NQBITS_TSERIES - 1;
smf_open_newfile( wf, mgrp, 1, SMF__INTEGER, 3, lbnd3d, ubnd3d, 0,
&mapdata, status);
flagmap = mapdata->pntr[0];
/* No. of pixels in one plane */
npix = ( ubnd3d[ 1 ] - lbnd3d[ 1 ] + 1 )*( ubnd3d[ 0 ] - lbnd3d[ 0 ] + 1 );
/* Loop over each quality bit (-1 == "no flags"). */
for( ibit = -1; ibit < SMF__NQBITS_TSERIES; ibit++ ) {
/* The test of each sample is performed by checking if the
sample's quality value ANDed with "mask" is equal to "target".
This is requires since ibit==-1 (i.e. "count all samples that
have no flags set") requires a different logic to the other
ibit values. */
if( ibit == -1 ) {
mask = SMF__Q_GOOD;
target = 0;
} else {
mask = BIT_TO_VAL(ibit);
target = mask;
}
/* Loop over subgroup index (subarray) */
for( idx=0; (idx<qua->ndat)&&(*status==SAI__OK); idx++ ) {
smf_get_dims( qua->sdata[idx], NULL, NULL, &nbolo, &ntslice,
NULL, &bstride, &tstride, status );
qua_data = (qua->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
/* Loop over bolometer and time slice and create map */
for( i=0; i<nbolo; i++ ) {
/* Skip bolometers only if SMF__Q_BADB is set both in the
data and the mask */
for( j=0; j<ntslice; j++ ) {
ii = i*bstride + j*tstride;
if( ( (qua_data[ii] & mask) == target ) &&
(lut_data[ii] != VAL__BADI) ) {
flagmap[lut_data[ii]]++;
}
}
}
}
/* Move the pointer on to th enext plane. */
flagmap += npix;
/* Next quality bit. */
}
/* Take a copy of the supplied FrameSet so we do not modify it. */
tfset = astCopy( outfset );
/* Set atributes for moving target if necessary. */
smf_set_moving( (AstFrame *) tfset, NULL, status );
/* Modify the WCS FrameSet so that the base and current Frames are
3-dimensional. The current Frame is expanded by adding in a simple
1D Frame representing quality bit, and the base Frame is expanded
by adding in a 3rd GRID axis. Other Frames are left unchanged.
The quality bit Frame and the new GRID axis are connected using
a ShiftMap that gives the right zero-based bit numbers (which
also correspond to PIXEL indices). */
shift[ 0 ] = -2.0;
atlAddWcsAxis( tfset, (AstMapping *) astShiftMap( 1, shift, " " ),
astFrame( 1, "Label(1)=Quality bit,Domain=QUALITY" ),
NULL, NULL, status );
/* Store the FrameSet in the 3D NDF. */
ndfPtwcs( tfset, mapdata->file->ndfid, status );
tfset = astAnnul( tfset );
}
/* Clean up */
if( mgrp ) grpDelet( &mgrp, status );
smf_close_file( wf, &mapdata, status );
}
void smf_calc_mapcoord( ThrWorkForce *wf, AstKeyMap *config, smfData *data,
AstFrameSet *outfset, int moving, int *lbnd_out,
int *ubnd_out, fts2Port fts_port, int flags,
int *status ) {
/* Local Variables */
AstSkyFrame *abskyfrm = NULL;/* Output SkyFrame (always absolute) */
AstMapping *bolo2map=NULL; /* Combined mapping bolo->map coordinates */
int bndndf=NDF__NOID; /* NDF identifier for map bounds */
void *data_pntr[1]; /* Array of pointers to mapped arrays in ndf */
int *data_index; /* Mapped DATA_ARRAY part of NDF */
int docalc=1; /* If set calculate the LUT */
int doextension=0; /* Try to write LUT to MAPCOORD extension */
smfFile *file=NULL; /* smfFile pointer */
AstObject *fstemp = NULL; /* AstObject version of outfset */
int ii; /* loop counter */
int indf_lat = NDF__NOID; /* Identifier for NDF to receive lat values */
int indf_lon = NDF__NOID; /* Identifier for NDF to receive lon values */
smfCalcMapcoordData *job_data=NULL; /* Array of job */
int lbnd[1]; /* Pixel bounds for 1d pointing array */
int lbnd_old[2]; /* Pixel bounds for existing LUT */
int lbnd_temp[1]; /* Bounds for bounds NDF component */
int lutndf=NDF__NOID; /* NDF identifier for coordinates */
AstMapping *map2sky_old=NULL;/* Existing mapping map->celestial coord. */
HDSLoc *mapcoordloc=NULL; /* HDS locator to the MAPCOORD extension */
int nw; /* Number of worker threads */
AstFrameSet *oldfset=NULL; /* Pointer to existing WCS info */
AstSkyFrame *oskyfrm = NULL; /* SkyFrame from the output WCS Frameset */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
double *lat_ptr = NULL; /* Pointer to array to receive lat values */
double *lon_ptr = NULL; /* Pointer to array to receive lon values */
int ubnd[1]; /* Pixel bounds for 1d pointing array */
int ubnd_old[2]; /* Pixel bounds for existing LUT */
int ubnd_temp[1]; /* Bounds for bounds NDF component */
int *lut = NULL; /* The lookup table */
dim_t nbolo=0; /* Number of bolometers */
dim_t ntslice=0; /* Number of time slices */
int nmap; /* Number of mapped elements */
AstMapping *sky2map=NULL; /* Mapping celestial->map coordinates */
size_t step; /* step size for dividing up work */
AstCmpMap *testcmpmap=NULL; /* Combined forward/inverse mapping */
AstMapping *testsimpmap=NULL;/* Simplified testcmpmap */
double *theta = NULL; /* Scan direction at each time slice */
int tstep; /* Time slices between full Mapping calculations */
int exportlonlat; /* Dump longitude and latitude values? */
/* Main routine */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Initialize bounds to avoid compiler warnings */
lbnd_old[0] = 0;
lbnd_old[1] = 0;
ubnd_old[0] = 0;
ubnd_old[1] = 0;
/* Check for pre-existing LUT and de-allocate it. This will only waste
time if the MAPCOORD extension is found to be valid and it has
to be re-loaded from disk. */
smf_close_mapcoord( data, status );
/* Assert ICD data order */
smf_dataOrder( data, 1, status );
/* Get the data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* If SMF__NOCREATE_FILE is not set, and file associated with an NDF,
map a new MAPCOORD extension (or verify an existing one) */
if( !(flags & SMF__NOCREATE_FILE) && data->file ) {
doextension = 1;
} else {
doextension = 0;
docalc = 1;
}
/* Create / check for existing MAPCOORD extension */
if( doextension ) {
file = data->file;
/* Check type of file before proceeding */
if( file->isSc2store ) {
*status = SAI__ERROR;
errRep(FUNC_NAME,
"File was opened by sc2store library (raw data?)",
status);
}
if( !file->isTstream ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "File does not contain time stream data",status);
}
/* Get HDS locator to the MAPCOORD extension */
mapcoordloc = smf_get_xloc( data, "MAPCOORD", "MAP_PROJECTION", "UPDATE",
0, 0, status );
/* Obtain NDF identifier/placeholder for LUT in MAPCOORD extension*/
lbnd[0] = 0;
ubnd[0] = nbolo*ntslice-1;
lutndf = smf_get_ndfid( mapcoordloc, "LUT", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd, ubnd, status );
if( *status == SAI__OK ) {
/* store the NDF identifier */
file->mapcoordid = lutndf;
/* Create sky to output grid mapping using the base coordinates to
get the coordinates of the tangent point if it hasn't been done
yet. */
sky2map = astGetMapping( outfset, AST__CURRENT, AST__BASE );
}
/* Before mapping the LUT, first check for existing WCS information
and LBND/UBND for the output map. If they are already correct don't
bother re-calculating the LUT! */
if( *status == SAI__OK ) {
/* Try reading in the WCS information */
kpg1Wread( mapcoordloc, "WCS", &fstemp, status );
oldfset = (AstFrameSet*)fstemp;
if( *status == SAI__OK ) {
/* Check that the old and new mappings are the same by
checking that combining one with the inverse of the other
reduces to a UnitMap. */
map2sky_old = astGetMapping( oldfset, AST__BASE, AST__CURRENT );
testcmpmap = astCmpMap( map2sky_old, sky2map, 1, " " );
testsimpmap = astSimplify( testcmpmap );
if( astIsAUnitMap( testsimpmap ) ) {
/* The mappings are the same, now just check the pixel
bounds in the output map */
lbnd_temp[0] = 1;
ubnd_temp[0] = 2;
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lbnd_old[0] = data_index[0];
lbnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
ubnd_old[0] = data_index[0];
ubnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
if( *status == SAI__OK ) {
/* If we get this far finally do the bounds check! */
if( (lbnd_old[0] == lbnd_out[0]) &&
(lbnd_old[1] == lbnd_out[1]) &&
(ubnd_old[0] == ubnd_out[0]) &&
(ubnd_old[1] == ubnd_out[1]) ) {
docalc = 0; /* We don't have to re-calculate the LUT */
msgOutif(MSG__VERB," ",FUNC_NAME ": Existing LUT OK",
status);
}
}
}
/* Bad status / AST errors at this point due to problems with
MAPCOORD. Annul and continue calculating new MAPCOORD extension. */
astClearStatus;
errAnnul(status);
} else {
/* Bad status due to non-existence of MAPCOORD. Annul and continue */
errAnnul(status);
}
}
}
/* If we need to calculate the LUT do it here */
if( docalc && (*status == SAI__OK) ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": Calculate new LUT",
status);
/* Get the increment in time slices between full Mapping calculations.
The Mapping for intermediate time slices will be approximated. */
dim_t dimval;
smf_get_nsamp( config, "TSTEP", data, &dimval, status );
tstep = dimval;
/* Get space for the LUT */
if( doextension ) {
/* Map the LUT array */
ndfMap( lutndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lut = data_index;
} else {
errRep( FUNC_NAME, "Unable to map LUT in MAPCOORD extension",
status);
}
} else {
/* alloc the LUT and THETA arrays */
lut = astMalloc( (nbolo*ntslice)*sizeof(*(data->lut)) );
theta = astMalloc( ntslice*sizeof(*(data->theta)) );
}
/* Retrieve the sky2map mapping from the output frameset (actually
map2sky) */
oskyfrm = astGetFrame( outfset, AST__CURRENT );
sky2map = astGetMapping( outfset, AST__BASE, AST__CURRENT );
/* If the longitude and latitude is being dumped, create new NDFs to
hold them, and map them. */
if( config ) {
astMapGet0I( config, "EXPORTLONLAT", &exportlonlat );
if( exportlonlat ) {
lon_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lon, 1, status );
lat_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lat, 2, status );
}
}
/* Invert the mapping to get Output SKY to output map coordinates */
astInvert( sky2map );
/* Create a SkyFrame in absolute coordinates */
abskyfrm = astCopy( oskyfrm );
astClear( abskyfrm, "SkyRefIs" );
astClear( abskyfrm, "SkyRef(1)" );
astClear( abskyfrm, "SkyRef(2)" );
if( *status == SAI__OK ) {
/* --- Begin parellelized portion ------------------------------------ */
/* Start a new job context. Each call to thrWait within this
context will wait until all jobs created within the context have
completed. Jobs created in higher contexts are ignored by thrWait. */
thrBeginJobContext( wf, status );
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Set up job data, and start calculating pointing for blocks of
time slices in different threads */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step;
pdata->t2 = (ii+1)*step-1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) && (pdata->t1<(ntslice-1)) ) {
pdata->t2=ntslice-1;
}
pdata->ijob = -1;
pdata->lut = lut;
pdata->theta = theta;
pdata->lbnd_out = lbnd_out;
pdata->moving = moving;
pdata->ubnd_out = ubnd_out;
pdata->tstep = tstep;
pdata->lat_ptr = lat_ptr;
pdata->lon_ptr = lon_ptr;
pdata->fts_port = fts_port;
/* Make deep copies of AST objects and unlock them so that each
thread can then lock them for their own exclusive use */
pdata->abskyfrm = astCopy( abskyfrm );
astUnlock( pdata->abskyfrm, 1 );
pdata->sky2map = astCopy( sky2map );
astUnlock( pdata->sky2map, 1 );
/* Similarly, make a copy of the smfData, including only the header
information which each thread will need in order to make calls to
smf_rebin_totmap */
pdata->data = smf_deepcopy_smfData( data, 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( pdata->data, 0, status );
}
for( ii=0; ii<nw; ii++ ) {
/* Submit the job */
pdata = job_data + ii;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfCalcMapcoordPar, 0, NULL, status );
}
/* Wait until all of the jobs submitted within the current job
context have completed */
thrWait( wf, status );
}
/* End the current job context. */
thrEndJobContext( wf, status );
/* --- End parellelized portion -------------------------------------- */
/* Set the lut pointer in data to the buffer */
data->lut = lut;
data->theta = theta;
/* Write the WCS for the projection to the extension */
if( doextension ) {
kpg1Wwrt( (AstObject*)outfset, "WCS", mapcoordloc, status );
/* Write the pixel bounds for the map to the extension */
lbnd_temp[0] = 1; /* Don't get confused! Bounds for NDF that will */
ubnd_temp[0] = 2; /* contain the bounds for the output 2d map! */
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = lbnd_out[0];
data_index[1] = lbnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map LBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = ubnd_out[0];
data_index[1] = ubnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map UBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
}
}
}
/* Clean Up */
if( testsimpmap ) testsimpmap = astAnnul( testsimpmap );
if( testcmpmap ) testcmpmap = astAnnul( testcmpmap );
if( map2sky_old ) map2sky_old = astAnnul( map2sky_old );
if( oldfset ) oldfset = astAnnul( oldfset );
if (sky2map) sky2map = astAnnul( sky2map );
if (bolo2map) bolo2map = astAnnul( bolo2map );
if( abskyfrm ) abskyfrm = astAnnul( abskyfrm );
if( oskyfrm ) oskyfrm = astAnnul( oskyfrm );
if( mapcoordloc ) datAnnul( &mapcoordloc, status );
if( indf_lat != NDF__NOID ) ndfAnnul( &indf_lat, status );
if( indf_lon != NDF__NOID ) ndfAnnul( &indf_lon, status );
/* If we get this far, docalc=0, and status is OK, there must be
a good LUT in there already. Map it so that it is accessible to
the caller; "UPDATE" so that the caller can modify it if desired. */
if( (*status == SAI__OK) && (docalc == 0) ) {
smf_open_mapcoord( data, "UPDATE", status );
}
/* Clean up job data */
if( job_data ) {
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
if( pdata->data ) {
smf_lock_data( pdata->data, 1, status );
smf_close_file( &(pdata->data), status );
}
astLock( pdata->abskyfrm, 0 );
pdata->abskyfrm = astAnnul( pdata->abskyfrm );
astLock( pdata->sky2map, 0 );
pdata->sky2map = astAnnul( pdata->sky2map );
}
job_data = astFree( job_data );
}
}
void
smf_flat_params( const smfData * refdata, const char resistpar[],
const char methpar[], const char orderpar[], const char snrminpar[],
double * refohms, double **resistance, int * outrows,
int * outcols, smf_flatmeth *flatmeth,
int * order, double * snrmin, smfData ** heateff,
int * status ) {
dim_t datarows = 0; /* Number of rows in refdata */
dim_t datacols = 0; /* Number of columns in refdata */
size_t j = 0; /* Counter, index */
char method[SC2STORE_FLATLEN]; /* flatfield method string */
size_t nbols; /* Number of bolometers */
double refohmsval = 0.0; /* Internal version of refohms */
AstKeyMap * resmap = NULL; /* Resistor map */
AstKeyMap * subarrays = NULL; /* Subarray lookup table */
char thissub[32]; /* This sub-instrument string */
if (resistance) *resistance = NULL;
if (*status != SAI__OK) return;
if (!refdata) {
*status = SAI__ERROR;
errRep( "", "Must provide reference data file to calculate flatfield parameters"
" (possible programming error)", status );
return;
}
/* Based on refdata we now need to calculate the default reference
resistance and retrieve the correct heater efficiency file for each array.
We need the unique subarray string so that we can set up a look up keymap.
There is no code in SMURF to return all the known subarrays but
we need to know all the options in order to use kpg1Config. */
subarrays = astKeyMap( " " );
astMapPut0I( subarrays, "CG450MK2_M0907D0501", 0, NULL );
astMapPut0I( subarrays, "CG850MK2_M0904D0503", 0, NULL );
astMapPut0I( subarrays, "SG850_M0906D1005", 0, NULL );
astMapPut0I( subarrays, "SG850_M1002D1006", 0, NULL );
astMapPut0I( subarrays, "SG850_M1005D1007", 0, NULL );
astMapPut0I( subarrays, "SG850_M1003D1004", 0, NULL );
astMapPut0I( subarrays, "SG450_M1004D1000", 0, NULL );
astMapPut0I( subarrays, "SG450_M1007D1002", 0, NULL );
astMapPut0I( subarrays, "SG450_M1006D1003", 0, NULL );
astMapPut0I( subarrays, "SG450_M1009D1008", 0, NULL );
/* and indicate which subarray we are interested in (uppercased) */
smf_fits_getS( refdata->hdr, "ARRAYID", thissub, sizeof(thissub), status );
{ /* need to uppercase */
size_t l = strlen(thissub);
for (j=0;j<l;j++) {
thissub[j] = toupper(thissub[j]);
}
}
astMapPut0I( subarrays, thissub, 1, NULL );
/* Read the config file */
resmap = kpg1Config( resistpar, "$SMURF_DIR/smurf_calcflat.def",
subarrays, 1, status );
subarrays = astAnnul( subarrays );
if (*status != SAI__OK) goto CLEANUP;
/* Read the reference resistance */
astMapGet0D( resmap, "REFRES", &refohmsval );
if (refohms && *status == SAI__OK) {
*refohms = refohmsval;
msgOutiff(MSG__VERB, "",
"Read reference resistance for subarray %s of %g ohms\n",
status, thissub, *refohms );
}
/* We no longer want to read per-bolometer resistor values from the
config file. To retain backwards compatibility with the current
implementation of smf_flat_standardpow we simply fill the
per-bol resistance array with the reference resistance which
effectively disables smf_flat_standardpow */
smf_get_dims( refdata, &datarows, &datacols, NULL, NULL, NULL, NULL, NULL, status );
nbols = datacols * datarows;
if (*status == SAI__OK && resistance ) {
*resistance = astMalloc( nbols*sizeof(**resistance) );
for (j = 0; j < (size_t)nbols; j++) {
(*resistance)[j] = refohmsval;
}
}
/* Get the heater efficiency file */
if (heateff && astMapHasKey( resmap, "HEATEFF" ) ) {
const char * heateffstr = NULL;
if (astMapGet0C( resmap, "HEATEFF", &heateffstr )) {
Grp * heateffgrp = NULL;
smfData * heatefftmp = NULL;
heateffgrp = grpNew( "heateff", status );
grpPut1( heateffgrp, heateffstr, 0, status );
smf_open_file( NULL, heateffgrp, 1, "READ", SMF__NOTTSERIES|SMF__NOFIX_METADATA, &heatefftmp, status );
/* Divorce the smfData from the underlying file. This file stays open for the entire
duration of the data processing and can some times lead to issues when we attempt
to close it an hour after we opened it (it's usually on an NFS disk) */
if (*status == SAI__OK) {
*heateff = smf_deepcopy_smfData( NULL, heatefftmp, 0,
SMF__NOCREATE_FILE | SMF__NOCREATE_FTS |
SMF__NOCREATE_DA,
0, 0, status );
smf_close_file(NULL, &heatefftmp, status);
}
/* Check the dimensions */
if (*status == SAI__OK) {
dim_t heatrows = 0;
dim_t heatcols = 0;
smf_get_dims( *heateff, &heatrows, &heatcols, NULL, NULL, NULL, NULL, NULL, status );
if (*status == SAI__OK) {
if ( datarows != heatrows || datacols != heatcols ) {
*status = SAI__ERROR;
errRepf( "", "Dimensions of heater efficiency file %s are (%zu, %zu)"
" but flatfield has dimensions (%zu, %zu)",
status, heateffstr, (size_t)heatrows, (size_t)heatcols,
(size_t)datarows, (size_t)datacols);
}
}
if (*status == SAI__OK) {
smf_dtype_check_fatal( *heateff, NULL, SMF__DOUBLE, status );
if (*status == SMF__BDTYP) {
errRepf("", "Heater efficiency data in %s should be double precision",
status, heateffstr);
}
}
if (*status == SAI__OK) {
char heateffarrid[32];
smf_fits_getS( refdata->hdr, "ARRAYID", heateffarrid, sizeof(heateffarrid), status );
if (*status != SAI__OK) errAnnul( status );
if (strcasecmp( thissub, heateffarrid ) != 0 ) {
if (*status == SAI__OK) {
*status = SAI__ERROR;
errRepf("", "Subarray associated with heater efficiency image (%s)"
" does not match that of the data to be flatfielded (%s)",
status, heateffarrid, thissub );
}
}
}
}
if (heateffgrp) grpDelet( &heateffgrp, status );
}
}
if (methpar && flatmeth) {
/* See if we want to use TABLE or POLYNOMIAL mode */
parChoic( methpar, "POLYNOMIAL", "POLYNOMIAL, TABLE", 1,
method, sizeof(method), status );
*flatmeth = smf_flat_methcode( method, status );
if (*flatmeth == SMF__FLATMETH_POLY) {
/* need an order for the polynomial */
if (order && orderpar) {
parGdr0i( orderpar, 1, 1, 3, 1, order, status );
/* and if the order is 1 then we can ask for the snr min */
if (snrminpar && *order == 1) {
parGet0d( snrminpar, snrmin, status );
}
}
} else {
/* need an snr min for table mode responsivities */
if (snrminpar) parGet0d( snrminpar, snrmin, status );
}
}
if (outrows) *outrows = datarows;
if (outcols) *outcols = datacols;
CLEANUP:
resmap = astAnnul( resmap );
if (*status != SAI__OK) {
if (resistance && *resistance) *resistance = astFree( *resistance );
if (heateff && *heateff) smf_close_file( NULL, heateff, status );
}
return;
}
void smurf_sc2fft( int *status ) {
int avpspec=0; /* Flag for doing average power spectrum */
double avpspecthresh=0; /* Threshold noise for detectors in avpspec */
Grp * basegrp = NULL; /* Basis group for output filenames */
smfArray *bbms = NULL; /* Bad bolometer masks */
smfArray *concat=NULL; /* Pointer to a smfArray */
size_t contchunk; /* Continuous chunk counter */
smfArray *darks = NULL; /* dark frames */
int ensureflat; /* Flag for flatfielding data */
Grp *fgrp = NULL; /* Filtered group, no darks */
smfArray *flatramps = NULL;/* Flatfield ramps */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
size_t gcount=0; /* Grp index counter */
size_t i; /* Loop counter */
smfGroup *igroup=NULL; /* smfGroup corresponding to igrp */
Grp *igrp = NULL; /* Input group of files */
int inverse=0; /* If set perform inverse transform */
int isfft=0; /* Are data fft or real space? */
dim_t maxconcat=0; /* Longest continuous chunk length in samples */
size_t ncontchunks=0; /* Number continuous chunks outside iter loop */
smfData *odata=NULL; /* Pointer to output smfData to be exported */
Grp *ogrp = NULL; /* Output group of files */
size_t outsize; /* Total number of NDF names in the output group */
int polar=0; /* Flag for FFT in polar coordinates */
int power=0; /* Flag for squaring amplitude coeffs */
size_t size; /* Number of files in input group */
smfData *tempdata=NULL; /* Temporary smfData pointer */
int weightavpspec=0; /* Flag for 1/noise^2 weighting */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
int zerobad; /* Zero VAL__BADD before taking FFT? */
/* Main routine */
ndfBegin();
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get input file(s) */
kpg1Rgndf( "IN", 0, 1, "", &igrp, &size, status );
/* Filter out darks */
smf_find_science( igrp, &fgrp, 1, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* input group is now the filtered group so we can use that and
free the old input group */
size = grpGrpsz( fgrp, status );
grpDelet( &igrp, status);
igrp = fgrp;
fgrp = NULL;
/* We now need to combine files from the same subarray and same sequence
to form a continuous time series */
smf_grp_related( igrp, size, 1, 0, 0, NULL, NULL, &maxconcat, NULL, &igroup,
&basegrp, NULL, status );
/* Get output file(s) */
size = grpGrpsz( basegrp, status );
if( size > 0 ) {
kpg1Wgndf( "OUT", basegrp, size, size, "More output files required...",
&ogrp, &outsize, status );
} else {
msgOutif(MSG__NORM, " ", TASK_NAME ": All supplied input frames were DARK,"
" nothing to do", status );
}
/* Get group of bolometer masks and read them into a smfArray */
smf_request_mask( "BBM", &bbms, status );
/* Obtain the number of continuous chunks and subarrays */
if( *status == SAI__OK ) {
ncontchunks = igroup->chunk[igroup->ngroups-1]+1;
}
msgOutiff( MSG__NORM, "", "Found %zu continuous chunk%s", status, ncontchunks,
(ncontchunks > 1 ? "s" : "") );
/* Are we flatfielding? */
parGet0l( "FLAT", &ensureflat, status );
/* Are we doing an inverse transform? */
parGet0l( "INVERSE", &inverse, status );
/* Are we using polar coordinates instead of cartesian for the FFT? */
parGet0l( "POLAR", &polar, status );
/* Are we going to assume amplitudes are squared? */
parGet0l( "POWER", &power, status );
/* Are we going to zero bad values first? */
parGet0l( "ZEROBAD", &zerobad, status );
/* Are we calculating the average power spectrum? */
parGet0l( "AVPSPEC", &avpspec, status );
if( avpspec ) {
power = 1;
parGet0d( "AVPSPECTHRESH", &avpspecthresh, status );
parGet0l( "WEIGHTAVPSPEC", &weightavpspec, status );
}
/* If power is true, we must be in polar form */
if( power && !polar) {
msgOutif( MSG__NORM, " ", TASK_NAME
": power spectrum requested so setting POLAR=TRUE", status );
polar = 1;
}
gcount = 1;
for( contchunk=0;(*status==SAI__OK)&&contchunk<ncontchunks; contchunk++ ) {
size_t idx;
/* Concatenate this continuous chunk but forcing a raw data read.
We will need quality. */
smf_concat_smfGroup( wf, NULL, igroup, darks, NULL, flatramps, heateffmap,
contchunk, ensureflat, 1, NULL, 0, NULL, NULL, 0, 0, 0,
&concat, NULL, status );
/* Now loop over each subarray */
/* Export concatenated data for each subarray to NDF file */
for( idx=0; (*status==SAI__OK)&&idx<concat->ndat; idx++ ) {
if( concat->sdata[idx] ) {
smfData * idata = concat->sdata[idx];
int provid = NDF__NOID;
dim_t nbolo; /* Number of detectors */
dim_t ndata; /* Number of data points */
/* Apply a mask to the quality array and data array */
smf_apply_mask( idata, bbms, SMF__BBM_QUAL|SMF__BBM_DATA, 0, status );
smf_get_dims( idata, NULL, NULL, &nbolo, NULL, &ndata, NULL, NULL,
status );
/* Check for double precision data */
if( idata->dtype != SMF__DOUBLE ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": data are not double precision.", status );
}
/* Are we zeroing VAL__BADD? */
if( (*status==SAI__OK) && zerobad ) {
double *data= (double *) idata->pntr[0];
for( i=0; i<ndata; i++ ) {
if( data[i] == VAL__BADD ) {
data[i] = 0;
}
}
}
/* Check whether we need to transform the data at all */
isfft = smf_isfft(idata,NULL,NULL,NULL,NULL,NULL,status);
if( isfft && avpspec && (*status == SAI__OK) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": to calculate average power spectrum input data cannot "
"be FFT", status );
}
if( (*status == SAI__OK) && (isfft == inverse) ) {
if( avpspec ) {
/* If calculating average power spectrum do the transforms with
smf_bolonoise so that we can also measure the noise of
each detector */
double *whitenoise=NULL;
smf_qual_t *bolomask=NULL;
double mean, sig, freqlo;
size_t ngood, newgood;
whitenoise = astCalloc( nbolo, sizeof(*whitenoise) );
bolomask = astCalloc( nbolo, sizeof(*bolomask) );
freqlo = 1. / (idata->hdr->steptime * idata->hdr->nframes);
smf_bolonoise( wf, idata, 1, freqlo, SMF__F_WHITELO,
SMF__F_WHITEHI, 1, 0, whitenoise, NULL, &odata,
status );
/* Initialize quality */
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] == VAL__BADD ) {
bolomask[i] = SMF__Q_BADB;
} else {
/* smf_bolonoise returns a variance, so take sqrt */
whitenoise[i] = sqrt(whitenoise[i]);
}
}
ngood=-1;
newgood=0;
/* Iteratively cut n-sigma noisy outlier detectors */
while( ngood != newgood ) {
ngood = newgood;
smf_stats1D( whitenoise, 1, nbolo, bolomask, 1, SMF__Q_BADB,
&mean, &sig, NULL, NULL, status );
msgOutiff( MSG__DEBUG, "", TASK_NAME
": mean=%lf sig=%lf ngood=%li\n", status,
mean, sig, ngood);
newgood=0;
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] != VAL__BADD ){
if( (whitenoise[i] - mean) > avpspecthresh *sig ) {
whitenoise[i] = VAL__BADD;
bolomask[i] = SMF__Q_BADB;
} else {
newgood++;
}
}
}
}
msgOutf( "", TASK_NAME
": Calculating average power spectrum of best %li "
" bolometers.", status, newgood);
/* If using 1/noise^2 weights, calculate 1/whitenoise^2 in-place
to avoid allocating another array */
if( weightavpspec ) {
msgOutif( MSG__VERB, "", TASK_NAME ": using 1/noise^2 weights",
status );
for( i=0; i<nbolo; i++ ) {
if( whitenoise[i] && (whitenoise[i] != VAL__BADD) ) {
whitenoise[i] = 1/(whitenoise[i]*whitenoise[i]);
}
}
}
/* Calculate the average power spectrum of good detectors */
tempdata = smf_fft_avpspec( odata, bolomask, 1, SMF__Q_BADB,
weightavpspec ? whitenoise : NULL,
status );
smf_close_file( &odata, status );
whitenoise = astFree( whitenoise );
bolomask = astFree( bolomask );
odata = tempdata;
tempdata = NULL;
/* Store the number of good bolometers */
parPut0i( "NGOOD", newgood, status );
} else {
/* Otherwise do forward/inverse transforms here as needed */
/* If inverse transform convert to cartesian representation first */
if( inverse && polar ) {
smf_fft_cart2pol( wf, idata, 1, power, status );
}
/* Tranform the data */
odata = smf_fft_data( wf, idata, NULL, inverse, 0, status );
smf_convert_bad( wf, odata, status );
if( inverse ) {
/* If output is time-domain, ensure that it is ICD bolo-ordered */
smf_dataOrder( odata, 1, status );
} else if( polar ) {
/* Store FFT of data in polar form */
smf_fft_cart2pol( wf, odata, 0, power, status );
}
}
/* open a reference input file for provenance propagation */
ndgNdfas( basegrp, gcount, "READ", &provid, status );
/* Export the data to a new file */
smf_write_smfData( odata, NULL, NULL, ogrp, gcount, provid,
MSG__VERB, 0, status );
/* Free resources */
ndfAnnul( &provid, status );
smf_close_file( &odata, status );
} else {
msgOutif( MSG__NORM, " ",
"Data are already transformed. No output will be produced",
status );
}
}
/* Update index into group */
gcount++;
}
/* Close the smfArray */
smf_close_related( &concat, status );
}
/* Write out the list of output NDF names, annulling the error if a null
parameter value is supplied. */
if( *status == SAI__OK ) {
grpList( "OUTFILES", 0, 0, NULL, ogrp, status );
if( *status == PAR__NULL ) errAnnul( status );
}
/* Tidy up after ourselves: release the resources used by the grp routines */
grpDelet( &igrp, status);
grpDelet( &ogrp, status);
if (basegrp) grpDelet( &basegrp, status );
if( igroup ) smf_close_smfGroup( &igroup, status );
if( flatramps ) smf_close_related( &flatramps, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
if (bbms) smf_close_related( &bbms, status );
ndfEnd( status );
/* Ensure that FFTW doesn't have any used memory kicking around */
fftw_cleanup();
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_calc_smoothedwvm ( ThrWorkForce *wf, const smfArray * alldata,
const smfData * adata, AstKeyMap* extpars, double **wvmtau,
size_t *nelems, size_t *ngoodvals, int * status ) {
size_t i;
size_t nrelated = 0; /* Number of entries in smfArray */
size_t nframes = 0; /* Number of timeslices */
size_t ngood = 0; /* Number of elements with good tau */
double *taudata = NULL; /* Local version of WVM tau */
const smfArray * thesedata = NULL; /* Collection of smfDatas to analyse */
smfArray * tmpthesedata = NULL; /* Local version of adata in a smfArray */
if (*status != SAI__OK) return;
if (alldata && adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm can not be given non-NULL alldata and non-NULL adata arguments"
" (possible programming error)", status );
return;
}
if (!alldata && !adata) {
*status = SAI__ERROR;
errRep("", "smf_calc_smoothedwvm: One of alldata or adata must be non-NULL",
status);
return;
}
if (!wvmtau) {
*status = SAI__ERROR;
errRep("", "Must supply a non-NULL pointer for wvmtau argument"
" (possible programming error)", status );
return;
}
/* if we have a single smfData put it in a smfArray */
if (alldata) {
if (alldata->ndat == 0 ) {
*status = SAI__ERROR;
errRep("", "No smfDatas present in supplied smfArray for WVM smoothing"
" (possible programming error)", status );
return;
}
thesedata = alldata;
} else {
tmpthesedata = smf_create_smfArray( status );
if (tmpthesedata) {
tmpthesedata->owndata = 0; /*not owned by the smfArray */
/* we know that the smfData here will not be touched in this
function so we do the BAD thing of casting const to non-const */
smf_addto_smfArray( tmpthesedata, (smfData *)adata, status );
}
thesedata = tmpthesedata;
}
/* Check that we have headers and that the smfData are the same length */
nrelated = thesedata->ndat;
for (i = 0; i < nrelated; i++ ) {
smfData * data = (thesedata->sdata)[i];
smfHead * hdr = data->hdr;
dim_t thisframes = 0;
if ( !hdr) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has no header. Aborting WVM smoothing",
status, i );
return;
}
smf_get_dims( data, NULL, NULL, NULL, &thisframes, NULL, NULL, NULL, status );
if (!nframes) nframes = thisframes;
if (thisframes != nframes) {
*status = SAI__ERROR;
errRepf( "", "smfData %zu has different length. Aborting WVM smoothing",
status, i );
return;
}
}
/* We will need the earliest and last airmass value in order
to calculate a zenith tau */
/* As a first step, just fill the time series with calculated WVM
tau values even though we know there are about 240 fewer tau
readings in reality. This initial approach will make it easier to
use the smoothed data directly rather than having to interpolate
from the 1.2 second data back into the 200 Hz data. */
taudata = astCalloc( nframes, sizeof(*taudata) );
if (*status == SAI__OK) {
double amprev = VAL__BADD;
double steptime;
size_t maxgap;
struct timeval tv1;
struct timeval tv2;
smfCalcWvmJobData *job_data = NULL;
int nworker;
/* We need to know the steptime so we can define the max good gap
in seconds and convert it to steps*/
steptime = (thesedata->sdata)[0]->hdr->steptime;
maxgap = (size_t)( 5.0 / steptime ); /* 5 seconds is just larger than 2 WVM readings */
/* Assume all files have the same airmass information */
smf_find_airmass_interval( (thesedata->sdata)[0]->hdr, &rev, NULL, NULL, NULL, status );
smf_timerinit( &tv1, &tv2, status );
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
if (*status == SAI__OK) {
dim_t tstep;
int iworker;
smfCalcWvmJobData *pdata = NULL;
/* Get the number of time slices to process in each thread. */
if( nworker > (int) nframes ) {
tstep = 1;
} else {
tstep = nframes/nworker;
}
/* to return the same values for one thread and multiple threads
we need to break the threads on wvm sample boundaries wherever
possible. We make an initial estimate of the number of WVM measurements
by assuming one every two seconds. */
{
smfData * curdata = NULL;
size_t nwvm = 0;
double prevtime = VAL__BADD;
double curtime;
size_t *boundaries = astGrow(NULL, nframes*(size_t)(steptime/2.0), sizeof(*boundaries));
for (i=0; i<nframes; i++) {
if (!curdata) {
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (curdata) smf_tslice_ast( curdata, i, 0, NO_FTS, status );
if ( !curdata || curdata->hdr->state->wvm_time == VAL__BADD ) {
/* Try the other datas */
SELECT_DATA( thesedata, curdata, VAL__BADD, wvm_time, i );
}
if (*status != SAI__OK) break;
if (!curdata) {
curtime = VAL__BADD;
} else {
curtime = curdata->hdr->state->wvm_time;
}
if (curtime != prevtime || nwvm == 0 ) {
/* Store the index in the boundaries array */
nwvm++;
boundaries = astGrow(boundaries, nwvm, sizeof(*boundaries));
if (!boundaries) { /* this is serious */
if (*status == SAI__OK) *status = SAI__ERROR;
errRep("", "Error allocating temporary memory for WVM calculation\n",
status );
break;
}
boundaries[nwvm-1] = i;
prevtime = curtime;
}
}
/* No point using too many threads */
if (*status == SAI__OK) {
if (nworker >= (int)nwvm) {
nworker = nwvm;
/* Allocate a measurement per thread */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->t1 = boundaries[iworker];
if (iworker+1 < nworker) pdata->t2 = boundaries[iworker+1]-1;
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
} else {
/* Allocate the workers to slices of approximate size tstep */
size_t prevend = 0; /* End of previous slice */
size_t prevbnd = 0; /* Index into previous boundaries[] array selection */
for( iworker = 0; iworker < nworker; iworker++ ) {
size_t belowidx = prevend+1;
size_t aboveidx = nframes;
size_t lbnd;
size_t ubnd;
size_t j;
size_t guess;
pdata = job_data + iworker;
if (iworker == 0) { /* always start at the beginning */
pdata->t1 = 0;
} else { /* Start one after the previous block */
pdata->t1 = prevend + 1;
}
/* Now we have to find the end of this slice */
guess = (iworker*tstep) + tstep - 1;
if (guess <= pdata->t1) guess = pdata->t1 + tstep;
/* find nearest boundaries */
for (j=prevbnd; j<nwvm; j++) {
if ( boundaries[j] > guess ) {
aboveidx = boundaries[j];
ubnd = j;
if (j>0) {
belowidx = boundaries[j-1];
lbnd = j -1 ;
} else {
lbnd = 0;
}
break;
}
}
/* Choose the closest, making sure that we are not choosing t1 */
if ( (guess - belowidx < aboveidx - guess) && belowidx > pdata->t1 ) {
pdata->t2 = belowidx - 1;
prevbnd = lbnd;
} else {
pdata->t2 = aboveidx - 1;
prevbnd = ubnd;
}
prevend = pdata->t2;
if (prevend == nframes - 1 && iworker < nworker-1 ) {
/* we have run out of slices so just use fewer workers */
nworker = iworker + 1;
break;
}
}
/* Ensure that the last thread picks up any left-over time slices */
pdata->t2 = nframes - 1;
}
/* Tidy up */
boundaries = astFree( boundaries );
}
}
/* Store all the other info needed by the worker threads, and submit the
jobs to fix the steps in each bolo, and then wait for them to complete. */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray *thrdata = NULL;
pdata = job_data + iworker;
pdata->nframes = nframes;
pdata->airmass = amprev; /* really need to get it from the start of each chunk */
pdata->taudata = taudata;
pdata->maxgap = maxgap;
/* Need to copy the smfDatas and create a new smfArray for each
thread */
thrdata = smf_create_smfArray( status );
for (i=0;i<nrelated;i++) {
smfData *tmpdata = NULL;
tmpdata = smf_deepcopy_smfData( wf, (thesedata->sdata)[i], 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( tmpdata, 0, status );
smf_addto_smfArray( thrdata, tmpdata, status );
}
pdata->thesedata = thrdata;
/* Need to do a deep copy of ast data and unlock them */
pdata->extpars = astCopy(extpars);
astUnlock( pdata->extpars, 1 );
/* Pass the job to the workforce for execution. */
thrAddJob( wf, THR__REPORT_JOB, pdata, smf__calc_wvm_job, 0, NULL,
status );
}
/* Wait for the workforce to complete all jobs. */
thrWait( wf, status );
/* Now free the resources we allocated during job creation
and calculate the number of good values */
for( iworker = 0; iworker < nworker; iworker++ ) {
smfArray * thrdata;
pdata = job_data + iworker;
astLock( pdata->extpars, 0 );
pdata->extpars = astAnnul( pdata->extpars );
thrdata = pdata->thesedata;
for (i=0;i<thrdata->ndat;i++) {
smf_lock_data( (thrdata->sdata)[i], 1, status );
}
smf_close_related( wf, &thrdata, status );
ngood += pdata->ngood;
}
}
job_data = astFree( job_data );
msgOutiff( MSG__NORM, "", FUNC_NAME ": %f s to calculate unsmoothed WVM tau values",
status, smf_timerupdate(&tv1,&tv2,status) );
}
if (*status == SAI__OK && extpars) {
/* Read extpars to see if we need to smooth */
double smoothtime = VAL__BADD;
if (astMapGet0D( extpars, "SMOOTHWVM", &smoothtime ) ) {
if (smoothtime != VAL__BADD && smoothtime > 0.0) {
smfData * data = (thesedata->sdata)[0];
double steptime = data->hdr->steptime;
dim_t boxcar = (dim_t)( smoothtime / steptime );
msgOutiff( MSG__VERB, "",
"Smoothing WVM data with %f s tophat function",
status, smoothtime );
smf_tophat1D( taudata, nframes, boxcar, NULL, 0, 0.0, status );
/* The tophat smoothing puts a bad value at the start and end of
the time series so we replace that with the adjacent value since
the step time is much smaller than WVM readout time. If more than
one value is bad we do not try to find the good value. */
taudata[0] = taudata[1];
taudata[nframes-1] = taudata[nframes-2];
}
}
}
/* Use this to get the raw WVM output for debugging */
/*
if (*status == SAI__OK) {
smfData *data = (thesedata->sdata)[0];
smfHead *hdr = data->hdr;
printf("# IDX TAU RTS_NUM RTS_END WVM_TIME\n");
for (i=0; i<nframes;i++) {
JCMTState * state;
state = &(hdr->allState)[i];
printf("%zu %.*g %d %.*g %.*g\n", i, DBL_DIG, taudata[i], state->rts_num,
DBL_DIG, state->rts_end, DBL_DIG, state->wvm_time);
}
} */
/* Free resources */
if (tmpthesedata) smf_close_related( wf, &tmpthesedata, status );
if (*status != SAI__OK) {
if (taudata) taudata = astFree( taudata );
*nelems = 0;
*ngoodvals = 0;
} else {
*wvmtau = taudata;
*nelems = nframes;
*ngoodvals = ngood;
}
}