void smf_history_add( smfData* data, const char * appl, int * status ) {
smfFile *file = NULL; /* data->file */
int done; /* Flag to denote whether history entry has been added */
AstKeyMap *history; /* AstKeyMap storing history information */
/* Check entry status */
if (*status != SAI__OK) return;
/* Check that we have a smfData */
if ( data == NULL ) {
*status = SAI__ERROR;
errRep( FUNC_NAME,
"Supplied smfData is a NULL pointer. Possible programming error.",
status);
return;
}
/* Now add entry to the history AstKeyMap if not already present */
if ( data->history == NULL ) {
history = astKeyMap(" " );
data->history = history;
astMapPut0I( data->history, appl, 1, " " );
} else if ( !astMapGet0I( data->history, appl, &done ) ) {
astMapPut0I( data->history, appl, 1, " " );
}
/* If we have a file, write the history entry into the file */
file = data->file;
if ( (file !=NULL) && (file->ndfid != NDF__NOID) ) {
smf_history_write( data, status);
}
msgOutiff( MSG__DEBUG2, " ", "Adding history item '%s'", status, appl);
}
void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
dim_t bolostep; /* Number of bolos per thread */
dim_t boxsize; /* No. of time slices in each noise box */
smfData *box = NULL; /* SmfData holding one box of input data */
size_t bstride; /* bolometer stride */
int calcfirst=0; /* Were bolo noises already measured? */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* Maximum allowed number of dc jumps */
dim_t dcfitbox; /* Width of box for DC step detection */
double dcthresh; /* Threshold for DC step detection */
dim_t dcsmooth; /* Width of median filter in DC step detection*/
double *din; /* Pointer to next input value */
double *dout; /* Pointer to next output value */
int fillgaps; /* If set perform gap filling */
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
int iw; /* Thread index */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local keymap */
size_t mbstride; /* model bolometer stride */
dim_t mntslice; /* Number of model time slices */
size_t mtstride; /* model time slice stride */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nbolo; /* Number of bolometers */
int nbox = 0; /* Number of noise boxes */
size_t nchisq; /* Number of data points in chisq calc */
dim_t nelbox; /* Number of data points in a noise box */
dim_t ndata; /* Total number of data points */
size_t nflag; /* Number of new flags */
int nleft; /* Number of samples not in a noise box */
dim_t ntslice; /* Number of time slices */
int nw; /* Number of worker threads */
size_t pend; /* Last non-PAD sample */
size_t pstart; /* First non-PAD sample */
smf_qual_t *qin; /* Pointer to next input quality value */
smf_qual_t *qout; /* Pointer to next output quality value */
smfArray *qua=NULL; /* Pointer to RES at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to RES at chunk */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
dim_t spikebox=0; /* Box size for spike detection */
double spikethresh=0; /* Threshold for spike detection */
size_t tend; /* Last input sample to copy */
size_t tstart; /* First input sample to copy */
size_t tstride; /* time slice stride */
double *var=NULL; /* Sample variance */
size_t xbstride; /* Box bolometer stride */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing NOI parameters */
astMapGet0A( keymap, "NOI", &kmap );
/* Assert bolo-ordered data */
smf_model_dataOrder( dat, allmodel, chunk, SMF__RES|SMF__QUA, 0, status );
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
/* Obtain parameters for NOI */
/* Data-cleaning parameters */
smf_get_cleanpar( kmap, res->sdata[0], NULL, &dcfitbox, &dcmaxsteps, &dcthresh,
&dcsmooth, &dclimcorr, NULL, &fillgaps, &zeropad, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
&spikethresh, &spikebox, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, status );
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );
/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status );
/* NOI model dimensions */
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &mntslice, NULL,
&mbstride, &mtstride, status );
/* Only estimate the white noise level once at the beginning - the
reason for this is to make measurements of the convergence
easier. We either do it prior to the start of iterations (in which
case the relative weights will be influeced by low-frequency noise,
this is initialized in smf_model_create), or else we calculate
the noise after the first iteration. */
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) ) {
/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Handle the first case now. */
if( mntslice == 1 ) {
var = astMalloc( nbolo*sizeof(*var) );
if (var) {
/* Measure the noise from power spectra */
smf_bolonoise( wf, res->sdata[idx], 0, 0.5, SMF__F_WHITELO,
SMF__F_WHITEHI, 0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT,
var, NULL, NULL, status );
for( i=0; i<nbolo; i++ ) if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Loop over time and store the variance for each sample */
for( j=0; j<mntslice; j++ ) {
model_data[i*mbstride+(j%mntslice)*mtstride] = var[i];
}
}
var = astFree( var );
}
/* If the NOI model is of the second form, the noise is estimated
in boxes of samples lasting "NOI.BOX_SIZE" seconds, and then the
noise level in the box is assigned to all samples in the box. */
} else if( mntslice == ntslice ) {
/* If not already done, get NOI.BOX_SIZE and convert from seconds to
samples. */
if( idx == 0 ) {
boxsize = 0;
smf_get_nsamp( kmap, "BOX_SIZE", res->sdata[0], &boxsize, status );
msgOutf( "", FUNC_NAME ": Calculating a NOI variance for each "
"box of %d samples.", status, (int) boxsize );
/* Find the indices of the first and last non-PAD sample. */
smf_get_goodrange( qua_data, ntslice, tstride, SMF__Q_PAD,
&pstart, &pend, status );
/* How many whole boxes fit into this range? */
nbox = ( pend - pstart + 1 ) / boxsize;
if( nbox == 0 ) nbox = 1;
/* How many samples would be left over at the end if we used this
many boxes? */
nleft = ( pend - pstart + 1 ) - nbox*boxsize;
/* Increase "boxsize" to reduce this number as far as possible.
Any samples that are left over after this increase of boxsize
will not be used when calculating the noise levels in each
bolometer. */
boxsize += nleft/nbox;
/* Create a smfData to hold one box-worth of input data. We
do not need to copy jcmtstate information. */
if( res->sdata[idx]->hdr ) {
instate = res->sdata[idx]->hdr->allState;
res->sdata[idx]->hdr->allState = NULL;
}
box = smf_deepcopy_smfData( res->sdata[idx], 0,
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY,
0, 0, status );
if( instate ) res->sdata[idx]->hdr->allState = instate;
/* Set the length of the time axis to the box size plus padding,
and create empty data and quality arrays for it. */
if( *status == SAI__OK ) {
box->dims[ box->isTordered?2:0 ] = boxsize + pstart + (ntslice - pend - 1);
smf_get_dims( box, NULL, NULL, NULL, NULL, &nelbox,
&xbstride, NULL, status );
box->pntr[0] = astMalloc( sizeof( double )*nelbox );
box->qual = astMalloc( sizeof( smf_qual_t )*nelbox );
/* For every bolometer, flag the start and end of the quality
array as padding, and store zeros in the data array. */
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
dout = ((double *) box->pntr[0]) + xbstride*ibolo;
qout = box->qual + xbstride*ibolo;
for( itime = 0; itime < pstart; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
dout = ((double *) box->pntr[0]) + xbstride*ibolo + pstart + boxsize;;
qout = box->qual + xbstride*ibolo + pstart + boxsize;
for( itime = pend + 1; itime < ntslice; itime++ ) {
*(qout++) = SMF__Q_PAD;
*(dout++) = 0.0;
}
}
}
}
/* Work space to hold the variance for each bolometer in a box */
var = astMalloc( nbolo*sizeof(*var) );
if( *status == SAI__OK ) {
/* Index of the first time slice within the input smfData
that is included in the first box. */
tstart = pstart;
/* Loop round each noise box */
for( ibox = 0; ibox < nbox; ibox++ ) {
/* Copy the data and quality values for this box from the
input smfData into "box", leaving room for padding at
both ends of box. Note, data is bolo-ordered so we
can assume that "tstride" is 1. */
din = ((double *)(res->sdata[idx]->pntr[0])) + tstart;
dout = ((double *)(box->pntr[0])) + pstart;
qin = qua_data + tstart;
qout = box->qual + pstart;
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
memcpy( dout, din, boxsize*sizeof( *din ) );
memcpy( qout, qin, boxsize*sizeof( *qin ) );
din += bstride;
dout += xbstride;
qin += bstride;
qout += xbstride;
}
/* Measure the noise from power spectra in the box. */
smf_bolonoise( wf, box, 0, 0.5, SMF__F_WHITELO, SMF__F_WHITEHI,
0, zeropad ? SMF__MAXAPLEN : SMF__BADSZT, var,
NULL, NULL, status );
/* Loop over time and store the variance for each sample in
the NOI model. On the last box, pick up any left over time
slices. */
if( ibox < nbox - 1 ) {
tend = tstart + boxsize - 1;
} else {
tend = pend;
}
for( ibolo = 0; ibolo < nbolo; ibolo++ ) {
if( !( qua_data[ ibolo*bstride ] & SMF__Q_BADB ) ) {
dout = model_data + ibolo*bstride + tstart;
for( itime = tstart; itime <= tend; itime++ ) {
*(dout++) = var[ ibolo ];
}
}
}
/* Update the index of the first time slice within the input
smfData that is included in the next box. */
tstart += boxsize;
}
var = astFree( var );
}
/* Report an error if the number of samples for each bolometer in
the NOI model is not 1 or "ntslice". */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", FUNC_NAME ": NOI model has %d samples - should be "
"%d or 1.", status, (int) mntslice, (int) ntslice);
}
}
if( kmap ) {
/* Flag spikes in the residual after first iteration */
if( spikethresh && !(flags&SMF__DIMM_FIRSTITER) ) {
/* Now re-flag */
smf_flag_spikes( wf, res->sdata[idx], SMF__Q_MOD,
spikethresh, spikebox, &nflag, status );
msgOutiff(MSG__VERB," ", " flagged %zu new %lf-sig spikes",
status, nflag, spikethresh );
}
if( dcthresh && dcfitbox ) {
smf_fix_steps( wf, res->sdata[idx], dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, 1, &nflag, NULL,
NULL, status );
msgOutiff(MSG__VERB, ""," detected %zu bolos with DC steps\n",
status, nflag);
}
}
/* Now calculate contribution to chi^2. This bit takes along time
if there is a lot of data so share the work out amongst the available
worker threads. How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Find how many bolometers to process in each worker thread. */
bolostep = nbolo/nw;
if( bolostep == 0 ) bolostep = 1;
/* Allocate job data for threads, and store the range of bolos to be
processed by each one. Ensure that the last thread picks up any
left-over bolos. */
SmfCalcModelNoiData *job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
SmfCalcModelNoiData *pdata;
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
pdata->b1 = iw*bolostep;
if( iw < nw - 1 ) {
pdata->b2 = pdata->b1 + bolostep - 1;
} else {
pdata->b2 = nbolo - 1 ;
}
/* Store other values common to all jobs. */
pdata->ntslice = ntslice;
pdata->mntslice = mntslice;
pdata->qua_data = qua_data;
pdata->model_data = model_data;
pdata->res_data = res_data;
pdata->bstride = bstride;
pdata->tstride = tstride;
pdata->mbstride = mbstride;
pdata->mtstride = mtstride;
/* Submit the job to the workforce. */
thrAddJob( wf, 0, pdata, smf1_calcmodel_noi, 0, NULL, status );
}
/* Wait for all jobs to complete. */
thrWait( wf, status );
/* Accumulate the results from all the worker threads. */
for( iw = 0; iw < nw; iw++ ) {
pdata = job_data + iw;
dat->chisquared[chunk] += pdata->chisquared;
nchisq += pdata->nchisq;
}
/* Free the job data. */
job_data = astFree( job_data );
}
}
}
/* Free resources */
if( box ) {
box->pntr[0] = astFree( box->pntr[0] );
box->qual = astFree( box->qual );
smf_close_file( &box, status );
}
/* Normalize chisquared for this chunk */
if( (*status == SAI__OK) && (nchisq >0) ) {
dat->chisquared[chunk] /= (double) nchisq;
}
/* Clean Up */
if( kmap ) kmap = astAnnul( kmap );
}
void smf_grp_related( const Grp *igrp, const size_t grpsize,
const int grouping, const int checksubinst,
double maxlen_s, double *srate_maxlen,
AstKeyMap *keymap, dim_t *maxconcatlen,
dim_t *maxfilelen, smfGroup **group,
Grp **basegrp, dim_t *pad, int *status ) {
/* Local variables */
size_t *chunk=NULL; /* Array of flags for continuous chunks */
dim_t * chunklen = NULL; /* Length of continuous chunk */
size_t currentindex = 0; /* Counter */
char cwave[10]; /* String containing wavelength */
smfData *data = NULL; /* Current smfData */
double downsampscale=0; /* Angular scale downsampling size */
double downsampfreq=0; /* Target downsampling frequency */
AstKeyMap * grouped = NULL; /* Primary AstKeyMap for grouping */
size_t i; /* Loop counter for index into Grp */
int isFFT=0; /* Set if data are 4d FFT */
size_t j; /* Loop counter */
int *keepchunk=NULL; /* Flag for chunks that will be kept */
dim_t maxconcat=0; /* Longest continuous chunk length */
dim_t maxflen=0; /* Max file length in time steps */
dim_t maxlen=0; /* Maximum concat length in samples */
int maxlen_scaled=0; /* Set once maxlen has been scaled, if needed */
dim_t maxpad=0; /* Maximum padding neeed for any input file */
size_t maxrelated = 0; /* Keep track of max number of related items */
size_t *new_chunk=NULL; /* keeper chunks associated with subgroups */
dim_t *new_tlen=NULL; /* tlens for new_subgroup */
size_t ngroups = 0; /* Counter for subgroups to be stored */
size_t nkeep = 0; /* Number of chunks to keep */
dim_t * piecelen = NULL; /* Length of single file */
smf_subinst_t refsubinst; /* Subinst of first file */
size_t **subgroups = NULL; /* Array containing index arrays to parent Grp */
smf_subinst_t subinst; /* Subinst of current file */
if ( *status != SAI__OK ) return;
if( maxlen_s < 0 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": maxlen_s cannot be < 0!", status );
return;
}
/* Get downsampling parameters */
if( keymap ) {
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, &downsampscale, &downsampfreq,
NULL, NULL, NULL, NULL, status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
return;
}
}
/* Initialize refcwave */
refsubinst = SMF__SUBINST_NONE;
/* Loop over files in input Grp: remember Grps are indexed from 1 */
grouped = astKeyMap( "SortBy=KeyUp" );
for (i=1; i<=grpsize; i++) {
char newkey[128];
char dateobs[81];
char subarray[10];
size_t nrelated = 0;
AstKeyMap * filemap = NULL;
AstKeyMap * indexmap = NULL;
/* First step: open file and harvest metadata */
smf_open_file( NULL, igrp, i, "READ", SMF__NOCREATE_DATA, &data, status );
if (*status != SAI__OK) break;
if( i==1 ) {
isFFT = smf_isfft( data, NULL, NULL, NULL, NULL, NULL, status );
} else if( smf_isfft(data, NULL, NULL, NULL, NULL, NULL, status) != isFFT ){
*status = SAI__ERROR;
errRep( "", FUNC_NAME
": mixture of time-series and FFT data encountered!",
status );
break;
}
/* If maxlen has not been set, do it here */
if( !maxlen && maxlen_s && data->hdr->steptime) {
maxlen = (dim_t) (maxlen_s / data->hdr->steptime );
}
/* Return srate_maxlen if requested: may want to know this number
even if maxlen_s is not set. Only calculate once, although it
gets overwritten once later if down-sampling. */
if( (i==1) && srate_maxlen && data->hdr->steptime ) {
*srate_maxlen = 1. / (double) data->hdr->steptime;
}
/* If requested check to see if we are mixing wavelengths */
if( checksubinst ) {
if( refsubinst == SMF__SUBINST_NONE ) {
refsubinst = smf_calc_subinst( data->hdr, status );
}
subinst = smf_calc_subinst( data->hdr, status );
if( subinst != refsubinst ) {
const char *refsubstr = smf_subinst_str( refsubinst, status );
const char *substr = smf_subinst_str( subinst, status );
*status = SAI__ERROR;
smf_smfFile_msg( data->file, "FILE", 1, "<unknown>" );
msgSetc( "REFSUB", refsubstr );
msgSetc( "SUB", substr );
errRep( "", FUNC_NAME
": ^FILE uses sub-instrument ^SUB which doesn't match "
"reference ^REFSUB", status );
}
}
/* Want to form a key that will be unique for a particular subscan
We know that DATE-OBS will be set for SCUBA-2 files and be the same
for a single set. Prefix by wavelength if we are grouping by wavelength.
*/
newkey[0] = '\0';
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL, status );
if( grouping == 1 ) {
/* Group different wavelengths separately */
smf_fits_getS( data->hdr, "WAVELEN", cwave, sizeof(cwave), status);
one_strlcat( newkey, cwave, sizeof(newkey), status );
one_strlcat( newkey, "_", sizeof(newkey), status );
}
if( grouping == 2 ) {
/* Group different subarrays separately */
one_strlcat( newkey, subarray, sizeof(newkey), status );
}
smf_fits_getS( data->hdr, "DATE-OBS", dateobs, sizeof(dateobs), status );
one_strlcat( newkey, dateobs, sizeof(newkey), status );
/* Include the dimentionality of the time series in the primary key
so that we do not end up doing something confusing like relating
a truncated file with a full length file */
if (*status == SAI__OK) {
dim_t dims[3];
char formatted[32];
smf_get_dims( data, &dims[0], &dims[1], NULL, &dims[2], NULL, NULL, NULL,
status );
sprintf(formatted, "_%" DIM_T_FMT "_%" DIM_T_FMT "_%" DIM_T_FMT, dims[0], dims[1], dims[2]);
one_strlcat( newkey, formatted, sizeof(newkey), status );
}
/* May want to read the dimensionality of the file outside of loop
so that we can compare values when storing in the keymap */
/* Now we want to create a keymap based on this key */
if (!astMapGet0A( grouped, newkey, &filemap ) ) {
int itemp = 0;
double steptime = data->hdr->steptime;
dim_t ntslice = 0;
dim_t thispad; /* Padding neeed for current input file */
filemap = astKeyMap( " " );
astMapPut0A( grouped, newkey, filemap, NULL );
/* Fill up filemap with general information on this file */
smf_find_seqcount( data->hdr, &itemp, status );
astMapPut0I( filemap, "SEQCOUNT", itemp, NULL );
smf_fits_getI( data->hdr, "NSUBSCAN", &itemp, status );
astMapPut0I( filemap, "NSUBSCAN", itemp, NULL );
/* Number of time slices */
smf_get_dims( data, NULL, NULL, NULL, &ntslice, NULL, NULL, NULL,
status );
/* Find length of down-sampled data, new steptime and maxlen */
if( (downsampscale || downsampfreq) && data->hdr && (*status==SAI__OK) ) {
double scalelen;
if( downsampscale ) {
if( data->hdr->scanvel != VAL__BADD ) {
double oldscale = steptime * data->hdr->scanvel;
scalelen = oldscale / downsampscale;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown scan velocity", status );
}
} else {
if( steptime ) {
double oldsampfreq = 1./steptime;
scalelen = downsampfreq / oldsampfreq;
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep( "", FUNC_NAME ": can't resample ^FILE because it has "
"unknown sample rate", status );
}
}
/* only down-sample if it will be a reasonable factor */
if( (*status==SAI__OK) && (scalelen <= SMF__DOWNSAMPLIMIT) ) {
smf_smfFile_msg(data->file, "FILE", 1, "" );
msgOutiff( MSG__VERB, "", FUNC_NAME
": will down-sample file ^FILE from %5.1lf Hz to "
"%5.1lf Hz", status, (1./steptime), (scalelen/steptime) );
ntslice = round(ntslice * scalelen);
/* If maxlen has been requested, and we have not already worked
out a scaled version (just uses the sample rates for the first
file... should be close enough -- the alternative is a 2-pass
system). */
if( !maxlen_scaled ) {
maxlen = round(maxlen*scalelen);
maxlen_scaled = 1;
msgOutiff( MSG__VERB, "", FUNC_NAME
": requested maxlen %g seconds = %" DIM_T_FMT " down-sampled "
"time-slices", status, maxlen_s, maxlen );
/* Return updated srate_maxlen for down-sampling if requested */
if( srate_maxlen ) {
*srate_maxlen = scalelen/steptime;
}
}
}
}
/* Check that an individual file is too long (we assume related
files are all the same) */
if( maxlen && (ntslice > maxlen) && *status == SAI__OK) {
*status = SAI__ERROR;
msgSeti("NTSLICE",ntslice);
msgSeti("MAXLEN",maxlen);
smf_smfFile_msg( data->file, "FILE", 1, "" );
errRep(FUNC_NAME,
"Number of time steps in file ^FILE time exceeds maximum "
"(^NTSLICE>^MAXLEN)", status);
}
/* Scaled values of ntslice and maximum length */
astMapPut0I( filemap, "NTSLICE", ntslice, NULL );
/* Work out the padding needed for this file including downsampling. */
if( keymap ) {
thispad = smf_get_padding( keymap, 0, data->hdr, VAL__BADD, status );
if( thispad > maxpad ) maxpad = thispad;
} else {
thispad = 0;
}
astMapPut0I( filemap, "PADDING", thispad, NULL );
/* Update maxflen */
if( ntslice > maxflen ) {
maxflen = ntslice;
}
/* Store OBSID or OBSIDSS depending on whether we are grouping by wavelength */
if (grouping) {
astMapPut0C( filemap, "OBSID", data->hdr->obsidss, NULL );
} else {
char obsid[81];
smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
astMapPut0C( filemap, "OBSID", obsid, NULL );
}
}
/* Store the file index in another keymap indexed by subarray */
if ( !astMapGet0A( filemap, "GRPINDICES", &indexmap ) ) {
indexmap = astKeyMap( "SortBy=KeyUp" );
astMapPut0A( filemap, "GRPINDICES", indexmap, NULL );
}
astMapPut0I( indexmap, subarray, i, NULL );
/* Need to track the largest number of related subarrays in a single slot */
nrelated = astMapSize( indexmap );
if (nrelated > maxrelated) maxrelated = nrelated;
/* Free resources */
filemap = astAnnul( filemap );
indexmap = astAnnul( indexmap );
smf_close_file( NULL, &data, status );
}
/* We now know how many groups there are */
ngroups = astMapSize( grouped );
/* Sort out chunking. The items are sorted by date and then by wavelength.
We define a continuous chunk if it has the same OBSID, the same SEQCOUNT
and NSUBSCAN increments by one from the previous entry.
Also count number of related items in each slot.
*/
if (*status == SAI__OK) {
typedef struct { /* somewhere to store the values easily */
char obsid[81];
char related[81]; /* for concatenated subarrays */
int nsubscan;
int seqcount;
} smfCompareSeq;
smfCompareSeq current;
smfCompareSeq previous;
dim_t totlen = 0;
size_t thischunk;
/* Get the chunk flags and also store the size of the chunk */
chunk = astCalloc( ngroups, sizeof(*chunk) );
chunklen = astCalloc( ngroups, sizeof(*chunklen) );
piecelen = astCalloc( ngroups, sizeof(*piecelen) );
thischunk = 0; /* The current chunk */
for (i=0; i<ngroups; i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
const char * tempstr = NULL;
int thistlen = 0;
size_t nsubarrays = 0;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get info for length limits */
astMapGet0I( thismap, "NTSLICE", &thistlen );
piecelen[i] = thistlen;
if (isFFT) {
/* Never concatenate FFT data */
thismap = astAnnul(thismap);
chunk[i] = i;
chunklen[i] = thistlen;
continue;
}
/* Get indices information and retrieve the sub-instrument names
in sort order to concatenate for comparison. We only store in
a continuous chunk if we have the same subarrays for the whole
chunk. */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
(current.related)[0] = '\0';
for (j = 0; j < nsubarrays; j++ ) {
one_strlcat( current.related, astMapKey(grpindices, j), sizeof(current.related), status );
}
grpindices = astAnnul( grpindices );
/* Fill in the current struct */
astMapGet0I( thismap, "SEQCOUNT", &(current.seqcount) );
astMapGet0I( thismap, "NSUBSCAN", &(current.nsubscan) );
astMapGet0C( thismap, "OBSID", &tempstr );
one_strlcpy( current.obsid, tempstr, sizeof(current.obsid), status );
/* First chunk is special, else compare */
if (i == 0) {
totlen = thistlen;
} else {
if ( ( current.seqcount == previous.seqcount ) &&
( current.nsubscan - previous.nsubscan == 1 ) &&
( strcmp( current.obsid, previous.obsid ) == 0 ) &&
( strcmp( current.related, previous.related ) == 0 ) ) {
/* continuous - check length */
totlen += thistlen;
if ( maxlen && totlen > maxlen ) {
thischunk++;
totlen = thistlen; /* reset length */
} else {
/* Continuous */
}
} else {
/* discontinuity */
thischunk++;
totlen = thistlen; /* Update length of current chunk */
}
}
chunklen[thischunk] = totlen;
chunk[i] = thischunk;
memcpy( &previous, ¤t, sizeof(current) );
thismap = astAnnul( thismap );
}
}
/* Decide if we are keeping a chunk by looking at the length. */
maxconcat = 0;
nkeep = 0;
keepchunk = astMalloc( ngroups*sizeof(*keepchunk) );
for (i=0; i<ngroups; i++) {
size_t thischunk;
thischunk = chunk[i];
if ( chunklen[thischunk] < SMF__MINCHUNKSAMP ) {
/* Warning message */
msgSeti("LEN",chunklen[thischunk]);
msgSeti("MIN",SMF__MINCHUNKSAMP);
msgOut( " ", "SMF_GRP_RELATED: ignoring short chunk (^LEN<^MIN)",
status);
keepchunk[i] = 0;
} else {
keepchunk[i] = 1;
if (maxconcat < chunklen[thischunk]) maxconcat = chunklen[thischunk];
nkeep++;
}
}
/* If no useful chunks generate an error */
if( (*status==SAI__OK) && (!nkeep) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No useful chunks.", status );
goto CLEANUP;
}
/* Allocate a subgroup array of the right size and fill it. They keymap
is sorted by date (and wavelength) so we can always index into it by using
indices from the subgroup. */
subgroups = astCalloc( nkeep, sizeof(*subgroups) );
new_chunk = astCalloc( nkeep, sizeof(*new_chunk) );
new_tlen = astCalloc( nkeep, sizeof(*new_tlen) );
currentindex = 0;
for (i=0;i<ngroups;i++) {
AstKeyMap * thismap = NULL;
AstKeyMap * grpindices = NULL;
size_t nsubarrays = 0;
size_t *indices = astCalloc( maxrelated, sizeof(*indices) );
/* skip if we are dropping this chunk */
if (!keepchunk[i]) continue;
/* Get the keymap entry for this slot */
astMapGet0A( grouped, astMapKey(grouped, i), &thismap );
/* Get the indices keymap */
astMapGet0A( thismap, "GRPINDICES", &grpindices );
nsubarrays = astMapSize( grpindices );
for (j=0; j<nsubarrays; j++) {
int myindex;
astMapGet0I( grpindices, astMapKey(grpindices, j), &myindex );
indices[j] = myindex;
}
grpindices = astAnnul( grpindices );
thismap = astAnnul( thismap );
subgroups[currentindex] = indices;
new_chunk[currentindex] = chunk[i];
new_tlen[currentindex] = piecelen[i];
currentindex++;
}
/* Create the smfGroup */
*group = smf_construct_smfGroup( igrp, subgroups, new_chunk, new_tlen,
nkeep, maxrelated, 0, status );
/* Return maxfilelen if requested */
if( maxfilelen ) {
*maxfilelen = maxflen;
}
/* Return maxconcatlen if requested */
if( maxconcatlen ) {
*maxconcatlen = maxconcat;
}
/* Create a base group for output files if required */
/* Create a base group of filenames */
if (*status == SAI__OK && basegrp ) {
*basegrp = smf_grp_new( (*group)->grp, "Base Group", status );
/* Loop over time chunks */
for( i=0; (*status==SAI__OK)&&(i<(*group)->ngroups); i++ ) {
size_t idx;
/* Check for new continuous chunk */
if( i==0 || ( (*group)->chunk[i] != (*group)->chunk[i-1]) ) {
/* Loop over subarray */
for( idx=0; idx<(*group)->nrelated; idx++ ) {
size_t grpindex = (*group)->subgroups[i][idx];
if ( grpindex > 0 ) {
ndgCpsup( (*group)->grp, grpindex, *basegrp, status );
}
}
}
}
}
CLEANUP:
keepchunk = astFree( keepchunk );
chunk = astFree( chunk );
chunklen = astFree( chunklen );
piecelen = astFree( piecelen );
grouped = astAnnul( grouped );
if( *status != SAI__OK ) {
/* free the group */
if (basegrp && *basegrp) grpDelet( basegrp, status );
if (group && *group) {
smf_close_smfGroup( group, status );
} else {
/* have to clean up manually */
new_chunk = astFree( new_chunk );
new_tlen = astFree( new_tlen );
if( subgroups ) {
size_t isub;
for( isub=0; isub<nkeep; isub++ ) {
subgroups[isub] = astFree( subgroups[isub] );
}
subgroups = astFree( subgroups );
}
}
}
/* Return the maximum padding if required. */
if( pad ) *pad = maxpad;
}
void smf_obsmap_fill( const smfData * data, AstKeyMap * obsmap, AstKeyMap * objmap,
int * status ) {
double dateobs = 0.0; /* MJD UTC of start of observation */
char object[SZFITSTR]; /* Object name */
char obsid[SZFITSTR]; /* Observation ID */
char instrume[SZFITSTR]; /* instrument name */
AstKeyMap * obsinfo = NULL; /* observation information */
if (*status != SAI__OK) return;
/* if there is no hdr we can not index it in the reported summary */
if (data->hdr && data->hdr->fitshdr) {
/* Get observation ID and see if we already have an entry in the map */
(void)smf_getobsidss( data->hdr->fitshdr, obsid, sizeof(obsid), NULL, 0, status );
/* As of 20080718 OBSID is not unique per obs so chop off date*/
obsid[22] = '\0';
if (!astMapHasKey(obsmap, obsid )) {
int itemp;
/* Create a sub keymap and populate it */
obsinfo = astKeyMap( " " );
/* fill with default value in case undefined */
one_strlcpy( object, "<undefined>", sizeof(object), status );
smf_getfitss( data->hdr, "OBJECT", object, sizeof(object),
status );
astMapPut0C( obsinfo, "OBJECT", object, NULL );
astMapPut0I( obsinfo, "OBSMODE", data->hdr->obsmode, NULL );
astMapPut0I( obsinfo, "OBSTYPE", data->hdr->obstype, NULL );
astMapPut0I( obsinfo, "SWMODE", data->hdr->swmode, NULL );
astMapPut0I( obsinfo, "INBEAM", data->hdr->inbeam, NULL );
smf_fits_getI( data->hdr, "OBSNUM", &itemp, status );
astMapPut0I( obsinfo, "OBSNUM", itemp, NULL );
smf_fits_getI( data->hdr, "UTDATE", &itemp, status );
astMapPut0I( obsinfo, "UTDATE", itemp, NULL );
smf_fits_getL( data->hdr, "SIMULATE", &itemp, status );
astMapPut0I( obsinfo, "SIMULATE", itemp, NULL );
smf_getfitss( data->hdr, "INSTRUME", instrume, sizeof(instrume),
status);
astMapPut0C( obsinfo, "INSTRUME", instrume, NULL );
/* store the MJD of observation for sorting purposes */
smf_find_dateobs( data->hdr, &dateobs, NULL, status );
astMapPut0D( obsinfo, "MJD-OBS", dateobs, NULL );
/* store information in the global observation map
and also track how many distinct objects we have */
astMapPut0A( obsmap, obsid, obsinfo, NULL );
astMapPut0I( objmap, object, 0, NULL );
obsinfo = astAnnul( obsinfo );
} else {
int curbeam = 0;
astMapGet0A( obsmap, obsid, &obsinfo );
/* INBEAM is interesting since technically each sequence can involve
different hardware being in the beam so we have to retrieve the
current value from the keymap and OR it with the current value
from the data header. */
/* we know that the value has been filled in previously so no
need to check */
astMapGet0I( obsinfo, "INBEAM", &curbeam );
curbeam |= data->hdr->inbeam;
astMapPut0I( obsinfo, "INBEAM", curbeam, NULL );
obsinfo = astAnnul( obsinfo );
}
}
return;
}
void smf_calcmodel_gai( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel, int flags,
int *status) {
/* Local Variables */
size_t bstride; /* bolometer stride */
dim_t gain_box=0; /* No. of time slices in a block */
size_t gbstride; /* GAIn bolo stride */
size_t gcstride; /* GAIn coeff stride */
int gflat=0; /* correct flatfield using GAI */
dim_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
dim_t j; /* Loop counter */
AstKeyMap *kmap=NULL; /* Local GAIn keymap */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
dim_t nblock; /* No. of time slice blocks */
dim_t nbolo; /* Number of bolometers */
dim_t ndata; /* Number of data points */
smfArray *noi=NULL; /* Pointer to NOI at chunk */
double *noi_data=NULL; /* Pointer to DATA component of model */
size_t noibstride; /* bolo stride for noise */
dim_t nointslice; /* number of time slices for noise */
size_t noitstride; /* Time stride for noise */
dim_t npar; /* No. of parameters per bolometer */
dim_t ntslice; /* Number of time slices */
int oldalg = 1; /* Is the old COM algorithm being used? */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DAT */
double *scale; /* Pointer to scale factor */
size_t tstride; /* time slice stride */
double *wg; /* Workspace holding time slice gains */
double *woff; /* Workspace holding time slice offsets */
/* Main routine */
if( *status != SAI__OK ) return;
if( !(flags&SMF__DIMM_INVERT) ) return;
/* See if the new sigma-clipping COM algorithm is being used. */
astMapGet0A( keymap, "COM", &kmap );
astMapGet0I( kmap, "OLDALG", &oldalg );
kmap = astAnnul( kmap );
/* Obtain pointer to sub-keymap containing GAI parameters */
if( !astMapHasKey( keymap, "GAI" ) ) return;
astMapGet0A( keymap, "GAI", &kmap );
astMapGet0I( kmap, "FLATFIELD", &gflat );
if( kmap ) kmap = astAnnul( kmap );
/* Report an error if gai.flatfield is used with the new COM algorithm. */
if( !oldalg && gflat && *status == SAI__OK ) {
errRep( "", "Cannot use GAI.FLATFIELD with new COM algorithm.", status );
}
/* Only have to do something if gai.flatfield set */
if( !gflat || *status != SAI__OK ) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
model = allmodel[chunk];
if(dat->noi) noi = dat->noi[chunk];
/* Get the number of blocks into which to split each time series. Each box
(except possibly the last one contains "gain_box" time slices. */
astMapGet0A( keymap, "COM", &kmap );
smf_get_nsamp( kmap, "GAIN_BOX", res->sdata[0], &gain_box, status );
if (kmap) kmap = astAnnul( kmap );
if (*status != SAI__OK) return;
/* Ensure everything is in bolo-order */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__NOI, 0, status );
/* Loop over index in subgrp (subarray) */
for( idx=0; idx<res->ndat; idx++ ) {
/* Get pointers to DATA components */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( noi ) {
smf_get_dims( noi->sdata[idx], NULL, NULL, NULL, &nointslice,
NULL, &noibstride, &noitstride, status);
noi_data = (double *)(noi->sdata[idx]->pntr)[0];
}
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep("", FUNC_NAME ": Null data in inputs", status);
} else {
/* Get the raw data dimensions */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice, &ndata,
&bstride, &tstride, status);
smf_get_dims( model->sdata[idx], NULL, NULL, NULL, &npar, NULL,
&gbstride, &gcstride, status);
/* If com.gain_box is zero, use a value of ntslice, so that a single
box will be used covering the whoel time stream. */
if( gain_box == 0 ) gain_box = ntslice;
/* Allocate work space for the gain and offset for each time slice. */
woff = astMalloc( ntslice*sizeof( *woff ) );
wg = astMalloc( ntslice*sizeof( *wg ) );
/* Get the number of blocks into which the time stream is divided.
Each block has a separate gain, offset and correlation factor
for each bolometer. */
nblock = npar/3;
/* Undo the gain correction stored in GAI (the gain is applied to
the signal and noise in smf_calcmodel_com) */
for( i=0; i<nbolo; i++ ) {
if( !(qua_data[i*bstride]&SMF__Q_BADB) ) {
/* Get the gain and offset for each time slice of this bolometer. */
smf_gandoff( i, 0, ntslice - 1, ntslice, gbstride, gcstride,
model_data, nblock, gain_box, wg, woff, NULL, status );
/* First undo the flatfield correction to the signal */
scale = wg;
for( j=0; j<ntslice; j++,scale++ ) {
if( !(qua_data[i*bstride + j*tstride]&SMF__Q_MOD) &&
*scale != VAL__BADD && *scale > 0.0 ) {
res_data[i*bstride + j*tstride] *= *scale;
}
}
/* Then scale the noise. */
if( noi ) {
scale = wg;
for( j=0; j<nointslice; j++,scale++ ) {
if( noi_data[i*noibstride + j*noitstride] != VAL__BADD &&
*scale != VAL__BADD && *scale > 0.0 ) {
noi_data[i*noibstride + j*noitstride] *= (*scale) * (*scale);
}
}
}
}
}
/* Free work space. */
woff = astFree( woff );
wg = astFree( wg );
}
}
}
void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status ) {
/*
* Name:
* atlKy2hd
* Purpose:
* Copies values from an AST KeyMap to a primitive HDS object.
* Language:
* C.
* Invocation:
* void atlKy2hd( AstKeyMap *keymap, HDSLoc *loc, int *status )
* Description:
* This routine copies the contents of an AST KeyMap into a supplied
* HDS structure.
* Arguments:
* keymap
* An AST pointer to the KeyMap.
* loc
* A locator for the HDS object into which the KeyMap contents
* are to be copied. A new component is added to the HDS object for
* each entry in the KeyMap.
* status
* The inherited status.
* Copyright:
* Copyright (C) 2008, 2010, 2012 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 29-APR-2008 (DSB):
* Original version.
* 2010-09-23 (TIMJ):
* Fix arrays of strings.
* 2010-09-30 (TIMJ):
* Make sure that we are using the correct status pointer in AST.
* 2010-10-01 (TIMJ):
* Sort the keys when writing to HDS structured.
* 2010-10-04 (TIMJ):
* Support Short ints in keymap
* 14-SEP-2012 (DSB):
* Moved from kaplibs to atl.
* 17-SEP-2012 (DSB):
* Add support for undefined values.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*/
/* Local Varianles: */
AstObject **objArray = NULL;
AstObject *obj = NULL;
HDSLoc *cloc = NULL;
HDSLoc *dloc = NULL;
const char *cval = NULL;
const char *key;
double dval;
float fval;
int i;
int ival;
int j;
int lenc;
int nval;
char *oldsortby;
int *oldstat = NULL;
int size;
int type;
int veclen;
size_t el;
void *pntr = NULL;
/* Check inherited status */
if( *status != SAI__OK ) return;
/* Make sure that we are checking AST status */
oldstat = astWatch( status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
int nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Loop round each entry in the KeyMap. */
size = astMapSize( keymap );
for( i = 0; i < size; i++ ) {
if (*status != SAI__OK) break;
/* Get the key. the data type and the vector length for the current
KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
veclen = astMapLength( keymap, key );
/* If the current entry holds one or more nested KeyMaps, then we call
this function recursively to add them into a new HDS component. */
if( type == AST__OBJECTTYPE ) {
/* First deal with scalar entries holding a single KeyMap. */
if( veclen == 1 ) {
datNew( loc, key, "KEYMAP_ENTRY", 0, NULL, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0A( keymap, key, &obj );
if( astIsAKeyMap( obj ) ) {
atlKy2hd( (AstKeyMap *) obj, cloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &cloc, status );
/* Now deal with vector entries holding multiple KeyMaps. */
} else {
datNew( loc, key, "KEYMAP_ENTRY", 1, &veclen, status );
datFind( loc, key, &cloc, status );
objArray = astMalloc( sizeof( AstObject *) * (size_t)veclen );
if( objArray ) {
(void) astMapGet1A( keymap, key, veclen, &nval, objArray );
for( j = 1; j <= veclen; j++ ) {
datCell( cloc, 1, &j, &dloc, status );
if( astIsAKeyMap( objArray[ j - 1 ] ) ) {
atlKy2hd( (AstKeyMap *) objArray[ j - 1 ], dloc, status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "atlKy2hd: Supplied KeyMap contains unusable AST "
"objects (programming error).", status );
}
datAnnul( &dloc, status );
}
objArray = astFree( objArray );
}
datAnnul( &cloc, status );
}
/* For primitive types... */
} else if( type == AST__INTTYPE ) {
if( veclen == 1 ) {
datNew0I( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0I( keymap, key, &ival );
datPut0I( cloc, ival, status );
datAnnul( &cloc, status );
} else {
datNew1I( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_INTEGER", "WRITE", &pntr, &el, status );
(void) astMapGet1I( keymap, key, veclen, &nval, (int *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__SINTTYPE ) {
short sval = 0;
if( veclen == 1 ) {
datNew0W( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0S( keymap, key, &sval );
datPut0W( cloc, sval, status );
datAnnul( &cloc, status );
} else {
datNew1W( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_WORD", "WRITE", &pntr, &el, status );
(void) astMapGet1S( keymap, key, veclen, &nval, (short *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__DOUBLETYPE ) {
if( veclen == 1 ) {
datNew0D( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0D( keymap, key, &dval );
datPut0D( cloc, dval, status );
datAnnul( &cloc, status );
} else {
datNew1D( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_DOUBLE", "WRITE", &pntr, &el, status );
(void) astMapGet1D( keymap, key, veclen, &nval, (double *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__FLOATTYPE ) {
if( veclen == 1 ) {
datNew0R( loc, key, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0F( keymap, key, &fval );
datPut0R( cloc, fval, status );
datAnnul( &cloc, status );
} else {
datNew1R( loc, key, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_REAL", "WRITE", &pntr, &el, status );
(void) astMapGet1F( keymap, key, veclen, &nval, (float *) pntr );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
} else if( type == AST__STRINGTYPE ) {
lenc = astMapLenC( keymap, key );
if( veclen == 1 ) {
datNew0C( loc, key, lenc, status );
datFind( loc, key, &cloc, status );
(void) astMapGet0C( keymap, key, &cval );
datPut0C( cloc, cval, status );
datAnnul( &cloc, status );
} else {
datNew1C( loc, key, lenc, veclen, status );
datFind( loc, key, &cloc, status );
datMapV( cloc, "_CHAR", "WRITE", &pntr, &el, status );
(void) atlMapGet1C( keymap, key, veclen*lenc, lenc, &nval,
(char *) pntr, status );
datUnmap( cloc, status );
datAnnul( &cloc, status );
}
/* KeyMap "UNDEF" values are always scalar and have no corresponding HDS
data type. So arbitrarily use an "_INTEGER" primitive with no defined
value to represent a KeyMap UNDEF value. */
} else if( type == AST__UNDEFTYPE ) {
datNew0L( loc, key, status );
/* Unknown or unsupported data types. */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "T", type );
errRep( "", "atlKy2hd: Supplied KeyMap contains entries with "
"unusable data type (^T) (programming error).", status );
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Reset AST status */
astWatch( oldstat );
}
void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
int *status ){
/*
*+
* Name:
* kpg1Kygp1
* Purpose:
* Creates a GRP group holding keyword/value pairs read from an AST KeyMap.
* Language:
* C.
* Invocation:
* void kpg1Kygp1( AstKeyMap *keymap, Grp **igrp, const char *prefix,
* int *status )
* Description:
* This function is the inverse of kpg1Kymp1. It extracts the values
* from the supplied AST KeyMap and creates a set of "name=value" strings
* which it appends to a supplied group (or creates a new group). If
* the KeyMap contains nested KeyMaps, then the "name" associated with
* each primitive value stored in the returned group is a hierarchical
* list of component names separated by dots.
* Arguments:
* keymap
* A pointer to the KeyMap. Numerical entries which have bad values
* (VAL__BADI for integer entries or VAL__BADD for floating point
* entries) are not copied into the group.
* igrp
* A location at which is stored a pointer to the Grp structure
* to which the name=value strings are to be appended. A new group is
* created and a pointer to it is returned if the supplied Grp
* structure is not valid.
* prefix
* A string to append to the start of each key extracted from the
* supplied KeyMap. If NULL, no prefix is used.
* status
* Pointer to the inherited status value.
* Notes:
* - This function provides a private implementation for the public
* KPG1_KYGRP Fortran routine and kpg1Kygrp C function.
* - Entries will be stored in the group in alphabetical order
* Copyright:
* Copyright (C) 2008,2010 Science & Technology Facilities Council.
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful,but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* TIMJ: Tim Jenness (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 7-NOV-2005 (DSB):
* Original version.
* 15-JUL-2008 (TIMJ):
* Tweak to GRP C API.
* 2010-07-19 (TIMJ):
* Handle vector keymap entries.
* 2010-08-12 (TIMJ):
* Store entries in group in alphabetical order.
* 13-AUG-2010 (DSB):
* Re-instate the original SortBy value before exiting.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *obj; /* Pointer to nested AST Object */
char *oldsortby; /* The old value of the KeyMap's SortBy attribute */
char *text; /* Sum of concatenated strings */
const char *key; /* Key string for current entry in KeyMap */
const char *value; /* Value of current entry in KeyMap */
double dval; /* Double value */
int *old_status; /* Pointer to original status variable */
int bad; /* Is the numerical entry value bad? */
int i; /* Index into supplied KeyMap */
int ival; /* Integer value */
int n; /* Number of entries in the KeyMap */
int nc; /* Length of "text" excluding trailing null */
int type; /* Data type of current entry in KeyMap */
int valid; /* Is the supplied GRP structure valid? */
/* Check the inherited status. */
if( *status != SAI__OK ) return;
/* Make AST use the Fortran status variable. */
old_status = astWatch( status );
/* Create a new GRP group if required. */
valid = grpValid( *igrp, status );
if( !valid ) *igrp = grpNew( "Created by kpg1_Kygp1", status );
/* If set, save the old SortBy value and then ensure alphabetical sorting.
We need to take a copy of the original string since the buffer in which
the string is stored may be re-used by subsequent incocations of astGetC. */
if( astTest( keymap, "SortBy" ) ) {
nc = 0;
oldsortby = astAppendString( NULL, &nc, astGetC( keymap, "SortBy" ) );
} else {
oldsortby = NULL;
}
astSet( keymap, "SortBy=KeyUp" );
/* Get the number of entries in the KeyMap. */
n = astMapSize( keymap );
/* Loop round all the entries in the KeyMap.*/
for( i = 0; i < n; i++ ) {
/* Get the name and type of the current KeyMap entry. */
key = astMapKey( keymap, i );
type = astMapType( keymap, key );
/* If the entry is an AST Object, get a pointer to it.*/
if( type == AST__OBJECTTYPE && astMapGet0A( keymap, key, &obj ) ) {
/* If it is a nested KeyMap, update the prefix and call this function
recursively. We ignore other forms of AST Objects. */
if( astIsAKeyMap( obj ) ) {
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "." );
kpg1Kygp1( (AstKeyMap *) obj, igrp, text, status );
text = astFree( text );
}
/* If it is a primitive, format it and add it to the group. */
} else {
/* If it is a numerical type, see if it has a bad value. */
bad = 0;
if( type == AST__INTTYPE && astMapGet0I( keymap, key, &ival ) ){
if( ival == VAL__BADI ) bad = 1;
} else if( type == AST__DOUBLETYPE && astMapGet0D( keymap, key, &dval ) ){
if( dval == VAL__BADD ) bad = 1;
}
/* If it not bad, get its formatted value. We also make sure that astMapGet0C returns
true because we intend to skip undefined values. */
if( !bad && astMapGet0C( keymap, key, &value ) ) {
size_t length;
/* Write the key and equals sign to a buffer */
nc = 0;
text = astAppendString( NULL, &nc, prefix );
text = astAppendString( text, &nc, key );
text = astAppendString( text, &nc, "=" );
length = astMapLength( keymap, key );
if( length > 1 ) {
/* Vector so we need to use (a,b,c) syntax */
char thiselem[GRP__SZNAM+1];
size_t l;
text = astAppendString( text, &nc, "(");
for ( l = 0; l < length; l++) {
if( astMapGetElemC( keymap, key, sizeof(thiselem), l, thiselem ) ) {
text = astAppendString( text, &nc, thiselem );
}
/* always deal with the comma. Even if value was undef we need to put in the comma */
if( l < (length - 1) ) {
text = astAppendString( text, &nc, "," );
}
}
text = astAppendString( text, &nc, ")");
} else {
/* Scalar */
text = astAppendString( text, &nc, value );
}
/* Put it in the group. */
grpPut1( *igrp, text, 0, status );
text = astFree( text );
}
}
}
/* If it was originally set, re-instate the old SortBy value in the KeyMap,
then free the memory. Otherwise, clear the SortBy attribute. */
if( oldsortby ) {
astSetC( keymap, "SortBy", oldsortby );
oldsortby = astFree( oldsortby );
} else {
astClear( keymap, "SortBy" );
}
/* Make AST use its original status variable. */
astWatch( old_status );
}
void smf_get_extpar( AstKeyMap *keymap, smf_tausrc *tausrc,
smf_extmeth *extmeth, int *import, int *status ) {
const char *tempstr = NULL;
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain parameters from keymap when non-NULL pointers given */
if( import ) {
if( ! astMapGet0I( keymap, "IMPORT", import ) ) {
*import = 0;
}
}
if( tausrc ) {
if( astMapGet0C( keymap, "TAUSRC", &tempstr ) ) {
switch( toupper(tempstr[0]) ) {
case 'A': /* AUTO */
*tausrc = SMF__TAUSRC_AUTO;
break;
case 'C': /* CSO* */
/* Need to compare CSOTAU to CSOFIT */
switch( toupper(tempstr[3]) ) {
case 'T':
*tausrc = SMF__TAUSRC_CSOTAU;
break;
case 'F':
*tausrc = SMF__TAUSRC_CSOFIT;
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
break;
case 'F': /* FILTERTAU */
*tausrc = SMF__TAUSRC_TAU;
break;
case 'W': /* WVM */
/* Need to compare WVMRAW to WVMFIT */
switch (toupper(tempstr[3])) {
case 'R':
*tausrc = SMF__TAUSRC_WVMRAW;
break;
case 'F':
*tausrc = SMF__TAUSRC_WVMFIT;
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
break;
default:
*tausrc = SMF__TAUSRC_NULL;
}
} else {
*tausrc = SMF__TAUSRC_NULL;
}
}
if( extmeth ) {
if( astMapGet0C( keymap, "TAUMETHOD", &tempstr ) ) {
switch( toupper(tempstr[0]) ) {
case 'A':
*extmeth = SMF__EXTMETH_ADAPT;
break;
case 'Q':
*extmeth = SMF__EXTMETH_SINGLE;
break;
case 'F':
*extmeth = SMF__EXTMETH_FULL;
break;
default:
*extmeth = SMF__EXTMETH_NONE;
}
} else {
*extmeth = SMF__EXTMETH_NONE;
}
}
}
void smf_clean_smfArray( ThrWorkForce *wf, smfArray *array,
smfArray **noisemaps, smfArray **com, smfArray **gai,
AstKeyMap *keymap, int *status ) {
/* Local Variables */
double badfrac; /* Fraction of bad samples to flag bad bolo */
smfData *data=NULL; /* Pointer to individual smfData */
int compreprocess; /* COMmon-mode cleaning as pre-processing step */
dim_t dcfitbox; /* width of box for measuring DC steps */
int dclimcorr; /* Min number of correlated steps */
int dcmaxsteps; /* number of DC steps/min. to flag bolo bad */
dim_t dcsmooth; /* median filter width before finding DC steps */
double dcthresh; /* n-sigma threshold for primary DC steps */
int dofft; /* are we doing a freq.-domain filter? */
int dkclean; /* Flag for dark squid cleaning */
smfFilter *filt=NULL; /* Frequency domain filter */
double flagfast; /* Threshold for flagging slow slews */
double flagslow; /* Threshold for flagging slow slews */
dim_t idx; /* Index within subgroup */
size_t nflag; /* Number of elements flagged */
double noisecliphigh = 0; /* Sigma clip high-noise outlier bolos */
double noisecliplow = 0; /* Sigma clip low-noise outlier bolos */
int noiseclipprecom = 0; /* Noise clipping before common-mode cleaning? */
const char *opteff=NULL; /* Pointer to optical efficiency NDF file name*/
int opteffdiv; /* Divide data by the optical efficiencies? */
int order; /* Order of polynomial for baseline fitting */
char param[ 20 ]; /* Buffer for config parameter name */
dim_t pcalen; /* Chunk length for PCA cleaning */
double pcathresh; /* n-sigma threshold for PCA cleaning */
double spikethresh; /* Threshold for finding spikes */
dim_t spikebox=0; /* Box size for spike finder */
struct timeval tv1, tv2; /* Timers */
int whiten; /* Apply whitening filter? */
int zeropad; /* Pad with zeros? */
/* Main routine */
if (*status != SAI__OK) return;
/*** TIMER ***/
smf_timerinit( &tv1, &tv2, status );
/* Check for valid inputs */
if( !array || (array->ndat < 1) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": No data supplied", status );
}
if( array->sdata[0]->ndims != 3 ) {
*status = SMF__WDIM;
errRepf( "", FUNC_NAME ": Supplied smfData has %zu dims, needs 3", status,
data->ndims );
return;
}
if( !keymap ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL AstKeyMap supplied", status );
return;
}
/* Get cleaning parameters */
smf_get_cleanpar( keymap, array->sdata[0], &badfrac, &dcfitbox, &dcmaxsteps,
&dcthresh, &dcsmooth, &dclimcorr, &dkclean,
NULL, &zeropad, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, &flagslow, &flagfast, &order,
&spikethresh, &spikebox, &noisecliphigh, &noisecliplow,
NULL, &compreprocess, &pcalen, &pcathresh, NULL, NULL, NULL,
&noiseclipprecom, status );
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
/* Update quality by synchronizing to the data array VAL__BADD values */
msgOutif(MSG__VERB,"", FUNC_NAME ": update quality", status);
smf_update_quality( data, 1, NULL, 0, badfrac, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s updating quality",
status, smf_timerupdate(&tv1,&tv2,status) );
/* Fix DC steps */
if( dcthresh && dcfitbox ) {
msgOutiff(MSG__VERB, "", FUNC_NAME
": Flagging bolos with %lf-sigma DC steps in %" DIM_T_FMT " "
"samples as bad, using %" DIM_T_FMT
"-sample median filter and max %d "
"DC steps per min before flagging entire bolo bad...", status,
dcthresh, dcfitbox, dcsmooth, dcmaxsteps);
smf_fix_steps( wf, data, dcthresh, dcsmooth, dcfitbox, dcmaxsteps,
dclimcorr, 0, &nflag, NULL, NULL, status );
msgOutiff(MSG__VERB, "", FUNC_NAME": ...%zd flagged\n", status, nflag);
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s fixing DC steps",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Flag Spikes */
if( spikethresh ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": flag spikes...", status);
smf_flag_spikes( wf, data, SMF__Q_FIT, spikethresh, spikebox,
&nflag, status );
msgOutiff(MSG__VERB,"", FUNC_NAME ": ...found %zd", status, nflag );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s flagging spikes",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Flag periods of stationary pointing, and update scanspeed to more
accurate value */
if( flagslow || flagfast ) {
if( data->hdr && data->hdr->allState ) {
double scanvel=0;
if( flagslow ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": Flagging regions with slew speeds < %.2lf arcsec/sec",
status, flagslow );
}
if( flagfast ) {
msgOutiff( MSG__VERB, "", FUNC_NAME
": Flagging regions with slew speeds > %.2lf arcsec/sec",
status, flagfast );
/* Check to see if this was a sequence type that involved
motion. If not, skip this section */
if( data && data->hdr && (
(data->hdr->seqtype==SMF__TYP_SCIENCE) ||
(data->hdr->seqtype==SMF__TYP_POINTING) ||
(data->hdr->seqtype==SMF__TYP_FOCUS) ||
(data->hdr->seqtype==SMF__TYP_SKYDIP))
&& (data->hdr->obsmode!=SMF__OBS_STARE) ) {
smf_flag_slewspeed( data, flagslow, flagfast, &nflag, &scanvel,
status );
msgOutiff( MSG__VERB,"", "%zu new time slices flagged", status,
nflag);
if( msgIflev( NULL, status ) >= MSG__VERB ) {
msgOutf( "", FUNC_NAME ": mean SCANVEL=%.2lf arcsec/sec"
" (was %.2lf)", status, scanvel, data->hdr->scanvel );
}
data->hdr->scanvel = scanvel;
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s flagging outlier slew speeds",
status, smf_timerupdate(&tv1,&tv2,status) );
} else {
msgOutif( MSG__VERB, "", FUNC_NAME
": not a moving sequence or missing header, "
"skipping slew speed flagging", status );
}
}
} else {
msgOutif( MSG__DEBUG, "", FUNC_NAME
": Skipping flagslow/flagfast because no header present",
status );
}
}
/* Clean out the dark squid signal */
if( dkclean ) {
msgOutif(MSG__VERB, "", FUNC_NAME
": Cleaning dark squid signals from data.", status);
smf_clean_dksquid( data, 0, 100, NULL, 0, 0, 0, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s DKSquid cleaning",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Apply optical efficiency corrections. */
one_strlcpy( param, "OPTEFF", sizeof(param), status );
smf_find_subarray( data->hdr, param + strlen(param),
sizeof(param) - strlen(param), NULL, status );
astChrCase( NULL, param, 1, 0 );
if( astMapHasKey( keymap, param ) ) {
astMapGet0I( keymap, "OPTEFFDIV", &opteffdiv );
if ( astMapGet0C( keymap, param, &opteff ) ) {
msgOutiff( MSG__VERB,"", FUNC_NAME ": %s bolometer values "
"by factors read from NDF %s", status,
opteffdiv ? "Dividing" : "Multiplying", opteff );
smf_scale_bols( wf, data, NULL, opteff, param, opteffdiv, status );
}
}
/* Remove baselines */
if( order >= 0 ) {
msgOutiff( MSG__VERB,"", FUNC_NAME
": Fitting and removing %i-order polynomial baselines",
status, order );
smf_fit_poly( wf, data, order, 1, NULL, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s removing poly baseline",
status, smf_timerupdate(&tv1,&tv2,status) );
}
}
/* Mask noisy bolos here if happening before common-mode cleaning */
if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) &&
noiseclipprecom ) {
smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow,
zeropad, &tv1, &tv2, status );
}
/* Optionally call smf_calcmodel_com to perform a subset of the following
tasks as a pre-processing step:
- remove the common-mode
- flag outlier data using common-mode rejection
- determine relative flatfields using amplitude of common-mode
In order to do this we need to set up some temporary model container
files so that the routine can be called properly. All of the same
COMmon-mode and GAIn model parameters (e.g. com.* and gai.*) will be
used here. However, in addition the "compreprocess" flag must be set
for this operation to be performed. */
if( compreprocess ) {
smfArray *comdata = NULL;
smfGroup *comgroup = NULL;
smfDIMMData dat;
smfArray *gaidata = NULL;
smfGroup *gaigroup = NULL;
smfArray *quadata = NULL;
smfData *thisqua=NULL;
msgOutif(MSG__VERB," ", FUNC_NAME ": Remove common-mode", status);
/* Create model containers for COM, GAI */
smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__COM,
0, NULL, 0, NULL, NULL, &comgroup, &comdata, keymap,
status );
smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__GAI,
0, NULL, 0, NULL, NULL, &gaigroup, &gaidata, keymap,
status );
/* Manually create quadata to share memory with the quality already
stored in array */
quadata = smf_create_smfArray( status );
for( idx=0; (*status==SAI__OK) && (idx<array->ndat); idx++ ) {
/* Create several new smfDatas, but they will all be freed
properly when we close quadata */
thisqua = smf_create_smfData( SMF__NOCREATE_DA | SMF__NOCREATE_HEAD |
SMF__NOCREATE_FILE, status );
/* Probably only need pntr->[0], but fill in the dimensionality
information to be on the safe side */
thisqua->dtype = SMF__QUALTYPE;
thisqua->ndims = array->sdata[idx]->ndims;
thisqua->isTordered = array->sdata[idx]->isTordered;
memcpy( thisqua->dims, array->sdata[idx]->dims, sizeof(thisqua->dims) );
memcpy( thisqua->lbnd, array->sdata[idx]->lbnd, sizeof(thisqua->lbnd) );
thisqua->pntr[0] = smf_select_qualpntr( array->sdata[idx], NULL, status );
smf_addto_smfArray( quadata, thisqua, status );
}
/* Set up the smfDIMMData and call smf_calcmodel_com */
memset( &dat, 0, sizeof(dat) );
dat.res = &array;
dat.gai = &gaidata;
dat.qua = &quadata;
dat.noi = NULL;
smf_calcmodel_com( wf, &dat, 0, keymap, &comdata, SMF__DIMM_FIRSTITER,
status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME
": ** %f s removing common-mode",
status, smf_timerupdate(&tv1,&tv2,status) );
/* Clean up and/or return values */
if( com ) {
*com = comdata;
} else {
if( comdata ) smf_close_related( &comdata, status );
}
if( gai ) {
*gai = gaidata;
} else {
if( gaidata ) smf_close_related( &gaidata, status );
}
if( comgroup ) smf_close_smfGroup( &comgroup, status );
if( gaigroup ) smf_close_smfGroup( &gaigroup, status );
/* Before closing quadata unset all the pntr[0] since this is shared
memory with the quality associated with array */
if( quadata ) {
for( idx=0; idx<quadata->ndat; idx++ ) {
quadata->sdata[idx]->pntr[0] = NULL;
}
if( quadata ) smf_close_related( &quadata, status );
}
}
/* PCA cleaning */
if( pcathresh ) {
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
smf_clean_pca_chunks( wf, data, pcalen, pcathresh, keymap, status );
}
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s PCA cleaning",
status, smf_timerupdate(&tv1,&tv2,status) );
}
/* Allocate space for noisemaps if required */
if( noisemaps ) {
*noisemaps = smf_create_smfArray( status );
}
/* Loop over subarray */
for( idx=0; (idx<array->ndat)&&(*status==SAI__OK); idx++ ) {
data = array->sdata[idx];
/* Filter the data. Note that we call smf_filter_execute to apply
a per-bolometer whitening filter even if there is no
explicitly requested smfFilter (in which case the
smf_filter_fromkeymap call will leave the real/imaginary parts
of the filter as NULL pointers and they will get ignored inside
smf_filter_execute). */
filt = smf_create_smfFilter( data, status );
smf_filter_fromkeymap( filt, keymap, data->hdr, &dofft, &whiten, status );
if( (*status == SAI__OK) && dofft ) {
msgOutif( MSG__VERB, "", FUNC_NAME ": frequency domain filter", status );
smf_filter_execute( wf, data, filt, 0, whiten, status );
/*** TIMER ***/
msgOutiff( SMF__TIMER_MSG, "", FUNC_NAME ": ** %f s filtering data",
status, smf_timerupdate(&tv1,&tv2,status) );
}
filt = smf_free_smfFilter( filt, status );
/* Mask noisy bolos here if happening after common-mode cleaning */
if( (*status == SAI__OK) && ((noisecliphigh>0.0) || (noisecliplow>0.0)) &&
!noiseclipprecom ) {
smf__noisymask( wf, data, noisemaps, noisecliphigh, noisecliplow,
zeropad, &tv1, &tv2, status );
}
}
}
/* Main entry */
void smurf_fixsteps( int *status ) {
/* Local Variables */
AstKeyMap *keymap; /* Default config parameter values */
AstKeyMap *sub_instruments; /* Info about sub-instruments */
FILE *fd = NULL; /* File descriptor */
Grp *igrp = NULL; /* Input group of files */
Grp *ogrp = NULL; /* Output group of files */
dim_t dcfitbox; /* DCFITBOX config parameter */
dim_t dcsmooth; /* DCSMOOTH config parameter */
dim_t nx; /* Length of first pixel axis */
double dcthresh; /* DCTHRESH config parameter */
double sizetol; /* Tolerance allowed on step height */
int changed; /* Have any step fixes changed? */
int dclimcorr; /* DCLIMCORR config parameter */
int dcmaxsteps; /* DCMAXSTEPS config parameter */
int first; /* Index of first change to report */
int itemp; /* Intermediate value */
int meanshift; /* Use a mean shift filter? */
int nnew; /* Number of new step fixes */
int nold; /* Number of old step fixes */
size_t nrej; /* Number of rejected bolometers */
size_t outsize; /* Total number of NDF names in the output group */
size_t size; /* Number of files in input group */
smfData *data = NULL; /* Output smfData */
smfData *indata = NULL; /* Input smfData */
smfStepFix *newsteps = NULL; /* New step fix descriptions */
smfStepFix *oldsteps = NULL; /* Old step fix descriptions */
ThrWorkForce *wf = NULL; /* Pointer to a pool of worker threads */
/* Check inherited status */
if (*status != SAI__OK) return;
/* begin an NDF context. */
ndfBegin();
/* Get the name of the input NDF. */
kpg1Rgndf( "IN", 1, 1, "", &igrp, &size, status );
/* Get output file(s) */
kpg1Wgndf( "OUT", igrp, size, 0, "More output files required...",
&ogrp, &outsize, status );
/* Open the input data file, read-only. */
smf_open_file( igrp, 1, "Read", 0, &indata, status );
/* Since we will be modifying the data values, we need a deep copy. */
data = smf_deepcopy_smfData( indata, 0, 0, 0, 0, status );
/* Place cleaning parameters into a keymap and set defaults. Note that we
use the map-maker defaults file here so that we populate the locked
keymap with all the parameters that people may come across to allow
them to load their map-maker config directly this application. */
sub_instruments = smf_subinst_keymap( SMF__SUBINST_NONE, data, NULL, 0,
status );
keymap = kpg1Config( "CONFIG", "$SMURF_DIR/smurf_makemap.def",
sub_instruments, status );
sub_instruments = astAnnul( sub_instruments );
/* Set the default for each of the step fixing config parameters. */
astMapGet0I( keymap, "DCSMOOTH", &itemp );
parDef0i( "DCSMOOTH", itemp, status );
astMapGet0I( keymap, "DCFITBOX", &itemp );
parDef0i( "DCFITBOX", itemp, status );
astMapGet0I( keymap, "DCMAXSTEPS", &itemp );
parDef0i( "DCMAXSTEPS", itemp, status );
astMapGet0I( keymap, "DCLIMCORR", &itemp );
parDef0i( "DCLIMCORR", itemp, status );
astMapGet0D( keymap, "DCTHRESH", &dcthresh );
parDef0d( "DCTHRESH", dcthresh, status );
/* Get values for the config params */
parGet0i( "DCSMOOTH", &itemp, status );
dcsmooth = itemp;
parGet0i( "DCFITBOX", &itemp, status );
dcfitbox = itemp;
parGet0i( "DCMAXSTEPS", &itemp, status );
dcmaxsteps = itemp;
parGet0i( "DCLIMCORR", &itemp, status );
dclimcorr = itemp;
parGet0d( "DCTHRESH", &dcthresh, status );
parGet0l( "MEANSHIFT", &meanshift, status );
/* Find the number of cores/processors available and create a pool of
threads of the same size. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Fix the steps. */
smf_fix_steps( wf, data, dcthresh, dcsmooth, dcfitbox, dcmaxsteps,
dclimcorr, meanshift, &nrej, &newsteps, &nnew, status );
/* Display a summary of what was done by the step fixer. */
msgBlank( status );
if( nrej == 0 ) {
msgOut( "", "No bolometers were rejected", status );
} else if( nrej == 1 ) {
msgOut( "", "One bolometer was rejected", status );
} else {
msgSeti( "NREJ", nrej );
msgOut( "", "^NREJ bolometers were rejected", status );
}
parPut0i( "NREJECTED", nrej, status );
if( nnew == 0 ) {
msgOut( "", "No steps were fixed", status );
} else if( nnew == 1 ) {
msgOut( "", "One step was fixed", status );
} else {
msgSeti( "NNEW", nnew );
msgOut( "", "^NNEW steps were fixed", status );
}
parPut0i( "NFIXED", nnew, status );
/* If required, write out to a text file details of the steps that were
fixed. */
fd = smf_open_textfile( "NEWSTEPS", "w", "<none>", status );
if( fd ) {
smf1_write_steps( fd, indata, nnew, newsteps, dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr, nrej, status );
fclose( fd );
}
/* If required, create the output NDF. */
if( outsize > 0 && indata && indata->file ) {
smf_write_smfData( data, NULL, NULL, ogrp, 1,
indata->file->ndfid, MSG__VERB, 0, status );
}
/* Save the length of the first pixel axis. */
nx = data ? data->dims[ 0 ] : 0;
/* Close the NDFs. */
smf_close_file( &data, status );
smf_close_file( &indata, status );
/* Attempt to open a file containing descriptions of steps fixed by a
previous invocation of this program. */
fd = smf_open_textfile( "OLDSTEPS", "r", "<none>", status );
if( fd ) {
/* Get SIZETOL - the minimum significant fractional error in step sizes. */
parGet0d( "SIZETOL", &sizetol, status );
/* Read the contents of the file, issuing a warning if the global
properties read from the file (e.g. parameters used, no. of steps
found, etc) differ from those of the current invocation. */
msgBlank( status );
oldsteps = smf1_read_steps( fd, dcthresh, dcsmooth,
dcfitbox, dcmaxsteps, dclimcorr,
nrej, nnew, &nold, status );
/* Get the index of the first change to report. */
parGet0i( "FIRST", &first, status );
/* Compare the new step fixes with the old step fixes, issuing a warning
for the first step fix that has changed. */
changed = smf1_check_steps( "CONTINUE", first, nx, sizetol,
nold, nnew, oldsteps, newsteps, status );
/* Store a flag indicating if any sstep fixes have chnaged. */
parPut0l( "CHANGED", changed, status );
/* Tell the user if nothing has changed. */
if( ! changed ) {
msgOut( "", "There are no significant differences "
"between old and new step fixes.", status );
}
msgBlank( status );
/* Close the old steps file, and free the memory holding the old step
descriptions. */
fclose( fd );
oldsteps = astFree( oldsteps );
}
/* Free resources. */
newsteps = astFree( newsteps );
grpDelet( &igrp, status );
grpDelet( &ogrp, status );
/* End the NDF context. */
ndfEnd( status );
/* If anything went wrong issue a context message. */
if( *status != SAI__OK ) msgOutif( MSG__VERB, " ", "FIXSTEPS failed.",
status );
}
int cupidConfigI( AstKeyMap *config, const char *name, int def,
int *status ){
/*
*+
* Name:
* cupidConfigI
* Purpose:
* Get the value of a configuration parameter.
* Language:
* Starlink C
* Synopsis:
* int cupidConfigI( AstKeyMap *config, const char *name, int def,
* int *status )
* Description:
* This function returns a named value from the supplied KeyMap. If
* the KeyMap does not contain the named value, an attempt is made to
* obtain a value from a secondary KeyMap which should be stored
* within the supplied KeyMap, using a key equal to the constant
* string CUPID__CONFIG. If the secondary KeyMap does not contain a
* value, then the supplied default value is returned. In either case,
* the returned value is stored in the KeyMap.
* Parameters:
* config
* An AST KeyMap holding the configuration parameters. If NULL is
* supplied, the default value is returned without error.
* name
* The name of the value to extract from the KeyMap.
* def
* The default value to use.
* status
* Pointer to the inherited status value.
* Returned Value:
* The required value. The supplied default value is returned if an
* error occurs.
* Copyright:
* Copyright (C) 2005 Particle Physics & Astronomy Research Council.
* Copyright (C) 2008 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street,Fifth Floor, Boston, MA
* 02110-1301, USA
* Authors:
* DSB: David S. Berry
* {enter_new_authors_here}
* History:
* 5-OCT-2005 (DSB):
* Original version.
* 26-MAR-2008 (DSB):
* Re-report a more friendly message iof the supplied text could
* not be interpreted as a numerical value.
* {enter_further_changes_here}
* Bugs:
* {note_any_bugs_here}
*-
*/
/* Local Variables: */
AstObject *sconfig; /* Object pointer obtained from KeyMap */
const char *text; /* Pointer to uninterprted text value */
int ret; /* The returned value */
/* Initialise */
ret = def;
/* Abort if an error has already occurred, or if no KeyMap was supplied. */
if( *status != SAI__OK || !config ) return ret;
/* Attempt to extract the named value from the supplied KeyMap. */
if( !astMapGet0I( config, name, &ret ) ) {
/* If the value was not found in the KeyMap, see if the KeyMap contains a
secondary KeyMap. */
if( astMapGet0A( config, CUPID__CONFIG, &sconfig ) ) {
/* If it does, see if the secondary KayMap contains a value for the named
entry. If it does, remove the value from the KeyMap so it does not
appear in the CUPID NDF extension. */
if( astMapGet0I( (AstKeyMap *) sconfig, name, &ret ) ) {
astMapRemove( (AstKeyMap *) sconfig, name );
/* If the text supplied by the user could not be interpreted as an
integer value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
/* If the value was not found in either KeyMap, return the default value. */
} else {
ret = def;
}
/* Free the pointer to the secondary KeyMap. */
sconfig = astAnnul( sconfig );
/* If no secondary KeyMap was found, return the default value. */
} else {
ret = def;
}
/* Store the returned value in the supplied KeyMap if it is good. */
if( ret != VAL__BADI ) astMapPut0I( config, name, ret, NULL );
/* If the text supplied by the user could not be interpreted as an
integer value, re-report the error. */
} else if( *status == AST__MPGER ) {
ret = def;
errAnnul( status );
if( astMapGet0C( config, name, &text ) ) {
*status = SAI__ERROR;
msgSetc( "T", text );
msgSetc( "N", name );
errRep( "", "Illegal value \"^T\" supplied for configuration "
"parameter ^N.", status );
}
}
/* Return the result. */
return ret;
}
dim_t smf_get_padding( AstKeyMap *keymap, int report, const smfHead *hdr,
double steptime, int *status ) {
/* Local Variables: */
AstObject *obj;
const char *key;
dim_t pad;
dim_t result;
double f_low;
double filt_edgehigh;
double filt_edgelarge;
double filt_edgelow;
double filt_edgesmall;
double filt_notchlow[ SMF__MXNOTCH ];
int f_nnotch;
int iel;
int nel;
int temp;
double scalelen;
double downsampscale;
double downsampfreq;
/* Initialise */
result = 0;
/* Main routine */
if (*status != SAI__OK) return result;
/* If the keymap contains a PAD value, return it. */
if( astMapGet0I( keymap, "PAD", &temp ) ) {
if( temp < 0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "PAD cannot be < 0.", status );
} else {
result = (dim_t) temp;
}
/* If the keymap contains no PAD value, calculate a default on the basis
of the FILT_ values. */
} else {
/* If the keymap does not contain a PAD value, not even an "<undef>"
value, an error will have been reported by astMapget0I above. In this
case, annull the error and continue since the KeyMap may contain FILT_
values that can be used to calculate a default PAD value. */
if( *status == AST__MPKER ) errAnnul( status );
/* Search for filtering parameters in the keymap. None of these parameters
represent a number of time clies, so we can set the smfData (the 2nd
argument) to NULL. */
f_nnotch = 0;
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, &filt_edgelow, &filt_edgehigh, &filt_edgesmall,
&filt_edgelarge, filt_notchlow, NULL, &f_nnotch, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, &downsampscale, &downsampfreq, NULL,
status );
if( downsampscale && downsampfreq ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": both downsampscale and downsampfreq are set",
status );
}
/* If any were not found, annul the error and continue with those that
were found (plus defaults for the others). */
if( *status == AST__MPKER ) errAnnul( status );
if( *status == SAI__OK ) {
/* Modify edge filters if spatial scales were requested */
smf_scale2freq( filt_edgesmall, filt_edgelarge, hdr, &filt_edgelow,
&filt_edgehigh, status );
if( *status == SMF__TELSTAT ) {
errAnnul( status );
}
/* Find the lowest of these frequencies. The lowest frequency will give
the greatest padding. */
f_low = ( filt_edgehigh > 0.0 ) ? filt_edgehigh : VAL__MAXD;
if( filt_edgelow > 0.0 && filt_edgelow < f_low ) f_low = filt_edgelow;
for( iel = 0; iel < f_nnotch; iel++ ) {
if( filt_notchlow[ iel ] > 0.0 && filt_notchlow[ iel ] < f_low ) f_low = filt_notchlow[ iel ];
}
/* If any down-sampling is occurring, set the step time to value implied by
the requested downsampling factor. This should match the calculation in
smf_grp_related */
if( downsampscale || downsampfreq ) {
if( downsampscale ) {
scalelen = hdr->steptime * hdr->scanvel / downsampscale;
} else {
if( hdr->steptime ) {
scalelen = downsampfreq / (1./hdr->steptime);
} else {
*status = SAI__ERROR;
scalelen = VAL__BADD;
errRep( "", FUNC_NAME ": can't resample because smfData has "
"unknown sample rate", status );
}
}
if( (*status==SAI__OK) && (scalelen<=SMF__DOWNSAMPLIMIT) )
steptime = hdr->steptime/scalelen;
/* If no downsampling is happening, use the steptime in the header unless
another steptime was supplied. */
} else if( steptime == VAL__BADD ) {
steptime = hdr->steptime;
}
/* Find the corresponding padding. */
if( f_low != VAL__MAXD ) {
result = 1.0/( steptime * f_low );
} else {
result = 0;
}
/* Now check the supplied keymap for any nested keymaps. Assumes that each
entry in the supplied KeyMap contain either a primitive value or a scalar
KeyMap pointer. */
nel = astMapSize( keymap );
for( iel = 0; iel < nel; iel++ ) {
key = astMapKey( keymap, iel );
/* If this entry is a KeyMap (assuming no other class of AST object is
stored in the KeyMap)... */
if( astMapType( keymap, key ) == AST__OBJECTTYPE ) {
/* Get a pointer to the KeyMap. */
(void) astMapGet0A( keymap, key, &obj );
/* Call this function to get the padding implied by the sub-KeyMap. */
pad = smf_get_padding( (AstKeyMap *) obj, report, hdr,
steptime, status );
/* Use the larger of the two paddings. */
if( pad > result ) result = pad;
}
}
/* If required, report the default value. */
if( report ) {
msgSeti( "P", (int) result );
msgOutif( MSG__VERB, "", "Padding each time stream with ^P zero "
"values at start and end.", status );
}
}
}
/* Return the result. */
return result;
}
void smf_addpolanal( AstFrameSet *fset, smfHead *hdr, AstKeyMap *config,
int *status ){
/* Local Variables */
AstCmpMap *tmap;
AstFrame *cfrm;
AstFrame *pfrm;
AstFrame *tfrm;
AstFrameSet *tfs;
AstPermMap *pm;
char *polnorth = NULL;
const char *cursys;
const char *trsys;
int aloff;
int icurr;
int inperm[2];
int outperm[2];
int pol2fp;
/* Check inherited status, and also check the supplied angle is not bad. */
if( *status != SAI__OK ) return;
/* Begin an AST object context. */
astBegin;
/* Get the value of the POLNORTH FITS keyword from the supplied header.
The rest only happens if the keyword is found. */
if( astGetFitsS( hdr->fitshdr, "POLNORTH", &polnorth ) ) {
/* Normally, we do not allow maps to be made from Q/U time streams that
use focal plane Y as the reference direction (because of the problems
of sky rotation). Therefore we report an error. However, we do need to
make such maps as part of the process of determining the parameters of
the Instrumental Polarisation (IP) model. So only report the error if
"pol2fp" config parameter is non-zero. */
if( !strcmp( polnorth, "FPLANE" ) ) {
astMapGet0I( config, "POL2FP", &pol2fp );
if( pol2fp ) {
msgBlank( status );
msgOut( "", "WARNING: The input NDFs hold POL-2 Q/U data specified "
"with respect to focal plane Y.",status );
msgOut( "", "Maps should normally be made from POL-2 data specified "
"with respect to celestial north.",status );
msgOut( "", "The output map will not contain a POLANAL Frame and "
"so will be unusable by POLPACK applications.",status );
msgBlank( status );
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "The input NDFs hold POL-2 Q/U data specified with "
"respect to focal plane Y.",status );
errRep( "", "Maps can only be made from POL-2 data specified with "
"respect to celestial north.",status );
}
/* If the ref. direction is celestial north, create a suitable Frame and
Mapping and add them into the supplied FrameSet. */
} else {
/* Check the current Frame is a SkyFrame. */
cfrm = astGetFrame( fset, AST__CURRENT );
if( astIsASkyFrame( cfrm ) ) {
/* Create a POLANAL Frame. */
pfrm = astFrame( 2, "Domain=POLANAL" );
astSet( pfrm, "Title=Polarimetry reference frame" );
astSet( pfrm, "Label(1)=Polarimetry reference direction" );
astSet( pfrm, "Label(2)=" );
/* Create a PermMap that ensures that axis 1 of the POLANAL Frame is parallel
to the latitude axis (i.e. north) of the curent Frame (the current Frame axes
may have been swapped). */
outperm[ 0 ] = astGetI( cfrm, "LatAxis" );
outperm[ 1 ] = astGetI( cfrm, "LonAxis" );
inperm[ outperm[ 0 ] - 1 ] = 1;
inperm[ outperm[ 1 ] - 1 ] = 2;
pm = astPermMap( 2, inperm, 2, outperm, NULL, " " );
/* Record the index of the original current Frame. */
icurr = astGetI( fset, "Current" );
/* Determine the system to use. */
if( !strcmp( polnorth, "TRACKING" ) ) {
trsys = sc2ast_convert_system( hdr->state->tcs_tr_sys, status );
} else {
trsys = polnorth;
}
/* If the current Frame in the supplied FrameSet has this system. Then we
use the above PermMap to connect the POLANAL Frame directly to the current
Frame. */
cursys = astGetC( cfrm, "System" );
if( trsys && cursys && !strcmp( cursys, trsys ) ) {
astAddFrame( fset, AST__CURRENT, pm, pfrm );
/* Otherwise we need to get a Mapping from the current Frame to the
required frame. */
} else {
/* Take a copy of the current Frame (in order to pick up epoch, observatory
position, etc), and set its System to the required system. */
tfrm = astCopy( cfrm );
astSetC( tfrm, "System", trsys );
/* Get the Mapping from the original current Frame to this modified copy.
Ensure alignment happens in absolute coords (alignment in offset
coords is always a unit mapping and so no rotation occurs). */
aloff = astGetI( cfrm, "AlignOffset" );
if( aloff ) {
astSetI( cfrm, "AlignOffset", 0 );
astSetI( tfrm, "AlignOffset", 0 );
}
tfs = astConvert( cfrm, tfrm, "SKY" );
if( aloff ) {
astSetI( cfrm, "AlignOffset", 1 );
astSetI( tfrm, "AlignOffset", 1 );
}
if( tfs ) {
/* Use it, in series with with the above PermMap, to connect the POLANAL frame
to the current Frame. */
tmap = astCmpMap( astGetMapping( tfs, AST__BASE,
AST__CURRENT ),
pm, 1, " " );
astAddFrame( fset, AST__CURRENT, astSimplify( tmap ),
pfrm );
/* Report an error if the mapping from current to required system could
not be found. */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "smf_addpolanal: Could not convert Frame "
"from %s to %s (prgramming error).", status,
cursys, trsys );
}
}
/* Re-instate the original current Frame. */
astSetI( fset, "Current", icurr );
/* Report an error if the current Frame is not a SkyFrame. */
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "smf_addpolanal: The current Frame in the "
"supplied FrameSet is not a SkyFrame (prgramming "
"error).", status );
}
}
}
/* End the AST object context. */
astEnd;
}
void smf_calc_mapcoord( ThrWorkForce *wf, AstKeyMap *config, smfData *data,
AstFrameSet *outfset, int moving, int *lbnd_out,
int *ubnd_out, fts2Port fts_port, int flags,
int *status ) {
/* Local Variables */
AstSkyFrame *abskyfrm = NULL;/* Output SkyFrame (always absolute) */
AstMapping *bolo2map=NULL; /* Combined mapping bolo->map coordinates */
int bndndf=NDF__NOID; /* NDF identifier for map bounds */
void *data_pntr[1]; /* Array of pointers to mapped arrays in ndf */
int *data_index; /* Mapped DATA_ARRAY part of NDF */
int docalc=1; /* If set calculate the LUT */
int doextension=0; /* Try to write LUT to MAPCOORD extension */
smfFile *file=NULL; /* smfFile pointer */
AstObject *fstemp = NULL; /* AstObject version of outfset */
int ii; /* loop counter */
int indf_lat = NDF__NOID; /* Identifier for NDF to receive lat values */
int indf_lon = NDF__NOID; /* Identifier for NDF to receive lon values */
smfCalcMapcoordData *job_data=NULL; /* Array of job */
int lbnd[1]; /* Pixel bounds for 1d pointing array */
int lbnd_old[2]; /* Pixel bounds for existing LUT */
int lbnd_temp[1]; /* Bounds for bounds NDF component */
int lutndf=NDF__NOID; /* NDF identifier for coordinates */
AstMapping *map2sky_old=NULL;/* Existing mapping map->celestial coord. */
HDSLoc *mapcoordloc=NULL; /* HDS locator to the MAPCOORD extension */
int nw; /* Number of worker threads */
AstFrameSet *oldfset=NULL; /* Pointer to existing WCS info */
AstSkyFrame *oskyfrm = NULL; /* SkyFrame from the output WCS Frameset */
smfCalcMapcoordData *pdata=NULL; /* Pointer to job data */
double *lat_ptr = NULL; /* Pointer to array to receive lat values */
double *lon_ptr = NULL; /* Pointer to array to receive lon values */
int ubnd[1]; /* Pixel bounds for 1d pointing array */
int ubnd_old[2]; /* Pixel bounds for existing LUT */
int ubnd_temp[1]; /* Bounds for bounds NDF component */
int *lut = NULL; /* The lookup table */
dim_t nbolo=0; /* Number of bolometers */
dim_t ntslice=0; /* Number of time slices */
int nmap; /* Number of mapped elements */
AstMapping *sky2map=NULL; /* Mapping celestial->map coordinates */
size_t step; /* step size for dividing up work */
AstCmpMap *testcmpmap=NULL; /* Combined forward/inverse mapping */
AstMapping *testsimpmap=NULL;/* Simplified testcmpmap */
double *theta = NULL; /* Scan direction at each time slice */
int tstep; /* Time slices between full Mapping calculations */
int exportlonlat; /* Dump longitude and latitude values? */
/* Main routine */
if (*status != SAI__OK) return;
/* How many threads do we get to play with */
nw = wf ? wf->nworker : 1;
/* Initialize bounds to avoid compiler warnings */
lbnd_old[0] = 0;
lbnd_old[1] = 0;
ubnd_old[0] = 0;
ubnd_old[1] = 0;
/* Check for pre-existing LUT and de-allocate it. This will only waste
time if the MAPCOORD extension is found to be valid and it has
to be re-loaded from disk. */
smf_close_mapcoord( data, status );
/* Assert ICD data order */
smf_dataOrder( data, 1, status );
/* Get the data dimensions */
smf_get_dims( data, NULL, NULL, &nbolo, &ntslice, NULL, NULL, NULL, status );
/* If SMF__NOCREATE_FILE is not set, and file associated with an NDF,
map a new MAPCOORD extension (or verify an existing one) */
if( !(flags & SMF__NOCREATE_FILE) && data->file ) {
doextension = 1;
} else {
doextension = 0;
docalc = 1;
}
/* Create / check for existing MAPCOORD extension */
if( doextension ) {
file = data->file;
/* Check type of file before proceeding */
if( file->isSc2store ) {
*status = SAI__ERROR;
errRep(FUNC_NAME,
"File was opened by sc2store library (raw data?)",
status);
}
if( !file->isTstream ) {
*status = SAI__ERROR;
errRep(FUNC_NAME, "File does not contain time stream data",status);
}
/* Get HDS locator to the MAPCOORD extension */
mapcoordloc = smf_get_xloc( data, "MAPCOORD", "MAP_PROJECTION", "UPDATE",
0, 0, status );
/* Obtain NDF identifier/placeholder for LUT in MAPCOORD extension*/
lbnd[0] = 0;
ubnd[0] = nbolo*ntslice-1;
lutndf = smf_get_ndfid( mapcoordloc, "LUT", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd, ubnd, status );
if( *status == SAI__OK ) {
/* store the NDF identifier */
file->mapcoordid = lutndf;
/* Create sky to output grid mapping using the base coordinates to
get the coordinates of the tangent point if it hasn't been done
yet. */
sky2map = astGetMapping( outfset, AST__CURRENT, AST__BASE );
}
/* Before mapping the LUT, first check for existing WCS information
and LBND/UBND for the output map. If they are already correct don't
bother re-calculating the LUT! */
if( *status == SAI__OK ) {
/* Try reading in the WCS information */
kpg1Wread( mapcoordloc, "WCS", &fstemp, status );
oldfset = (AstFrameSet*)fstemp;
if( *status == SAI__OK ) {
/* Check that the old and new mappings are the same by
checking that combining one with the inverse of the other
reduces to a UnitMap. */
map2sky_old = astGetMapping( oldfset, AST__BASE, AST__CURRENT );
testcmpmap = astCmpMap( map2sky_old, sky2map, 1, " " );
testsimpmap = astSimplify( testcmpmap );
if( astIsAUnitMap( testsimpmap ) ) {
/* The mappings are the same, now just check the pixel
bounds in the output map */
lbnd_temp[0] = 1;
ubnd_temp[0] = 2;
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lbnd_old[0] = data_index[0];
lbnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "READ", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp,
status );
if( *status == SAI__OK ) {
ndfMap( bndndf, "DATA", "_INTEGER", "READ", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
ubnd_old[0] = data_index[0];
ubnd_old[1] = data_index[1];
}
ndfAnnul( &bndndf, status );
}
if( *status == SAI__OK ) {
/* If we get this far finally do the bounds check! */
if( (lbnd_old[0] == lbnd_out[0]) &&
(lbnd_old[1] == lbnd_out[1]) &&
(ubnd_old[0] == ubnd_out[0]) &&
(ubnd_old[1] == ubnd_out[1]) ) {
docalc = 0; /* We don't have to re-calculate the LUT */
msgOutif(MSG__VERB," ",FUNC_NAME ": Existing LUT OK",
status);
}
}
}
/* Bad status / AST errors at this point due to problems with
MAPCOORD. Annul and continue calculating new MAPCOORD extension. */
astClearStatus;
errAnnul(status);
} else {
/* Bad status due to non-existence of MAPCOORD. Annul and continue */
errAnnul(status);
}
}
}
/* If we need to calculate the LUT do it here */
if( docalc && (*status == SAI__OK) ) {
msgOutif(MSG__VERB," ", FUNC_NAME ": Calculate new LUT",
status);
/* Get the increment in time slices between full Mapping calculations.
The Mapping for intermediate time slices will be approximated. */
dim_t dimval;
smf_get_nsamp( config, "TSTEP", data, &dimval, status );
tstep = dimval;
/* Get space for the LUT */
if( doextension ) {
/* Map the LUT array */
ndfMap( lutndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
lut = data_index;
} else {
errRep( FUNC_NAME, "Unable to map LUT in MAPCOORD extension",
status);
}
} else {
/* alloc the LUT and THETA arrays */
lut = astMalloc( (nbolo*ntslice)*sizeof(*(data->lut)) );
theta = astMalloc( ntslice*sizeof(*(data->theta)) );
}
/* Retrieve the sky2map mapping from the output frameset (actually
map2sky) */
oskyfrm = astGetFrame( outfset, AST__CURRENT );
sky2map = astGetMapping( outfset, AST__BASE, AST__CURRENT );
/* If the longitude and latitude is being dumped, create new NDFs to
hold them, and map them. */
if( config ) {
astMapGet0I( config, "EXPORTLONLAT", &exportlonlat );
if( exportlonlat ) {
lon_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lon, 1, status );
lat_ptr = smf1_calc_mapcoord1( data, nbolo, ntslice, oskyfrm,
&indf_lat, 2, status );
}
}
/* Invert the mapping to get Output SKY to output map coordinates */
astInvert( sky2map );
/* Create a SkyFrame in absolute coordinates */
abskyfrm = astCopy( oskyfrm );
astClear( abskyfrm, "SkyRefIs" );
astClear( abskyfrm, "SkyRef(1)" );
astClear( abskyfrm, "SkyRef(2)" );
if( *status == SAI__OK ) {
/* --- Begin parellelized portion ------------------------------------ */
/* Start a new job context. Each call to thrWait within this
context will wait until all jobs created within the context have
completed. Jobs created in higher contexts are ignored by thrWait. */
thrBeginJobContext( wf, status );
/* Allocate job data for threads */
job_data = astCalloc( nw, sizeof(*job_data) );
if( *status == SAI__OK ) {
/* Set up job data, and start calculating pointing for blocks of
time slices in different threads */
if( nw > (int) ntslice ) {
step = 1;
} else {
step = ntslice/nw;
}
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
/* Blocks of time slices */
pdata->t1 = ii*step;
pdata->t2 = (ii+1)*step-1;
/* Ensure that the last thread picks up any left-over tslices */
if( (ii==(nw-1)) && (pdata->t1<(ntslice-1)) ) {
pdata->t2=ntslice-1;
}
pdata->ijob = -1;
pdata->lut = lut;
pdata->theta = theta;
pdata->lbnd_out = lbnd_out;
pdata->moving = moving;
pdata->ubnd_out = ubnd_out;
pdata->tstep = tstep;
pdata->lat_ptr = lat_ptr;
pdata->lon_ptr = lon_ptr;
pdata->fts_port = fts_port;
/* Make deep copies of AST objects and unlock them so that each
thread can then lock them for their own exclusive use */
pdata->abskyfrm = astCopy( abskyfrm );
astUnlock( pdata->abskyfrm, 1 );
pdata->sky2map = astCopy( sky2map );
astUnlock( pdata->sky2map, 1 );
/* Similarly, make a copy of the smfData, including only the header
information which each thread will need in order to make calls to
smf_rebin_totmap */
pdata->data = smf_deepcopy_smfData( data, 0, SMF__NOCREATE_FILE |
SMF__NOCREATE_DA |
SMF__NOCREATE_FTS |
SMF__NOCREATE_DATA |
SMF__NOCREATE_VARIANCE |
SMF__NOCREATE_QUALITY, 0, 0,
status );
smf_lock_data( pdata->data, 0, status );
}
for( ii=0; ii<nw; ii++ ) {
/* Submit the job */
pdata = job_data + ii;
pdata->ijob = thrAddJob( wf, THR__REPORT_JOB, pdata,
smfCalcMapcoordPar, 0, NULL, status );
}
/* Wait until all of the jobs submitted within the current job
context have completed */
thrWait( wf, status );
}
/* End the current job context. */
thrEndJobContext( wf, status );
/* --- End parellelized portion -------------------------------------- */
/* Set the lut pointer in data to the buffer */
data->lut = lut;
data->theta = theta;
/* Write the WCS for the projection to the extension */
if( doextension ) {
kpg1Wwrt( (AstObject*)outfset, "WCS", mapcoordloc, status );
/* Write the pixel bounds for the map to the extension */
lbnd_temp[0] = 1; /* Don't get confused! Bounds for NDF that will */
ubnd_temp[0] = 2; /* contain the bounds for the output 2d map! */
bndndf = smf_get_ndfid( mapcoordloc, "LBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = lbnd_out[0];
data_index[1] = lbnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map LBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
bndndf = smf_get_ndfid( mapcoordloc, "UBND", "UPDATE", "UNKNOWN",
"_INTEGER", 1, lbnd_temp, ubnd_temp, status );
ndfMap( bndndf, "DATA", "_INTEGER", "WRITE", data_pntr, &nmap,
status );
data_index = data_pntr[0];
if( *status == SAI__OK ) {
data_index[0] = ubnd_out[0];
data_index[1] = ubnd_out[1];
} else {
errRep( FUNC_NAME, "Unable to map UBND in MAPCOORD extension",
status);
}
ndfAnnul( &bndndf, status );
}
}
}
/* Clean Up */
if( testsimpmap ) testsimpmap = astAnnul( testsimpmap );
if( testcmpmap ) testcmpmap = astAnnul( testcmpmap );
if( map2sky_old ) map2sky_old = astAnnul( map2sky_old );
if( oldfset ) oldfset = astAnnul( oldfset );
if (sky2map) sky2map = astAnnul( sky2map );
if (bolo2map) bolo2map = astAnnul( bolo2map );
if( abskyfrm ) abskyfrm = astAnnul( abskyfrm );
if( oskyfrm ) oskyfrm = astAnnul( oskyfrm );
if( mapcoordloc ) datAnnul( &mapcoordloc, status );
if( indf_lat != NDF__NOID ) ndfAnnul( &indf_lat, status );
if( indf_lon != NDF__NOID ) ndfAnnul( &indf_lon, status );
/* If we get this far, docalc=0, and status is OK, there must be
a good LUT in there already. Map it so that it is accessible to
the caller; "UPDATE" so that the caller can modify it if desired. */
if( (*status == SAI__OK) && (docalc == 0) ) {
smf_open_mapcoord( data, "UPDATE", status );
}
/* Clean up job data */
if( job_data ) {
for( ii=0; (*status==SAI__OK)&&(ii<nw); ii++ ) {
pdata = job_data + ii;
if( pdata->data ) {
smf_lock_data( pdata->data, 1, status );
smf_close_file( &(pdata->data), status );
}
astLock( pdata->abskyfrm, 0 );
pdata->abskyfrm = astAnnul( pdata->abskyfrm );
astLock( pdata->sky2map, 0 );
pdata->sky2map = astAnnul( pdata->sky2map );
}
job_data = astFree( job_data );
}
}
static AstTable *ReadNextTable( FILE *fd, const char *fname, int *iline,
int *status ) {
/*
* Name:
* ReadOneTable
* Purpose:
* Reads a single Table from a text file.
* Description:
* This function reads text from the supplied file descriptor until it
* reaches the end of file or encounters an end-of-table marker (a line
* consisting just of two or more minus signs with no leading spaces).
* It creates an AstTable from the text and returns a pointer to it.
* Arguments:
* fd
* The file descriptor.
* fname
* The file name - used for error messages.
* iline
* Pointer to an int holding the number of lines read from the
* file so far. Updated on exit to include the lines read by the
* invocation of this function.
* status
* Pointer to the global status variable.
* Returned Value:
* A pointer to the Table read from the file, or NULL if an error occurs.
*/
/* Local Variables: */
AstTable *result;
AstTable *subtable;
char **cols;
char **words;
char *last_com;
char *line;
char *p;
char *tname;
char key[ 200 ];
const char *cval;
const char *oldname;
const char *newname;
double dval;
int *types;
int blank;
int c;
int com;
int eot;
int first;
int icol;
int irow;
int ival;
int iword;
int line_len;
int max_line_len;
int more;
int nc;
int ncol;
int nrow;
int nword;
int skip;
size_t len;
/* Initialise */
result = NULL;
/* Check the inherited status. */
if( *status != SAI__OK ) return result;
/* Create an empty Table. */
result = astTable( " " );
/* Allocate a buffer for one one line of text. This will be increased in
size as required. */
max_line_len = 80;
line = astMalloc( max_line_len*sizeof( *line ) );
/* Read each line of text from the file. */
eot = 0;
skip = 1;
line_len = 0;
more = 1;
last_com = NULL;
cols = NULL;
first = 1;
irow = 0;
types = NULL;
while( more && *status == SAI__OK ) {
(*iline)++;
line_len = 0;
/* Loop reading characters from the file until a newline or the end of file is
reached. */
while( ( c = fgetc( fd ) ) != EOF && c != '\n' ) {
/* Increment the current line length, and double the size of the line buffer
if it is full. */
if( ++line_len >= max_line_len ) {
max_line_len *= 2;
line = astRealloc( line, max_line_len*sizeof( *line ) );
if( *status != SAI__OK ) break;
}
/* Store the character. Ignore leading white space. */
if( skip ) {
if( ! isspace( c ) ) {
line[ line_len - 1 ] = c;
skip = 0;
} else {
line_len--;
}
} else {
line[ line_len - 1 ] = c;
}
}
/* If the end-of-file was reached indicate that we should leave the main
loop after processing the current line. */
if( c == EOF ) more = 0;
/* Terminate the line. */
line[ line_len ] = 0;
/* Terminate it again to exclude trailing white space. */
line[ astChrLen( line ) ] = 0;
/* Assume the line is a blank non-comment, and store a pointer to the first
character to use. */
blank = 1;
com = 0;
p = line;
/* Skip blank lines. */
if( line[ 0 ] ) {
/* If the line starts with a comment character... */
if( line[ 0 ] == '#' || line[ 0 ] == '!' ) {
com = 1;
/* Get a pointer to the first non-space/tab character after the comment
character. */
p = line + 1;
while( *p == ' ' || *p == '\t' ) p++;
/* Note if it is blank. */
if( *p ) blank = 0;
/* If it is not a comment line, then the line is not blank. */
} else {
blank = 0;
/* See if it is the end-of-table marker - a line containing just two or
more minus signs with no leading spaces. */
eot = ( strspn( line, "-" ) > 1 );
}
}
/* Skip blank lines, whether comment or not. */
if( ! blank ) {
/* First handle comment lines. */
if( com ) {
/* Does it look like a parameter assignment... */
words = astChrSplitRE( p, "^\\s*(\\w+)\\s*=\\s*(.*)$", &nword, NULL );
if( words ) {
/* Add a parameter declaration to the Table. */
astAddParameter( result, words[ 0 ] );
/* Store the parameter value, using an appropriate data type. */
len = strlen( words[ 1 ] );
if( nc = 0, ( 1 == astSscanf( words[ 1 ], "%d%n", &ival, &nc ) )
&& ( nc >= len ) ) {
astMapPut0I( result, words[ 0 ], ival, NULL );
} else if( nc = 0, ( 1 == astSscanf( words[ 1 ], "%lg%n", &dval, &nc ) )
&& ( nc >= len ) ) {
astMapPut0D( result, words[ 0 ], dval, NULL );
} else {
astMapPut0C( result, words[ 0 ], words[ 1 ], NULL );
}
/* Free the words returned by astChrSplitRE. */
for( iword = 0; iword < nword; iword++ ) {
words[ iword ] = astFree( words[ iword ] );
}
words = astFree( words );
/* If it does not look like a parameter assignment... */
} else {
/* Save a copy of it in case it turns out to be the last non-blank comment
line before the first row of data values (in which case it should
contain the column names). */
last_com = astStore( last_com, p, strlen( p ) + 1 );
}
/* If the line is not a comment see if it is an end of table marker. If so
indicate that we should leave the loop. */
} else if( eot ) {
more = 0;
/* If the line is not a comment or an end of table marker ... */
} else {
/* Get the words from the row. */
words = astChrSplit( p, &nword );
/* If this is the first non-blank non-comment line, get the column names from
the previous non-blank comment line. */
if( first ) {
if( last_com ) {
first = 0;
cols = astChrSplit( last_com, &ncol );
/* Create an array to hold the data type for each colum, and initialise
them to "integer". */
types = astMalloc( ncol*sizeof( int ) ) ;
for( iword = 0; iword < nword && astOK; iword++ ) {
if( iword < ncol ) {
types[ iword ] = AST__INTTYPE;
/* The columns are stored initially using interim names which have "T_"
prepended to the names given in the file. */
tname = NULL;
nc = 0;
tname = astAppendString( tname, &nc, "T_" );
tname = astAppendString( tname, &nc, cols[ iword ] );
astFree( cols[ iword ] );
cols[ iword ] = tname;
/* Create the column definition within the returned Table. We store them
initially as strings and then convert to the appropriate column data type
later (once all rows have been read and the the data types are known). */
astAddColumn( result, cols[ iword ], AST__STRINGTYPE,
0, NULL, " " );
}
}
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetc( "F", fname );
errRep( " ", "No column headers found in file ^F.", status );
}
}
/* Report an error if the line has the wrong number of values. */
if( nword != ncol ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSeti( "N", nword );
msgSeti( "I", (*iline) );
msgSeti( "M", ncol );
msgSetc( "F", fname );
errRep( " ", "Wrong number of values (^N) at line ^I in "
"file ^F (should be ^M).", status );
}
/* Otherwise increment the number of rows read. */
} else {
irow++;
/* Store each string value in the table, excluding "null" strings. Also check
the data type of each string an dupdate the column data types if necessary. */
for( iword = 0; iword < nword && *status == SAI__OK; iword++ ) {
if( strcmp( words[ iword ], "null" ) ) {
sprintf( key, "%s(%d)", cols[ iword ], irow );
astMapPut0C( result, key, words[ iword ], NULL );
/* If the column is currently thought to hold integers, check that the
current word looks like an integer. If not, down-grade the column type
to double. */
len = strlen( words[ iword ] );
if( types[ iword ] == AST__INTTYPE ) {
if( nc = 0, ( 1 != astSscanf( words[ iword ], "%d%n",
&ival, &nc ) ) ||
( nc < len ) ) {
types[ iword ] = AST__DOUBLETYPE;
}
}
/* If the column is currently thought to hold doubles, check that the
current word looks like an doubler. If not, down-grade the column type
to string. */
if( types[ iword ] == AST__DOUBLETYPE ) {
if( nc = 0, ( 1 != astSscanf( words[ iword ], "%lg%n",
&dval, &nc ) ) ||
( nc < len ) ) {
types[ iword ] = AST__STRINGTYPE;
}
}
}
}
}
/* Free the words returned by astChrSplit. */
for( iword = 0; iword < nword; iword++ ) {
words[ iword ] = astFree( words[ iword ] );
}
words = astFree( words );
}
}
}
/* The entire file has now been read, and a Table created in which every
column holds strings. We also have flags indicating whether the values
in each column are all integers or doubles. Modify the type of the
column within the Table to match these flags. */
nrow = astGetI( result, "Nrow" );
for( icol = 0; icol < ncol && *status == SAI__OK; icol++ ) {
/* The column will be re-named from "T_<name>" to "<name>". */
oldname = cols[ icol ];
newname = oldname + 2;
/* First convert string columns to integer columns if all the values in
the column look like integers. */
if( types[ icol ] == AST__INTTYPE ) {
/* Create the new column */
astAddColumn( result, newname, AST__INTTYPE, 0, NULL, " " );
/* Copy each cell of the current column, converting from string to integer. */
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0I( result, key, &ival ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0I( result, key, ival, NULL );
}
}
/* Now do double columns in the same way. */
} else if( types[ icol ] == AST__DOUBLETYPE ) {
astAddColumn( result, newname, AST__DOUBLETYPE, 0, NULL, " " );
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0D( result, key, &dval ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0D( result, key, dval, NULL );
}
}
/* Copy string values without change. */
} else {
astAddColumn( result, newname, AST__STRINGTYPE, 0, NULL, " " );
for( irow = 1; irow <= nrow; irow++ ) {
sprintf( key, "%s(%d)", oldname, irow );
if( astMapGet0C( result, key, &cval ) ) {
sprintf( key, "%s(%d)", newname, irow );
astMapPut0C( result, key, cval, NULL );
}
}
}
/* Remove the old column. */
astRemoveColumn( result, oldname );
}
/* Free resources. */
line = astFree( line );
last_com = astFree( last_com );
types = astFree( types );
if( cols ) {
for( icol = 0; icol < ncol; icol++ ) {
cols[ icol ] = astFree( cols[ icol ] );
}
cols = astFree( cols );
}
/* If the table ended with an end-of-table marker, there may be another
Table in the file. Call this function recursively to read it. */
if( eot ) {
subtable = ReadNextTable( fd, fname, iline, status );
/* Store the subtable as a table parameter in the returned table. */
if( subtable ) {
astAddParameter( result, "SubTable" );
astMapPut0A( result, "SubTable", subtable, NULL );
/* The Table clones the pointer, so we must annull our local copy of it. */
subtable = astAnnul( subtable );
}
}
/* Return the Table pointer. */
return result;
}
void smurf_calcqu( int *status ) {
/* Local Variables: */
AstFitsChan *fc; /* Holds FITS headers for output NDFs */
AstKeyMap *config; /* Holds all cleaning parameters */
AstKeyMap *dkpars; /* Holds dark squid cleaning parameters */
AstKeyMap *heateffmap = NULL; /* Heater efficiency data */
AstKeyMap *sub_instruments;/* Indicates which instrument is being used */
Grp *bgrp = NULL; /* Group of base names for each chunk */
Grp *igrp = NULL; /* Group of input files */
Grp *ogrp = NULL; /* Group of output files */
Grp *sgrp = NULL; /* Group of science files */
HDSLoc *loci = NULL; /* Locator for output I container file */
HDSLoc *locq = NULL; /* Locator for output Q container file */
HDSLoc *locu = NULL; /* Locator for output U container file */
NdgProvenance *oprov; /* Provenance to store in each output NDF */
ThrWorkForce *wf; /* Pointer to a pool of worker threads */
char headval[ 81 ]; /* FITS header value */
char ndfname[ 30 ]; /* Name of output Q or U NDF */
char polcrd[ 81 ]; /* FITS 'POL_CRD' header value */
char subarray[ 10 ]; /* Subarray name (e.g. "s4a", etc) */
double angrot; /* Angle from focal plane X axis to fixed analyser */
double paoff; /* WPLATE value corresponding to POL_ANG=0.0 */
float arcerror; /* Max acceptable error (arcsec) in one block */
int block_end; /* Index of last time slice in block */
int block_start; /* Index of first time slice in block */
int dkclean; /* Clean dark squids? */
int fix; /* Fix the POL-2 triggering issue? */
int iblock; /* Index of current block */
int iplace; /* NDF placeholder for current block's I image */
int ipolcrd; /* Reference direction for waveplate angles */
int maxsize; /* Max no. of time slices in a block */
int minsize; /* Min no. of time slices in a block */
int nc; /* Number of characters written to a string */
int pasign; /* +1 or -1 indicating sense of POL_ANG value */
int qplace; /* NDF placeholder for current block's Q image */
int submean; /* Subtract mean value from each time slice? */
int uplace; /* NDF placeholder for current block's U image */
size_t ichunk; /* Continuous chunk counter */
size_t idx; /* Subarray counter */
size_t igroup; /* Index for group of related input NDFs */
size_t inidx; /* Index into group of science input NDFs */
size_t nchunk; /* Number continuous chunks outside iter loop */
size_t ssize; /* Number of science files in input group */
smfArray *concat = NULL; /* Pointer to smfArray holding bolometer data */
smfArray *darks = NULL; /* dark frames */
smfArray *dkarray = NULL; /* Pointer to smfArray holding dark squid data */
smfArray *flatramps = NULL;/* Flatfield ramps */
smfData *data = NULL; /* Concatenated data for one subarray */
smfData *dkdata = NULL; /* Concatenated dark squid data for one subarray */
smfGroup *sgroup = NULL; /* smfGroup corresponding to sgrp */
/* Check inhereited status */
if( *status != SAI__OK ) return;
/* Start new AST and NDF contexts. */
astBegin;
ndfBegin();
/* Find the number of cores/processors available and create a work force
holding the same number of threads. */
wf = thrGetWorkforce( thrGetNThread( SMF__THREADS, status ), status );
/* Get a group of input files */
kpg1Rgndf( "IN", 0, 1, " Give more NDFs...", &igrp, &ssize, status );
/* Get a group containing just the files holding science data. */
smf_find_science( igrp, &sgrp, 0, NULL, NULL, 1, 1, SMF__NULL, &darks,
&flatramps, &heateffmap, NULL, status );
/* Check we have at least once science file. */
ssize = grpGrpsz( sgrp, status );
if( ssize == 0 ) {
msgOutif( MSG__NORM, " ", "All supplied input frames were DARK.",
status );
} else {
/* See if a correction should be made for the POL2 triggering issue. */
parGet0l( "FIX", &fix, status );
/* Create HDS container files to hold the output NDFs. */
datCreat( "OUTQ", "CALCQU", 0, 0, status );
datCreat( "OUTU", "CALCQU", 0, 0, status );
/* Associate the locators with the structures. */
datAssoc( "OUTQ", "WRITE", &locq, status );
datAssoc( "OUTU", "WRITE", &locu, status );
/* The I images are optional. */
if( *status == SAI__OK ) {
datCreat( "OUTI", "CALCQU", 0, 0, status );
datAssoc( "OUTI", "WRITE", &loci, status );
if( *status == PAR__NULL ) {
errAnnul( status );
loci = NULL;
}
}
/* Group the input files so that all files within a single group have the
same wavelength and belong to the same subscan of the same observation.
Also identify chunks of data that are contiguous in time, and
determine to which such chunk each group belongs. All this information
is returned in a smfGroup structure ("*sgroup"). */
smf_grp_related( sgrp, ssize, 1, 1, 0, NULL, NULL, NULL,
NULL, &sgroup, &bgrp, NULL, status );
/* Obtain the number of contiguous chunks. */
if( *status == SAI__OK ) {
nchunk = sgroup->chunk[ sgroup->ngroups - 1 ] + 1;
} else {
nchunk = 0;
}
/* Indicate we have not yet found a value for the ARCERROR parameter. */
arcerror = 0.0;
/* Loop over all contiguous chunks */
for( ichunk = 0; ichunk < nchunk && *status == SAI__OK; ichunk++ ) {
/* Display the chunk number. */
if( nchunk > 1 ) {
msgOutiff( MSG__VERB, "", " Doing chunk %d of %d.",
status, (int) ichunk + 1, (int) nchunk );
}
/* Concatenate the data within this contiguous chunk. This produces a
smfArray ("concat") containing a smfData for each subarray present in
the chunk. Each smfData holds the concatenated data for a single
subarray. */
smf_concat_smfGroup( wf, NULL, sgroup, darks, NULL, flatramps,
heateffmap, ichunk, 1, 1, NULL, 0, NULL, NULL,
0, 0, 0, &concat, NULL, status );
/* Get a KeyMap holding values for the configuration parameters. Since we
sorted by wavelength when calling smf_grp_related, we know that all
smfDatas in the current smfArray (i.e. chunk) will relate to the same
wavelength. Therefore we can use the same parameters for all smfDatas in
the current smfArray. */
sub_instruments = smf_subinst_keymap( SMF__SUBINST_NONE,
concat->sdata[ 0 ], NULL,
0, status );
config = kpg1Config( "CONFIG", "$SMURF_DIR/smurf_calcqu.def",
sub_instruments, status );
sub_instruments = astAnnul( sub_instruments );
/* Get the CALCQU specific parameters. */
if( !astMapGet0I( config, "PASIGN", &pasign ) ) pasign = 1;
msgOutiff( MSG__VERB, "", "PASIGN=%d", status, pasign );
if( !astMapGet0D( config, "PAOFF", &paoff ) ) paoff = 0.0;
msgOutiff( MSG__VERB, "", "PAOFF=%g", status, paoff );
if( !astMapGet0D( config, "ANGROT", &angrot ) ) angrot = 90.0;
msgOutiff( MSG__VERB, "", "ANGROT=%g", status, angrot );
if( !astMapGet0I( config, "SUBMEAN", &submean ) ) submean = 0;
msgOutiff( MSG__VERB, "", "SUBMEAN=%d", status, submean );
/* See if the dark squids should be cleaned. */
if( !astMapGet0I( config, "DKCLEAN", &dkclean ) ) dkclean = 0;
/* If required, clean the dark squids now since we might need to use them to
clean the bolometer data. */
if( dkclean ) {
/* Create a smfArray containing the dark squid data. For each one, store
a pointer to the main header so that smf_clean_smfArray can get at the
JCMTState information. */
dkarray = smf_create_smfArray( status );
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
if( data && data->da && data->da->dksquid ) {
dkdata = data->da->dksquid;
dkdata->hdr = data->hdr;
smf_addto_smfArray( dkarray, dkdata, status );
}
}
/* Clean the smfArray containing the dark squid data. Use the "CLEANDK.*"
parameters. */
(void) astMapGet0A( config, "CLEANDK", &dkpars );
smf_clean_smfArray( wf, dkarray, NULL, NULL, NULL, dkpars, status );
dkpars = astAnnul( dkpars );
/* Nullify the header pointers so that we don't accidentally close any. */
if( dkarray ) {
for( idx = 0; idx < dkarray->ndat; idx++ ) {
dkdata = dkarray->sdata[ idx ];
dkdata->hdr = NULL;
}
/* Free the smfArray holding the dark squid data, but do not free the
individual smfDatas within it. */
dkarray->owndata = 0;
smf_close_related( &dkarray, status );
}
}
/* Now clean the bolometer data */
smf_clean_smfArray( wf, concat, NULL, NULL, NULL, config, status );
/* If required correct for the POL2 triggering issue. */
if( fix ) smf_fix_pol2( wf, concat, status );
/* Loop round each sub-array in the current contiguous chunk of data. */
for( idx = 0; idx < concat->ndat && *status == SAI__OK; idx++ ) {
data = concat->sdata[ idx ];
/* Find the name of the subarray that generated the data. */
smf_find_subarray( data->hdr, subarray, sizeof(subarray), NULL,
status );
/* Display the sub-array. */
if( concat->ndat > 1 ) {
msgOutiff( MSG__VERB, "", " Doing sub-array %s.",
status, subarray );
}
/* Create an empty provenance structure. Each input NDF that contributes
to the current chunk and array will be added as an ancestor to this
structure, which will later be stored in each output NDF created for
this chunk and array. */
oprov = ndgReadProv( NDF__NOID, "SMURF:CALCQU", status );
/* Indicate we do not yet have any FITS headers for the output NDFs */
fc = NULL;
/* Indicate we do not yet know the coordinate reference frame for the
half-waveplate angle. */
polcrd[ 0 ] = 0;
ipolcrd = 0;
/* Go through the smfGroup looking for groups of related input NDFs that
contribute to the current chunk. */
for( igroup = 0; igroup < sgroup->ngroups; igroup++ ) {
if( sgroup->chunk[ igroup ] == ichunk ) {
/* Get the integer index into the GRP group (sgrp) that holds the input NDFs.
This index identifies the input NDF that provides the data for the current
chunk and subarray. This assumes that the order in which smf_concat_smfGroup
stores arrays in the "concat" smfArray matches the order in which
smf_grp_related stores arrays within the sgroup->subgroups. */
inidx = sgroup->subgroups[ igroup ][ idx ];
/* Add this input NDF as an ancestor into the output provenance structure. */
smf_accumulate_prov( NULL, sgrp, inidx, NDF__NOID,
"SMURF:CALCQU", &oprov, status );
/* Merge the FITS headers from the current input NDF into the FitsChan
that holds headers for the output NDFs. The merging retains only those
headers which have the same value in all input NDFs. */
smf_fits_outhdr( data->hdr->fitshdr, &fc, status );
/* Get the polarimetry related FITS headers and check that all input NDFs
have usabie values. */
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_MODE", headval,
sizeof(headval), status );
if( strcmp( headval, "CONSTANT" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Input NDF ^N does not contain "
"polarimetry data obtained with a spinning "
"half-waveplate.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLWAVIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Half-waveplate was not in the beam for "
"input NDF ^N.", status );
}
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POLANLIN", headval,
sizeof(headval), status );
if( strcmp( headval, "Y" ) && *status == SAI__OK ) {
*status = SAI__ERROR;
grpMsg( "N", sgrp, inidx );
errRep( " ", "Analyser was not in the beam for input "
"NDF ^N.", status );
}
if( polcrd[ 0 ] ) {
headval[ 0 ] = 0;
smf_getfitss( data->hdr, "POL_CRD", headval,
sizeof(headval), status );
if( strcmp( headval, polcrd ) && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( " ", "Input NDFs have differing values for "
"FITS header 'POL_CRD'.", status );
}
} else {
smf_getfitss( data->hdr, "POL_CRD", polcrd,
sizeof(polcrd), status );
if( !strcmp( polcrd, "FPLANE" ) ) {
ipolcrd = 0;
} else if( !strcmp( polcrd, "AZEL" ) ) {
ipolcrd = 1;
} else if( !strcmp( polcrd, "TRACKING" ) ) {
ipolcrd = 2;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
msgSetc( "N", data->file->name );
msgSetc( "V", polcrd );
errRep( " ", "Input NDF ^N contains unknown value "
"'^V' for FITS header 'POL_CRD'.", status );
}
}
}
}
/* If not already done, get the maximum spatial drift (in arc-seconds) that
can be tolerated whilst creating a single I/Q/U image. The default value is
half the makemap default pixel size. Also get limits on the number of
time slices in any block. */
if( arcerror == 0.0 ) {
parDef0d( "ARCERROR", 0.5*smf_calc_telres( data->hdr->fitshdr,
status ), status );
parGet0r( "ARCERROR", &arcerror, status );
parGet0i( "MAXSIZE", &maxsize, status );
parGet0i( "MINSIZE", &minsize, status );
if( maxsize > 0 && maxsize < minsize && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "Value of parameter MAXSIZE (%d) is less "
"than value of parameter MINSIZE (%d)", status,
maxsize, minsize );
}
}
/* The algorithm that calculates I, Q and U assumes that all samples for a
single bolometer measure flux from the same point on the sky. Due to
sky rotation, this will not be the case - each bolometer will drift
slowly across the sky. However, since the drift is (or should be)
slow we can apply the I/Q/U algorithm to blocks of contiguous data over
which the bolometers do not move significantly. We produce a separate
I, Q and U image for each such block. The first block starts at the first
time slice in the smfData. */
block_start = 0;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. */
block_end = smf_block_end( data, block_start, ipolcrd, arcerror,
maxsize, status );
/* Loop round creating I/Q/U images for each block. Count them. */
iblock = 0;
while( block_end >= 0 && *status == SAI__OK ) {
/* Skip very short blocks. */
if( block_end - block_start > minsize ) {
/* Display the start and end of the block. */
msgOutiff( MSG__VERB, "", " Doing time slice block %d "
"-> %d", status, (int) block_start,
(int) block_end );
/* Get the name for the Q NDF for this block. Start of with "Q" followed by
the block index. */
iblock++;
nc = sprintf( ndfname, "Q%d", iblock );
/* Append the subarray name to the NDF name. */
nc += sprintf( ndfname + nc, "_%s", subarray );
/* Append the chunk index to the NDF name. */
nc += sprintf( ndfname + nc, "_%d", (int) ichunk );
/* Get NDF placeholder for the Q NDF. The NDFs are created inside the
output container file. */
ndfPlace( locq, ndfname, &qplace, status );
/* The name of the U NDF is the same except the initial "Q" is changed to
"U". */
ndfname[ 0 ] = 'U';
ndfPlace( locu, ndfname, &uplace, status );
/* The name of the I NDF is the same except the initial "Q" is changed to
"I". */
if( loci ) {
ndfname[ 0 ] = 'I';
ndfPlace( loci, ndfname, &iplace, status );
} else {
iplace = NDF__NOPL;
}
/* Store the chunk and block numbers as FITS headers. */
atlPtfti( fc, "POLCHUNK", (int) ichunk, "Chunk index used by CALCQU", status );
atlPtfti( fc, "POLBLOCK", iblock, "Block index used by CALCQU", status );
/* Create the Q and U images for the current block of time slices from
the subarray given by "idx", storing them in the output container
file. */
smf_calc_iqu( wf, data, block_start, block_end, ipolcrd,
qplace, uplace, iplace, oprov, fc,
pasign, AST__DD2R*paoff, AST__DD2R*angrot,
submean, status );
/* Warn about short blocks. */
} else {
msgOutiff( MSG__VERB, "", " Skipping short block of %d "
"time slices (parameter MINSIZE=%d).", status,
block_end - block_start - 1, minsize );
}
/* The next block starts at the first time slice following the previous
block. */
block_start = block_end + 1;
/* Find the time slice at which the corner bolometers have moved
a critical distance (given by parameter ARCERROR) from their
positions at the start of the block. Then back off some time slices
to ensure that the block holds an integral number of half-waveplate
rotations. This returns -1 if all time slices have been used. */
block_end = smf_block_end( data, block_start, ipolcrd,
arcerror, maxsize, status );
}
/* Free resources */
oprov = ndgFreeProv( oprov, status );
fc = astAnnul( fc );
}
config = astAnnul( config );
/* Close the smfArray. */
smf_close_related( &concat, status );
}
/* Annul the locators for the output container files. */
datAnnul( &locq, status );
datAnnul( &locu, status );
if( loci ) datAnnul( &loci, status );
/* The parameter system hangs onto a primary locator for each container
file, so cancel the parameters to annul these locators. */
datCancl( "OUTQ", status );
datCancl( "OUTU", status );
datCancl( "OUTI", status );
}
/* Free resources. */
smf_close_related( &darks, status );
smf_close_related( &flatramps, status );
if( igrp ) grpDelet( &igrp, status);
if( sgrp ) grpDelet( &sgrp, status);
if( bgrp ) grpDelet( &bgrp, status );
if( ogrp ) grpDelet( &ogrp, status );
if( sgroup ) smf_close_smfGroup( &sgroup, status );
if (heateffmap) heateffmap = smf_free_effmap( heateffmap, status );
/* End the NDF and AST contexts. */
ndfEnd( status );
astEnd;
/* Issue a status indication.*/
if( *status == SAI__OK ) {
msgOutif( MSG__VERB, " ", "CALCQU succeeded.", status);
} else {
msgOutif( MSG__VERB, " ", "CALCQU failed.", status);
}
}
void smf_calcmodel_pln( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
int *lut_data = NULL; /* Lut data for one subarray */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
smfArray *lut=NULL; /* Pointer to LUT at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
size_t tstride; /* Time slice stride in data array */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "PLN", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping PLN this iteration", status );
return;
}
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
lut = dat->lut[chunk];
/* Assert ICD-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA|SMF__LUT,
1, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, NULL,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
lut_data = (lut->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
if( *status == SAI__OK ) {
/* Place last iteration of plane signal back into residual */
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
/* Copy the residual+old model into model_data where it will be
fitted again in this iteration. */
memcpy( model_data, res_data,
ndata*smf_dtype_size(res->sdata[idx],status) );
}
/* Calculate the fit and subtract it*/
smf_subtract_plane3( wf, res->sdata[idx], dat->mdims, lut_data, status );
/* Store the difference between the plane-subtracted signal and the residual
in the model container */
if( *status == SAI__OK ) {
for (i=0; i< nbolo*ntslice; i++) {
if ( !(qua_data[i]&SMF__Q_MOD) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
model_data[i] -= res_data[i];
} else {
model_data[i] = VAL__BADD;
}
}
}
}
}
if( kmap ) kmap = astAnnul( kmap );
model_data_copy = astFree( model_data_copy );
}
unsigned char *smf_get_mask( ThrWorkForce *wf, smf_modeltype mtype,
AstKeyMap *config, smfDIMMData *dat, int flags,
int *status ) {
/* Local Variables: */
AstCircle *circle; /* AST Region used to mask a circular area */
AstKeyMap *akm; /* KeyMap holding AST config values */
AstKeyMap *subkm; /* KeyMap holding model config values */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
char words[100]; /* Buffer for variable message words */
const char *cval; /* The ZERO_MASK string value */
const char *modname; /* The name of the model being masked */
const char *skyrefis; /* Pointer to SkyRefIs attribute value */
dim_t i; /* Pixel index */
double *pd; /* Pointer to next element of map data */
double *predef; /* Pointer to mask defined by previous run */
double *ptr; /* Pointer to NDF Data array */
double *pv; /* Pointer to next element of map variance */
double centre[ 2 ]; /* Coords of circle centre in radians */
double meanhits; /* Mean hits in the map */
double radius[ 1 ]; /* Radius of circle in radians */
double zero_circle[ 3 ]; /* LON/LAT/Radius of circular mask */
double zero_lowhits; /* Fraction of mean hits at which to threshold */
double zero_snr; /* Higher SNR at which to threshold */
double zero_snrlo; /* Lower SNR at which to threshold */
int *ph; /* Pointer to next hits value */
int have_mask; /* Did a mask already exist on entry? */
int imask; /* Index of next mask type */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int isstatic; /* Are all used masks static? */
int lbnd_grid[ 2 ]; /* Lower bounds of map in GRID coords */
int mask_types[ NTYPE ]; /* Identifier for the types of mask to use */
int munion; /* Use union of supplied masks */
int nel; /* Number of mapped NDF pixels */
int nmask; /* The number of masks to be combined */
int nsource; /* No. of source pixels in final mask */
int skip; /* No. of iters for which AST is not subtracted */
int thresh; /* Absolute threshold on hits */
int ubnd_grid[ 2 ]; /* Upper bounds of map in GRID coords */
int zero_c_n; /* Number of zero circle parameters read */
int zero_mask; /* Use the reference NDF as a mask? */
int zero_niter; /* Only mask for the first "niter" iterations. */
int zero_notlast; /* Don't zero on last iteration? */
size_t ngood; /* Number good samples for stats */
smf_qual_t *pq; /* Pinter to map quality */
unsigned char **mask; /* Address of model's mask pointer */
unsigned char *newmask; /* Individual mask work space */
unsigned char *pm; /* Pointer to next returned mask pixel */
unsigned char *pn; /* Pointer to next new mask pixel */
unsigned char *result; /* Returned mask pointer */
/* Initialise returned values */
result = NULL;
/* Check inherited status. Also check that a map is being created. */
if( *status != SAI__OK || !dat || !dat->map ) return result;
/* Begin an AST context. */
astBegin;
/* Get the sub-keymap containing the configuration parameters for the
requested model. Also get a pointer to the mask array to use (there is
one for each maskable model)*/
if( mtype == SMF__COM ) {
modname = "COM";
mask = &(dat->com_mask);
} else if( mtype == SMF__AST ) {
modname = "AST";
mask = &(dat->ast_mask);
} else if( mtype == SMF__FLT ) {
modname = "FLT";
mask = &(dat->flt_mask);
} else {
modname = NULL;
mask = NULL;
*status = SAI__ERROR;
errRepf( " ", "smf_get_mask: Unsupported model type %d supplied - "
"must be COM, FLT or AST.", status, mtype );
}
subkm = NULL;
astMapGet0A( config, modname, &subkm );
/* Get the "ast.skip" value - when considering "zero_niter" and
"zero_freeze", we only count iterations for which the AST model
is subtracted (i.e. the ones following the initial "ast.skip"
iterations). */
astMapGet0A( config, "AST", &akm );
astMapGet0I( akm, "SKIP", &skip );
akm = astAnnul( akm );
/* Get the number of iterations over which the mask is to be applied. Zero
means all. Return with no mask if this number of iterations has
already been performed. */
zero_niter = 0;
astMapGet0I( subkm, "ZERO_NITER", &zero_niter );
if( zero_niter == 0 || dat->iter < zero_niter + skip ) {
/* Only return a mask if this is not the last iteration, or if ZERO_NOTLAST
is unset. */
zero_notlast = 0;
astMapGet0I( subkm, "ZERO_NOTLAST", &zero_notlast );
if( !( flags & SMF__DIMM_LASTITER ) || !zero_notlast ) {
/* Create a list of the mask types to be combined to get the final mask by
looking for non-default values for the corresponding configuration
parameters in the supplied KeyMap. Static masks (predefined, circles
or external NDFs) may be used on any iteration, but dynamic masks
(lowhits, snr) will only be avialable once the map has been determined
at the end of the first iteration. This means that when masking anything
but the AST model (which is determined after the map), the dynamic masks
cannot be used on the first iteration. Make a note if all masks being
used are static. */
isstatic = 1;
nmask = 0;
zero_lowhits = 0.0;
astMapGet0D( subkm, "ZERO_LOWHITS", &zero_lowhits );
if( zero_lowhits > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = LOWHITS;
isstatic = 0;
}
} else if( zero_lowhits < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_LOWHITS (%g) - "
"it must not be negative.", status, modname, zero_lowhits );
}
if( astMapGet1D( subkm, "ZERO_CIRCLE", 3, &zero_c_n, zero_circle ) ) {
if( zero_c_n == 1 || zero_c_n == 3 ) {
mask_types[ nmask++] = CIRCLE;
} else if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad number of values (%d) for config parameter "
"%s.ZERO_CIRCLE - must be 1 or 3.", status, zero_c_n,
modname );
}
}
cval = NULL;
astMapGet0C( subkm, "ZERO_MASK", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( subkm, "ZERO_MASK", &zero_mask );
cval = ( zero_mask > 0 ) ? "REF" : NULL;
}
if( cval ) {
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
mask_types[ nmask++] = REFNDF;
}
}
zero_snr = 0.0;
astMapGet0D( subkm, "ZERO_SNR", &zero_snr );
if( zero_snr > 0.0 ) {
if( mtype == SMF__AST || !( flags & SMF__DIMM_FIRSTITER ) ) {
mask_types[ nmask++] = SNR;
isstatic = 0;
}
} else if( zero_snr < 0.0 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter %s.ZERO_SNR (%g) - "
"it must not be negative.", status, modname, zero_snr );
}
if( astMapHasKey( subkm, "ZERO_MASK_POINTER" ) ) {
astMapGet0P( subkm, "ZERO_MASK_POINTER", (void **) &predef );
if( predef ) mask_types[ nmask++] = PREDEFINED;
}
/* No need to create a mask if no masking was requested or possible. */
if( nmask > 0 ) {
/* Decide if we are using the union or intersection of the masks. */
astMapGet0I( subkm, "ZERO_UNION", &munion );
/* Note if a mask existed on entry. If not, create a mask now, and
initialise it to hold the mask defined by the initial sky map. */
if( *mask == NULL ) {
have_mask = 0;
if( dat->initqual ) {
*mask = astMalloc( dat->msize*sizeof( **mask ) );
if( *mask ) {
pm = *mask;
pq = dat->initqual;
for( i = 0; i < dat->msize; i++ ) {
*(pm++) = ( *(pq++) != 0 );
}
}
} else{
*mask = astCalloc( dat->msize, sizeof( **mask ) );
}
} else {
have_mask = 1;
}
/* If we are combining more than one mask, we need work space to hold
an individual mask independently of the total mask. If we are using
only one mask, then just use the main mask array. */
if( nmask > 1 ) {
newmask = astMalloc( dat->msize*sizeof( *newmask ) );
} else {
newmask = *mask;
}
/* Get the number of iterations after which the mask is to be frozen.
Zero means "never freeze the mask". */
int zero_freeze = 0;
astMapGet0I( subkm, "ZERO_FREEZE", &zero_freeze );
/* Loop round each type of mask to be used. */
for( imask = 0; imask < nmask && *status == SAI__OK; imask++ ){
/* If the mask is now frozen, we just return the existing mask. So leave the
loop. */
if( zero_freeze != 0 && dat->iter > zero_freeze + skip ) {
break;
/* Low hits masking... */
} else if( mask_types[ imask ] == LOWHITS ) {
/* Set hits pixels with 0 hits to VAL__BADI so that stats1 ignores them */
ph = dat->hitsmap;
for( i = 0; i < dat->msize; i++,ph++ ) {
if( *ph == 0 ) *ph = VAL__BADI;
}
/* Find the mean hits in the map */
smf_stats1I( dat->hitsmap, 1, dat->msize, NULL, 0, 0, &meanhits,
NULL, NULL, &ngood, status );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: mean hits = %lf, ngood "
"= %zd", status, meanhits, ngood );
/* Create the mask */
thresh = meanhits*zero_lowhits;
ph = dat->hitsmap;
pn = newmask;
for( i = 0; i < dat->msize; i++,ph++ ) {
*(pn++) = ( *ph != VAL__BADI && *ph < thresh ) ? 1 : 0;
}
/* Report masking info. */
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: masking %s "
"model at hits = %d.", status, modname, thresh );
/* Circle masking... */
} else if( mask_types[ imask ] == CIRCLE ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the circle
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* If only one parameter supplied it is radius, assume reference
LON/LAT from the frameset to get the centre. If the SkyFrame
represents offsets from the reference position (i.e. the source is
moving), assume the circle is to be centred on the origin. */
if( zero_c_n == 1 ) {
zero_circle[ 2 ] = zero_circle[ 0 ];
skyrefis = astGetC( dat->outfset, "SkyRefIs" );
if( skyrefis && !strcmp( skyrefis, "Origin" ) ) {
zero_circle[ 0 ] = 0.0;
zero_circle[ 1 ] = 0.0;
} else {
zero_circle[ 0 ] = astGetD( dat->outfset, "SkyRef(1)" );
zero_circle[ 1 ] = astGetD( dat->outfset, "SkyRef(2)" );
}
zero_circle[ 0 ] *= AST__DR2D;
zero_circle[ 1 ] *= AST__DR2D;
}
/* The supplied bounds are for pixel coordinates... we need bounds for grid
coordinates which have an offset */
lbnd_grid[ 0 ] = 1;
lbnd_grid[ 1 ] = 1;
ubnd_grid[ 0 ] = dat->ubnd_out[ 0 ] - dat->lbnd_out[ 0 ] + 1;
ubnd_grid[ 1 ] = dat->ubnd_out[ 1 ] - dat->lbnd_out[ 1 ] + 1;
/* Coordinates & radius of the circular region converted from degrees
to radians */
centre[ 0 ] = zero_circle[ 0 ]*AST__DD2R;
centre[ 1 ] = zero_circle[ 1 ]*AST__DD2R;
radius[ 0 ] = zero_circle[ 2 ]*AST__DD2R;
/* Create the Circle, defined in the current Frame of the FrameSet (i.e.
the sky frame). */
circle = astCircle( astGetFrame( dat->outfset, AST__CURRENT), 1,
centre, radius, NULL, " " );
/* Fill the mask with zeros. */
memset( newmask, 0, sizeof( *newmask )*dat->msize );
/* Get the mapping from the sky frame (current) to the grid frame (base),
and then set the mask to 1 for all of the values outside this circle */
astMaskUB( circle, astGetMapping( dat->outfset, AST__CURRENT,
AST__BASE ),
0, 2, lbnd_grid, ubnd_grid, newmask, 1 );
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model will"
" be masked %s using a circle of "
"radius %g arc-secs, centred at %s=%s, %s=%s.",
status, modname, words, radius[0]*AST__DR2D*3600,
astGetC( dat->outfset, "Symbol(1)" ),
astFormat( dat->outfset, 1, centre[ 0 ] ),
astGetC( dat->outfset, "Symbol(2)" ),
astFormat( dat->outfset, 2, centre[ 1 ] ) );
}
/* Reference NDF masking... */
} else if( mask_types[ imask ] == REFNDF ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the NDF
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter. */
ndfAssoc( refparam, "READ", &indf1, status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2,
status );
/* Map the section. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr,
&nel, status );
/* Check we can use the pointer safely. */
if( *status == SAI__OK ) {
/* Find bad pixels in the NDF and set those pixels to 1 in the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(ptr++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
ndfMsg( "N", indf2 );
msgSetc( "M", modname );
if( zero_niter == 0 ) {
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked on each iteration "
"using the bad pixels in NDF '^N'.",
status );
} else {
msgSeti( "I", zero_niter );
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The ^M "
"model will be masked for ^I iterations "
"using the bad pixels in NDF '^N'.",
status );
}
}
/* End the NDF context. */
ndfEnd( status );
}
/* SNR masking... */
} else if( mask_types[ imask ] == SNR ) {
/* Get the lower SNR limit. */
zero_snrlo = 0.0;
astMapGet0D( subkm, "ZERO_SNRLO", &zero_snrlo );
if( zero_snrlo <= 0.0 ) {
zero_snrlo = zero_snr;
} else if( zero_snrlo > zero_snr && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( " ", "Bad value for config parameter "
"%s.ZERO_SNRLO (%g) - it must not be higher "
"than %s.ZERO_SNR (%g).", status, modname,
zero_snrlo, modname, zero_snr );
}
/* If the higher and lower SNR limits are equal, just do a simple
threshold on the SNR values to get the mask. */
if( zero_snr == zero_snrlo ) {
pd = dat->map;
pv = dat->mapvar;
pn = newmask;
for( i = 0; i < dat->msize; i++,pd++,pv++ ) {
*(pn++) = ( *pd != VAL__BADD && *pv != VAL__BADD &&
*pv >= 0.0 && *pd < zero_snr*sqrt( *pv ) ) ? 1 : 0;
}
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g.",
status, modname, words, zero_snr );
}
/* If the higher and lower SNR limits are different, create an initial
mask by thresholding at the ZERO_SNR value, and then extend the source
areas within the mask down to an SNR limit of ZERO_SNRLO. */
} else {
smf_snrmask( wf, dat->map, dat->mapvar, dat->mdims,
zero_snr, zero_snrlo, newmask, status );
/* Report masking info. */
if( !have_mask ) {
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using an SNR limit of %g "
"extended down to %g.", status, modname,
words, zero_snr, zero_snrlo );
}
}
/* Predefined masking... */
} else if( mask_types[ imask ] == PREDEFINED ) {
/* If we had a mask on entry, then there is no need to create a new one
since it will not have changed. But we need to recalculate the
mask if are combining it with any non-static masks. */
if( ! have_mask || ! isstatic ) {
/* Find bad pixels in the predefined array and set those pixels to 1 in
the mask. */
pn = newmask;
for( i = 0; i < dat->msize; i++ ) {
*(pn++) = ( *(predef++) == VAL__BADD ) ? 1 : 0;
}
/* Report masking info. */
if( zero_niter == 0 ) {
sprintf( words, "on each iteration" );
} else {
sprintf( words, "for %d iterations", zero_niter );
}
msgOutiff( MSG__DEBUG, " ", "smf_get_mask: The %s model "
"will be masked %s using a smoothed form of "
"the final mask created with the previous map.",
status, modname, words );
}
}
/* If required, add the new mask into the returned mask. If this is the
first mask, we just copy the new mask to form the returned mask.
Otherwise, we combine it with the existing returned mask. */
if( ! have_mask || ! isstatic ) {
if( nmask > 1 ) {
if( imask == 0 ) {
memcpy( *mask, newmask, dat->msize*sizeof(*newmask));
} else {
pm = *mask;
pn = newmask;
if( munion ) {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 0 ) *pm = 0;
}
} else {
for( i = 0; i < dat->msize; i++,pm++ ) {
if( *(pn++) == 1 ) *pm = 1;
}
}
}
}
}
}
/* Free the individual mask work array if it was used. */
if( nmask > 1 ) newmask = astFree( newmask );
/* Check that the mask has some source pixels (i.e. pixels that have non-bad data values -
we do not also check variance values since they are not available until the second
iteration). */
if( *status == SAI__OK ) {
nsource = 0;
pm = *mask;
pd = dat->map;
for( i = 0; i < dat->msize; i++,pd++,pv++,pm++ ) {
if( *pd != VAL__BADD && *pm == 0 ) nsource++;
}
if( nsource < 5 && *status == SAI__OK ) {
*status = SAI__ERROR;
errRepf( "", "The %s mask being used has fewer than 5 "
"source pixels.", status, modname );
if( zero_snr > 0.0 ) {
errRepf( "", "Maybe your zero_snr value (%g) is too high?",
status, zero_snr );
}
}
}
/* Return the mask pointer if all has gone well. */
if( *status == SAI__OK ) result = *mask;
}
}
}
/* End the AST context, annulling all AST Objects created in the context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
void smf_find_science(const Grp * ingrp, Grp **outgrp, int reverttodark,
Grp **darkgrp, Grp **flatgrp, int reducedark,
int calcflat, smf_dtype darktype, smfArray ** darks,
smfArray **fflats, AstKeyMap ** heateffmap,
double * meanstep, int * status ) {
smfSortInfo *alldarks; /* array of sort structs for darks */
smfSortInfo *allfflats; /* array of fast flat info */
Grp * dgrp = NULL; /* Internal dark group */
double duration_darks = 0.0; /* total duration of all darks */
double duration_sci = 0.0; /* Duration of all science observations */
size_t dkcount = 0; /* Dark counter */
size_t ffcount = 0; /* Fast flat counter */
Grp * fgrp = NULL; /* Fast flat group */
size_t i; /* loop counter */
smfData *infile = NULL; /* input file */
size_t insize; /* number of input files */
size_t nsteps_dark = 0; /* Total number of steps for darks */
size_t nsteps_sci = 0; /* Total number of steps for science */
AstKeyMap * heatermap = NULL; /* Heater efficiency map */
AstKeyMap * obsmap = NULL; /* Info from all observations */
AstKeyMap * objmap = NULL; /* All the object names used */
AstKeyMap * scimap = NULL; /* All non-flat obs indexed by unique key */
Grp *ogrp = NULL; /* local copy of output group */
size_t sccount = 0; /* Number of accepted science files */
struct timeval tv1; /* Timer */
struct timeval tv2; /* Timer */
if (meanstep) *meanstep = VAL__BADD;
if (outgrp) *outgrp = NULL;
if (darkgrp) *darkgrp = NULL;
if (darks) *darks = NULL;
if (fflats) *fflats = NULL;
if (heateffmap) *heateffmap = NULL;
if (*status != SAI__OK) return;
/* Sanity check to make sure we return some information */
if ( outgrp == NULL && darkgrp == NULL && darks == NULL && fflats == NULL) {
*status = SAI__ERROR;
errRep( " ", FUNC_NAME ": Must have some non-NULL arguments"
" (possible programming error)", status);
return;
}
/* Start a timer to see how long this takes */
smf_timerinit( &tv1, &tv2, status );
/* Create new group for output files */
ogrp = smf_grp_new( ingrp, "Science", status );
/* and a new group for darks */
dgrp = smf_grp_new( ingrp, "DarkFiles", status );
/* and for fast flats */
fgrp = smf_grp_new( ingrp, "FastFlats", status );
/* and also create a keymap for the observation description */
obsmap = astKeyMap( "KeyError=1" );
/* and an object map */
objmap = astKeyMap( "KeyError=1" );
/* This keymap contains the sequence counters for each related
subarray/obsidss/heater/shutter combination and is used to decide
if a bad flat is relevant */
scimap = astKeyMap( "KeyError=1,KeyCase=0" );
/* This keymap is used to contain relevant heater efficiency data */
heatermap = astKeyMap( "KeyError=1,KeyCase=0" );
/* Work out how many input files we have and allocate sufficient sorting
space */
insize = grpGrpsz( ingrp, status );
alldarks = astCalloc( insize, sizeof(*alldarks) );
allfflats = astCalloc( insize, sizeof(*allfflats) );
/* check each file in turn */
for (i = 1; i <= insize; i++) {
int seqcount = 0;
char keystr[100]; /* Key for scimap entry */
/* open the file but just to get the header */
smf_open_file( ingrp, i, "READ", SMF__NOCREATE_DATA, &infile, status );
if (*status != SAI__OK) break;
/* Fill in the keymap with observation details */
smf_obsmap_fill( infile, obsmap, objmap, status );
/* Find the heater efficiency map if required */
if (*status == SAI__OK && heateffmap) {
char arrayidstr[32];
smf_fits_getS( infile->hdr, "ARRAYID", arrayidstr, sizeof(arrayidstr),
status );
if (!astMapHasKey( heatermap, arrayidstr ) ) {
smfData * heateff = NULL;
dim_t nbolos = 0;
smf_flat_params( infile, "RESIST", NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &heateff, status );
smf_get_dims( heateff, NULL, NULL, &nbolos, NULL, NULL, NULL, NULL,
status );
if (heateff) astMapPut0P( heatermap, arrayidstr, heateff, NULL );
}
}
/* Get the sequence counter for the file. We do not worry about
duplicate sequence counters (at the moment) */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* The key identifying this subarray/obsidss/heater/shutter combo */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
if (smf_isdark( infile, status )) {
/* Store the sorting information */
dkcount = smf__addto_sortinfo( infile, alldarks, i, dkcount, "Dark", status );
smf__addto_durations( infile, &duration_darks, &nsteps_dark, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
} else {
/* compare sequence type with observation type and drop it (for now)
if they differ */
if ( infile->hdr->obstype == infile->hdr->seqtype ) {
/* Sanity check the header for corruption. Compare RTS_NUM with SEQSTART
and SEQEND. The first RTS_NUM must either be SEQSTART or else between
SEQSTART and SEQEND (if someone has giving us a section) */
int seqstart = 0;
int seqend = 0;
int firstnum = 0;
JCMTState *tmpState = NULL;
smf_getfitsi( infile->hdr, "SEQSTART", &seqstart, status );
smf_getfitsi( infile->hdr, "SEQEND", &seqend, status );
tmpState = infile->hdr->allState;
if( tmpState ) {
firstnum = (tmpState[0]).rts_num;
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
if ( firstnum >= seqstart && firstnum <= seqend ) {
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif(MSG__DEBUG, " ", "Non-dark file: ^F",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
} else {
msgOutif( MSG__QUIET, "",
"File ^F has a corrupt FITS header. Ignoring it.",
status );
}
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
/* store the file in the output group */
ndgCpsup( ingrp, i, ogrp, status );
msgOutif( MSG__DEBUG, " ",
"File ^F lacks JCMTState: assuming it is non-dark",status);
smf__addto_durations( infile, &duration_sci, &nsteps_sci, status );
astMapPutElemI( scimap, keystr, -1, seqcount );
sccount++;
}
} else if (infile->hdr->seqtype == SMF__TYP_FASTFLAT ) {
ffcount = smf__addto_sortinfo( infile, allfflats, i, ffcount, "Fast flat", status );
} else {
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>");
msgOutif(MSG__DEBUG, " ", "Sequence type mismatch with observation type: ^F",status);
}
}
/* close the file */
smf_close_file( &infile, status );
}
/* Store output group in return variable or else free it */
if (outgrp) {
*outgrp = ogrp;
} else {
grpDelet( &ogrp, status );
}
/* process flatfields if necessary */
if (ffcount > 0 && fflats ) {
smfArray * array = NULL;
/* sort flats into order */
qsort( allfflats, ffcount, sizeof(*allfflats), smf_sort_bydouble);
if (fflats) array = smf_create_smfArray( status );
/* now open the flats and store them if requested */
if (*status == SAI__OK && array && ffcount) {
size_t start_ffcount = ffcount;
AstKeyMap * flatmap = NULL;
if (calcflat) {
/* Use AgeUp so that we get the keys out in the sorted order
that allfflats used */
flatmap = astKeyMap( "KeyCase=0,KeyError=1,SortBy=AgeDown" );
}
/* Read each flatfield. Calculate a responsivity image and a flatfield
solution. Store these in a keymap along with related information
which is itself stored in a keymap indexed by a string made of
OBSIDSS, reference heater value, shutter and subarray.
*/
for (i = 0; i < start_ffcount; i++ ) {
size_t ori_index = (allfflats[i]).index;
smfData * outfile = NULL;
char keystr[100];
AstKeyMap * infomap = astKeyMap( "KeyError=1" );
int oplen = 0;
char thisfile[MSG__SZMSG];
int seqcount = 0;
/* read filename from group */
infile = NULL;
smf_open_file( ingrp, ori_index, "READ", 0, &infile, status );
if ( *status != SAI__OK ) {
/* This should not happen because we have already opened
the file. If it does happen we abort with error. */
if (infile) smf_close_file( &infile, status );
break;
}
/* Calculate the key for this observation */
smf__calc_flatobskey( infile->hdr, keystr, sizeof(keystr), status );
/* Get the file name for error messages */
smf_smfFile_msg( infile->file, "F", 1, "<unknown file>" );
msgLoad( "", "^F", thisfile, sizeof(thisfile), &oplen, status );
/* And the sequence counter to link against science observations */
smf_find_seqcount( infile->hdr, &seqcount, status );
/* Prefill infomap */
astMapPut0C( infomap, "FILENAME", thisfile, "");
astMapPut0I( infomap, "SEQCOUNT", seqcount, "");
/* Collapse it */
if (*status == SAI__OK) {
smf_flat_fastflat( infile, &outfile, status );
if (*status == SMF__BADFLAT) {
errFlush( status );
if (calcflat) {
/* Need to generate an outfile like smf_flat_fastflat
and one heater setting will force smf_flat_calcflat to fail */
smf_flat_malloc( 1, infile, NULL, &outfile, status );
} else {
if (outfile) smf_close_file( &outfile, status );
if (infile) smf_close_file( &infile, status );
infomap = astAnnul( infomap );
ffcount--;
continue;
}
}
}
if (outfile && *status == SAI__OK) {
smf_close_file( &infile, status );
infile = outfile;
if (calcflat) {
size_t ngood = 0;
smfData * curresp = NULL;
int utdate;
if (*status == SAI__OK) {
ngood = smf_flat_calcflat( MSG__VERB, NULL, "RESIST",
"FLATMETH", "FLATORDER", NULL, "RESPMASK",
"FLATSNR", NULL, infile, &curresp, status );
if (*status != SAI__OK) {
/* if we failed to calculate a flatfield we continue but force the
flatfield to be completely bad. This will force the science data associated
with the flatfield to be correctly blanked. We do not annul though
if we have a SUBPAR error telling us that we have failed to define
our parameters properly. */
if (*status != SUBPAR__NOPAR) errAnnul(status);
/* parameters of flatfield */
ngood = 0;
/* Generate a blank flatfield and blank responsivity image */
smf_flat_badflat( infile, &curresp, status );
}
/* Retrieve the UT date so we can decide whether to compare
flatfields */
smf_getfitsi( infile->hdr, "UTDATE", &utdate, status );
/* Store the responsivity data for later on and the processed
flatfield until we have vetted it */
astMapPut0P( infomap, "CALCFLAT", infile, "" );
astMapPut0P( infomap, "RESP", curresp, "" );
astMapPut0I( infomap, "UTDATE", utdate, "" );
astMapPut0I( infomap, "ISGOOD", 1, "" );
astMapPut0I( infomap, "NGOOD", ngood, "" );
astMapPut0I( infomap, "GRPINDEX", ori_index, "" );
astMapPut0I( infomap, "SMFTYP", infile->hdr->obstype, "" );
astMapPutElemA( flatmap, keystr, -1, infomap );
}
} else { /* if (calcflat) */
/* Store the collapsed flatfield - the processed flat is not stored here yet */
smf_addto_smfArray( array, infile, status );
/* Copy the group info */
ndgCpsup( ingrp, ori_index, fgrp, status );
}
} /* if (outfile) */
/* Annul the keymap (will be fine if it is has been stored in another keymap) */
infomap = astAnnul( infomap );
} /* End loop over flatfields */
/* Now we have to loop over the related flatfields to disable
bolometers that are not good and also decide whether we
need to set status to bad. */
if (*status == SAI__OK && calcflat ) {
size_t nkeys = astMapSize( flatmap );
for (i = 0; i < nkeys; i++ ) {
const char *key = astMapKey( flatmap, i );
int nf = 0;
AstKeyMap ** kmaps = NULL;
int nelem = astMapLength( flatmap, key );
kmaps = astMalloc( sizeof(*kmaps) * nelem );
astMapGet1A( flatmap, key, nelem, &nelem, kmaps );
for ( nf = 0; nf < nelem && *status == SAI__OK; nf++ ) {
AstKeyMap * infomap = kmaps[nf];
int isgood = 0;
astMapGet0I( infomap, "ISGOOD", &isgood );
if (isgood) {
/* The flatfield worked */
size_t ngood = 0;
int itemp;
int utdate = 0;
int ratioFlats = 0;
/* Get the UT date - we do not compare flatfields after
the time we enabled heater tracking at each sequence. */
astMapGet0I( infomap, "UTDATE", &utdate );
/* Get the number of good bolometers at this point */
astMapGet0I( infomap, "NGOOD", &itemp );
ngood = itemp;
/* Decide if we want to do the ratio test. We default to
not doing it between 20110901 and 20120827 which is
the period when we did mini-heater tracks before each
flat. ! indicates that we choose based on date. */
if (*status == SAI__OK) {
parGet0l( "FLATUSENEXT", &ratioFlats, status );
if ( *status == PAR__NULL ) {
errAnnul( status );
if (utdate >= 20110901 || utdate <= 20120827 ) {
ratioFlats = 0;
} else {
ratioFlats = 1;
}
}
}
/* Can we compare with the next flatfield? */
if (ngood < SMF__MINSTATSAMP || !ratioFlats ) {
/* no point doing all the ratio checking for this */
} else if ( nelem - nf >= 2 ) {
AstKeyMap * nextmap = kmaps[nf+1];
const char *nextfname = NULL;
const char *fname = NULL;
smfData * curresp = NULL;
smfData * nextresp = NULL;
smfData * curflat = NULL;
void *tmpvar = NULL;
size_t bol = 0;
smfData * ratio = NULL;
double *in1 = NULL;
double *in2 = NULL;
double mean = VAL__BADD;
size_t nbolo;
double *out = NULL;
double sigma = VAL__BADD;
float clips[] = { 5.0, 5.0 }; /* 5.0 sigma iterative clip */
size_t ngoodz = 0;
astMapGet0C( nextmap, "FILENAME", &nextfname );
astMapGet0C( infomap, "FILENAME", &fname );
/* Retrieve the responsivity images from the keymap */
astMapGet0P( infomap, "RESP", &tmpvar );
curresp = tmpvar;
astMapGet0P( nextmap, "RESP", &tmpvar );
nextresp = tmpvar;
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
nbolo = (curresp->dims)[0] * (curresp->dims)[1];
/* get some memory for the ratio if we have not already.
We could get some memory once assuming each flat has the
same number of bolometers... */
ratio = smf_deepcopy_smfData( curresp, 0, 0, 0, 0, status );
if( *status == SAI__OK ) {
/* divide: smf_divide_smfData ? */
in1 = (curresp->pntr)[0];
in2 = (nextresp->pntr)[0];
out = (ratio->pntr)[0];
for (bol=0; bol<nbolo;bol++) {
if ( in1[bol] != VAL__BADD && in1[bol] != 0.0 &&
in2[bol] != VAL__BADD && in2[bol] != 0.0 ) {
out[bol] = in1[bol] / in2[bol];
} else {
out[bol] = VAL__BADD;
}
}
}
/* find some statistics */
smf_clipped_stats1D( out, 2, clips, 1, nbolo, NULL, 0, 0, &mean,
&sigma, NULL, 0, &ngoodz, status );
if (*status == SMF__INSMP) {
errAnnul(status);
msgOutiff( MSG__QUIET, "",
"Flatfield ramp ratio of %s with %s had too few bolometers (%zu < %d).",
status, fname, nextfname, ngoodz, SMF__MINSTATSAMP );
ngood = ngoodz; /* Must be lower or equal to original ngood */
} else if (*status == SAI__OK && mean != VAL__BADD && sigma != VAL__BADD && curflat->da) {
/* Now flag the flatfield as bad for bolometers that have changed
more than n%. We expect the variation to be 1+/-a small bit */
const double pmrange = 0.10;
double thrlo = 1.0 - pmrange;
double thrhi = 1.0 + pmrange;
size_t nmasked = 0;
double *flatcal = curflat->da->flatcal;
msgOutiff( MSG__DEBUG, "", "Flatfield fast ramp ratio mean = %g +/- %g (%zu bolometers)",
status, mean, sigma, ngood);
/* we can just set the first slice of the flatcal to bad. That should
be enough to disable the entire bolometer. We have just read these
data so they should be in ICD order. */
for (bol=0; bol<nbolo;bol++) {
if ( out[bol] != VAL__BADD &&
(out[bol] < thrlo || out[bol] > thrhi ) ) {
flatcal[bol] = VAL__BADD;
nmasked++;
} else if ( in1[bol] != VAL__BADD && in2[bol] == VAL__BADD ) {
/* A bolometer is bad next time but good now so we must set it bad now */
flatcal[bol] = VAL__BADD;
nmasked++;
}
}
if ( nmasked > 0 ) {
msgOutiff( MSG__NORM, "", "Masked %zu bolometers in %s from unstable flatfield",
status, nmasked, fname );
/* update ngood to take into account the masking */
ngood -= nmasked;
}
}
smf_close_file( &ratio, status );
} /* End of flatfield responsivity comparison */
/* if we only have a few bolometers left we now consider this
a bad flat unless it is actually an engineering measurement
where expect some configurations to give zero bolometers */
if (ngood < SMF__MINSTATSAMP) {
const char *fname = NULL;
void * tmpvar = NULL;
int smftyp = 0;
smfData * curflat = NULL;
astMapGet0I( infomap, "SMFTYP", &smftyp );
astMapGet0C( infomap, "FILENAME", &fname );
if (smftyp != SMF__TYP_NEP) {
msgOutiff( MSG__QUIET, "",
"Flatfield %s has %zu good bolometer%s.%s",
status, fname, ngood, (ngood == 1 ? "" : "s"),
( ngood == 0 ? "" : " Keeping none.") );
isgood = 0;
/* Make sure that everything is blanked. */
if (ngood > 0) {
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
curflat = tmpvar;
if (curflat && curflat->da) {
size_t bol;
size_t nbolo = (curflat->dims)[0] * (curflat->dims)[1];
double *flatcal = curflat->da->flatcal;
for (bol=0; bol<nbolo; bol++) {
/* Just need to set the first element to bad */
flatcal[bol] = VAL__BADD;
}
}
}
} else {
msgOutiff( MSG__NORM, "",
"Flatfield ramp file %s has %zu good bolometer%s. Eng mode.",
status, fname, ngood, (ngood == 1 ? "" : "s") );
}
}
/* We do not need the responsivity image again */
{
void *tmpvar = NULL;
smfData * resp = NULL;
astMapGet0P( infomap, "RESP", &tmpvar );
resp = tmpvar;
if (resp) smf_close_file( &resp, status );
astMapRemove( infomap, "RESP" );
}
} /* End of isgood comparison */
/* We are storing flats even if they failed. Let the downstream
software worry about it */
{
int ori_index;
smfData * flatfile = NULL;
void *tmpvar = NULL;
/* Store in the output group */
astMapGet0I( infomap, "GRPINDEX", &ori_index );
ndgCpsup( ingrp, ori_index, fgrp, status );
/* And store in the smfArray */
astMapGet0P( infomap, "CALCFLAT", &tmpvar );
astMapRemove( infomap, "CALCFLAT" );
flatfile = tmpvar;
smf_addto_smfArray( array, flatfile, status );
}
/* Free the object as we go */
kmaps[nf] = astAnnul( kmaps[nf] );
} /* End of loop over this obsidss/subarray/heater */
kmaps = astFree( kmaps );
}
}
if (array->ndat) {
if (fflats) *fflats = array;
} else {
smf_close_related(&array, status );
if (fflats) *fflats = NULL;
}
}
}
/* no need to do any more if neither darks nor darkgrp are defined or we might
be wanting to revert to darks. */
if (dkcount > 0 && (darks || darkgrp || reverttodark ) ) {
smfArray * array = NULL;
/* sort darks into order */
qsort( alldarks, dkcount, sizeof(*alldarks), smf_sort_bydouble);
if (darks) array = smf_create_smfArray( status );
/* now open the darks and store them if requested */
if (*status == SAI__OK) {
for (i = 0; i < dkcount; i++ ) {
size_t ori_index = (alldarks[i]).index;
/* Store the entry in the output group */
ndgCpsup( ingrp, ori_index, dgrp, status );
if (darks) {
/* read the value from the new group */
smf_open_file( dgrp, i+1, "READ", 0, &infile, status );
/* do we have to process these darks? */
if (reducedark) {
smfData *outfile = NULL;
smf_reduce_dark( infile, darktype, &outfile, status );
if (outfile) {
smf_close_file( &infile, status );
infile = outfile;
}
}
smf_addto_smfArray( array, infile, status );
}
}
if (darks) *darks = array;
}
}
/* free memory */
alldarks = astFree( alldarks );
allfflats = astFree( allfflats );
if( reverttodark && outgrp && (grpGrpsz(*outgrp,status)==0) &&
(grpGrpsz(dgrp,status)>0) ) {
/* If outgrp requested but no science observations were found, and
dark observations were found, return darks in outgrp and set
flatgrp and darkgrp to NULL. This is to handle cases where we
want to process data taken in the dark like normal science
data. To activate this behaviour set reverttodark */
msgOutiff( MSG__NORM, "", "Treating the dark%s as science data",
status, ( dkcount > 1 ? "s" : "" ) );
*outgrp = dgrp;
if( darkgrp ){
*darkgrp = NULL;
}
if( flatgrp ) {
*flatgrp = NULL;
}
grpDelet( &ogrp, status);
grpDelet( &fgrp, status);
if (meanstep && nsteps_dark > 0) *meanstep = duration_darks / nsteps_dark;
/* Have to clear the darks smfArray as well */
if (darks) smf_close_related( darks, status );
} else {
/* Store the output groups in the return variable or free it */
if (darkgrp) {
*darkgrp = dgrp;
} else {
grpDelet( &dgrp, status);
}
if (flatgrp) {
*flatgrp = fgrp;
} else {
grpDelet( &fgrp, status);
}
if (meanstep && nsteps_sci > 0) *meanstep = duration_sci / nsteps_sci;
}
msgSeti( "ND", sccount );
msgSeti( "DK", dkcount );
msgSeti( "FF", ffcount );
msgSeti( "TOT", insize );
if ( insize == 1 ) {
if (dkcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a dark",
status);
} else if (ffcount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was a fast flatfield",
status);
} else if (sccount == 1) {
msgOutif( MSG__VERB, " ", "Single input file was accepted (observation type same as sequence type)",
status);
} else {
msgOutif( MSG__VERB, " ", "Single input file was not accepted.",
status);
}
} else {
if (dkcount == 1) {
msgSetc( "DKTXT", "was a dark");
} else {
msgSetc( "DKTXT", "were darks");
}
if (ffcount == 1) {
msgSetc( "FFTXT", "was a fast flat");
} else {
msgSetc( "FFTXT", "were fast flats");
}
if (sccount == 1) {
msgSetc( "NDTXT", "was science");
} else {
msgSetc( "NDTXT", "were science");
}
/* This might be a useful message */
msgOutif( MSG__NORM, " ", "Out of ^TOT input files, ^DK ^DKTXT, ^FF ^FFTXT "
"and ^ND ^NDTXT", status );
}
if (meanstep && *meanstep != VAL__BADD) {
msgOutiff( MSG__VERB, "", "Mean step time for input files = %g sec",
status, *meanstep );
}
/* Store the heater efficiency map */
if (*status != SAI__OK) heatermap = smf_free_effmap( heatermap, status );
if (heateffmap) *heateffmap = heatermap;
/* Now report the details of the observation */
smf_obsmap_report( MSG__NORM, obsmap, objmap, status );
obsmap = astAnnul( obsmap );
objmap = astAnnul( objmap );
scimap = astAnnul( scimap );
msgOutiff( SMF__TIMER_MSG, "",
"Took %.3f s to find science observations",
status, smf_timerupdate( &tv1, &tv2, status ) );
return;
}
void smf_filter_fromkeymap( smfFilter *filt, AstKeyMap *keymap,
const smfHead *hdr, int *dofilt, int *whiten,
int *status ) {
int dofft=0; /* Set if freq. domain filtering the data */
double f_edgelow; /* Freq. cutoff for low-pass edge filter */
double f_edgehigh; /* Freq. cutoff for high-pass edge filter */
double f_edgesmall; /* Select low-pass based on spatial scale */
double f_edgelarge; /* Select high-pass based on spatial scale */
double f_low; /* Lowest edge frequency */
double f_notchlow[SMF__MXNOTCH]; /* Array low-freq. edges of notch filters */
double f_notchhigh[SMF__MXNOTCH];/* Array high-freq. edges of notch filters */
int f_nnotch=0; /* Number of notch filters in array */
int i; /* Loop count */
int ival = 0; /* Dummy integer argument */
int whitening; /* Will we apply a whitening filter? */
/* Main routine */
if( *status != SAI__OK ) return;
if( !filt ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": NULL smfFilter pointer", status );
return;
}
if( filt->ndims != 1 ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": can only create time-series filters at present",
status );
return;
}
/* Search for filtering parameters in the keymap. None of these
parameters represent a number of time clies, so we can set the
smfData (the 2nd argument) to NULL. */
smf_get_cleanpar( keymap, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, &f_edgelow, &f_edgehigh, &f_edgesmall,
&f_edgelarge, f_notchlow, f_notchhigh, &f_nnotch, &dofft,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, &whitening, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, status );
/* Modify edge filters if spacial scales were requested */
smf_scale2freq( f_edgesmall, f_edgelarge, hdr, &f_edgelow, &f_edgehigh,
status );
/* Return dofilt if requested */
if( dofilt ) {
*dofilt = dofft;
}
if( f_edgelarge ) {
}
/* If filtering parameters given, create filter */
if( dofft ) {
if( f_edgelow ) {
smf_filter_edge( filt, f_edgelow, 1, status );
}
if( f_edgehigh ) {
smf_filter_edge( filt, f_edgehigh, 0, status );
}
if( f_nnotch ) {
smf_filter_notch( filt, f_notchlow, f_notchhigh, f_nnotch, status );
}
if( ! astMapGet0D( keymap, "FILT_WLIM", &(filt->wlim) ) ) {
filt->wlim = VAL__BADD;
}
/* If no apodisation length has been given, use a default of
1/(steptime*freq) where freq is the lowest edge or notch
frequency. We only apodise if we are padding data with zeros. */
if( astMapGet0I( keymap, "ZEROPAD", &ival ) && ival ) {
if( astMapGet0I( keymap, "APOD", &ival ) ) {
filt->apod_length = ival;
} else {
f_low = ( f_edgehigh > 0.0 ) ? f_edgehigh : VAL__MAXD;
if( f_edgelow > 0.0 && f_edgelow < f_low ) f_low = f_edgelow;
for( i = 0; i < f_nnotch; i++ ) {
if( f_notchlow[ i ] > 0.0 && f_notchlow[ i ] < f_low ) f_low = f_notchlow[ i ];
}
if( f_low != VAL__MAXD ) {
filt->apod_length = 0.5*filt->df[0]*filt->rdims[0]/f_low;
} else {
filt->apod_length = 0;
}
msgSeti( "P", (int) filt->apod_length );
msgOutif( MSG__VERB, "", "Apodising ^P samples at start and end of "
"each time stream.", status );
}
} else {
filt->apod_length = SMF__BADSZT;
msgOutif( MSG__VERB, "", " Data will not be apodised since time streams "
"are being padded with artificial data rather than zeros.", status );
}
}
/* Return whiten if requested */
if( whiten ) *whiten = whitening;
}
int *smf_jsatiles_region( AstRegion *region, smfJSATiling *skytiling,
int *ntile, int *status ){
/* Local Variables */
AstFrameSet *fs;
AstKeyMap *km;
AstRegion *region2;
AstRegion *space_region;
AstRegion *tregion;
AstSkyFrame *skyframe;
char text[ 200 ];
const char *key;
double *mesh = NULL;
double *xmesh;
double *ymesh;
int *tiles = NULL;
int axes[ 2 ];
int i;
int ineb;
int itile2;
int itile;
int ix;
int iy;
int key_index;
int lbnd[ 2 ];
int mapsize;
int npoint;
int old_sv;
int overlap;
int ubnd[ 2 ];
int value;
int xoff[ 4 ] = { -1, 0, 1, 0 };
int xt;
int yoff[ 4 ] = { 0, 1, 0, -1 };
int yt;
/* Initialise */
*ntile = 0;
/* Check inherited status */
if( *status != SAI__OK ) return tiles;
/* Start an AST context so that all AST objects created in this function
are annulled automatically. */
astBegin;
/* Identify the celestial axes in the Region. */
atlFindSky( (AstFrame *) region, &skyframe, axes + 1, axes, status );
/* Report an error if no celestial axes were found. */
if( !skyframe && *status == SAI__OK ) {
space_region = NULL;
*status = SAI__ERROR;
errRep( "", "The current WCS Frame in the supplied Region or "
"NDF does not include celestial longitude and latitude axes.",
status );
/* Otherwise, if the Region itself is 2-dimensional, it does not contain
any other axes, so just use it as is. */
} else if( astGetI( region, "Naxes" ) == 2 ) {
space_region = astClone( region );
/* Otherwise, create a new Region by picking the celestial axes from the
supplied Region. Report an error if a Region cannot be created in this
way. */
} else {
space_region = astPickAxes( region, 2, axes, NULL );
if( !astIsARegion( space_region ) && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "The celestial longitude and latitude axes in the "
"supplied Region or NDF are not independent of the other "
"axes.", status );
}
}
/* Create a FrameSet describing the whole sky in which each pixel
corresponds to a single tile in SMF__JSA_HPX projection. The current
Frame is ICRS (RA,Dec) and the base Frame is grid coords in which each
grid pixel corresponds to a single tile. */
smf_jsatile( -1, skytiling, 0, SMF__JSA_HPX, NULL, &fs, NULL, lbnd, ubnd,
status );
/* Map the Region using the FrameSet obtained above so that the new Region
describes offsets in tiles from the lower left tile. If "space_region"
is a Polygon, ensure that the SimpVertices attribute is set so that the
simplify method will take non-linearities into account (such as the
region being split by the RA=12h meridian). */
astInvert( fs );
fs = astConvert( space_region, fs, "SKY" );
if( !fs && *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "Cannot convert the supplied Region to ICRS.", status );
goto L999;
}
old_sv = -999;
if( astIsAPolygon( space_region ) ){
if( astTest( space_region, "SimpVertices" ) ) {
old_sv = astGetI( space_region, "SimpVertices" );
}
astSetI( space_region, "SimpVertices", 0 );
}
region2 = astMapRegion( space_region, fs, fs );
if( astIsAPolygon( space_region ) ){
if( old_sv == -999 ) {
astClear( space_region, "SimpVertices" );
} else {
astSetI( space_region, "SimpVertices", old_sv );
}
}
/* Get a mesh of all-sky "grid" positions (actually tile X and Y indices)
covering the region. Since the mesh positions are limited in number
and placed arbitrarily within the Region, the mesh will identify some,
but potentially not all, of the tiles that overlap the Region. */
astGetRegionMesh( region2, 0, 0, 2, &npoint, NULL );
mesh = astMalloc( 2*npoint*sizeof( *mesh ) );
astGetRegionMesh( region2, 0, npoint, 2, &npoint, mesh );
/* Find the index of the tile containing each mesh position, and store
them in a KeyMap using the tile index as the key and "1" (indicating
the tile overlaps the region) as the value. The KeyMap is sorted by
age of entry. Neighbouring tiles will be added to this KeyMap later.
If an entry has a value of zero, it means the tile does not overlap
the supplied Region. If the value is positive, it means the tile
does overlap the supplied Region. If the value is negative, it means
the tile has not yet been tested to see if it overlaps the supplied
Region. */
km = astKeyMap( "SortBy=KeyAgeDown" );
xmesh = mesh;
ymesh = mesh + npoint;
for( i = 0; i < npoint && *status == SAI__OK; i++ ) {
ix = (int)( *(xmesh++) + 0.5 ) - 1;
iy = (int)( *(ymesh++) + 0.5 ) - 1;
itile = smf_jsatilexy2i( ix, iy, skytiling, status );
if (itile != VAL__BADI) {
sprintf( text, "%d", itile );
astMapPut0I( km, text, 1, NULL );
}
}
/* Starting with the oldest entry in the KeyMap, loop round checking all
entries, in the order they were added, until all have been checked.
Checking an entry may cause further entries to be added to the end of
the KeyMap. */
key_index = 0;
mapsize = astMapSize( km );
while( key_index < mapsize && *status == SAI__OK ) {
key = astMapKey( km, key_index++ );
/* Convert the key string to an integer tile index. */
itile = atoi( key );
/* Get the integer value associated with the tile. */
astMapGet0I( km, key, &value );
/* If the tile associated with the current KeyMap entry has not yet been
tested for overlap with the requested Region (as shown by the entry
value being -1), test it now. */
if( value == -1 ) {
/* Get a Region covering the tile. */
smf_jsatile( itile, skytiling, 0, SMF__JSA_HPX, NULL, NULL, &tregion,
lbnd, ubnd, status );
/* See if this Region overlaps the user supplied region. Set the value of
the KeyMap entry to +1 or 0 accordingly. */
overlap = astOverlap( tregion, space_region );
if( overlap == 0 ) {
if( *status == SAI__OK ) {
*status = SAI__ERROR;
errRep( "", "Cannot align supplied Region with the sky "
"tile coordinate system (programming error).",
status );
}
} else if( overlap == 1 || overlap == 6 ) {
value = 0;
} else {
value = 1;
}
astMapPut0I( km, key, value, NULL );
}
/* Skip the current KeyMap entry if the corresponding tile does not
overlap the requested Region (as shown by the entry value being zero). */
if( value == 1 ) {
/* The current tile overlaps the supplied Region, so add the tile index to
the returned list of tile indices. */
tiles = astGrow( tiles, ++(*ntile), sizeof( *tiles ) );
if( *status == SAI__OK ) {
tiles[ *ntile - 1 ] = itile;
/* Add the adjoining tiles to the end of the KeyMap so that they will be
tested in their turn, giving them a value of -1 to indicate that they
have not yet been tested to see if they overlap the supplied Region.
Ignore adjoining tiles that are already in the keyMap. */
smf_jsatilei2xy( itile, skytiling, &xt, &yt, NULL, status );
for( ineb = 0; ineb < 4; ineb++ ) {
itile2 = smf_jsatilexy2i( xt + xoff[ ineb ], yt + yoff[ ineb ],
skytiling, status );
if( itile2 != VAL__BADI ) {
sprintf( text, "%d", itile2 );
if( !astMapHasKey( km, text ) ) {
astMapPut0I( km, text, -1, NULL );
mapsize++;
}
}
}
}
}
}
/* Arrive here if an error occurs. */
L999:;
/* Free resources. */
mesh = astFree( mesh );
if( *status != SAI__OK ) {
tiles = astFree( tiles );
*ntile = 0;
}
astEnd;
return tiles;
}
/* Main entry point. */
void smf_calcmodel_smo( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
AstKeyMap *keymap, smfArray **allmodel,
int flags __attribute__((unused)),
int *status) {
/* Local Variables */
size_t bstride; /* bolo stride */
dim_t boxcar = 0; /* size of boxcar smooth window */
smf_filt_t filter_type; /* The type of smoothing to perform */
size_t i; /* Loop counter */
dim_t idx=0; /* Index within subgroup */
int iworker; /* Owkrer index */
smfCalcmodelSmoJobData *job_data=NULL; /* Pointer to all job data structures */
AstKeyMap *kmap=NULL; /* Pointer to PLN-specific keys */
smfArray *model=NULL; /* Pointer to model at chunk */
double *model_data=NULL; /* Pointer to DATA component of model */
double *model_data_copy=NULL; /* Copy of model_data for one bolo */
dim_t nbolo=0; /* Number of bolometers */
dim_t ndata=0; /* Total number of data points */
int notfirst=0; /* flag for delaying until after 1st iter */
dim_t ntslice=0; /* Number of time slices */
int nworker; /* No. of worker threads in supplied Workforce */
smfCalcmodelSmoJobData *pdata=NULL; /* Pointer to current data structure */
smfArray *qua=NULL; /* Pointer to QUA at chunk */
smf_qual_t *qua_data=NULL; /* Pointer to quality data */
smfArray *res=NULL; /* Pointer to RES at chunk */
double *res_data=NULL; /* Pointer to DATA component of res */
int step; /* Number of bolometers per thread */
size_t tstride; /* Time slice stride in data array */
const char * typestr = NULL; /* smo.type value */
/* Main routine */
if (*status != SAI__OK) return;
/* Obtain pointers to relevant smfArrays for this chunk */
res = dat->res[chunk];
qua = dat->qua[chunk];
/* Obtain pointer to sub-keymap containing PLN parameters. Something will
always be available.*/
astMapGet0A( keymap, "SMO", &kmap );
/* Are we skipping the first iteration? */
astMapGet0I(kmap, "NOTFIRST", ¬first);
if( notfirst && (flags & SMF__DIMM_FIRSTITER) ) {
msgOutif( MSG__VERB, "", FUNC_NAME
": skipping SMO this iteration", status );
return;
}
/* Get the boxcar size */
if( kmap ) smf_get_nsamp( kmap, "BOXCAR", res->sdata[0], &boxcar, status );
/* Get the type of smoothing filter to use. Anthing that is not "MEDIAN" is mean */
filter_type = SMF__FILT_MEAN;
if (astMapGet0C( kmap, "TYPE", &typestr ) ) {
if (strncasecmp( typestr, "MED", 3 ) == 0 ) {
filter_type = SMF__FILT_MEDIAN;
}
}
/* Assert bolo-ordered data */
smf_model_dataOrder( wf, dat, allmodel, chunk, SMF__RES|SMF__QUA,
0, status );
smf_get_dims( res->sdata[0], NULL, NULL, NULL, &ntslice,
&ndata, NULL, NULL, status);
model = allmodel[chunk];
msgOutiff(MSG__VERB, "",
" Calculating smoothed model using boxcar of width %" DIM_T_FMT " time slices",
status, boxcar);
/* Create structures used to pass information to the worker threads. */
nworker = wf ? wf->nworker : 1;
job_data = astMalloc( nworker*sizeof( *job_data ) );
/* Loop over index in subgrp (subarray) and put the previous iteration
of the filtered component back into the residual before calculating
and removing the new filtered component */
for( idx=0; (*status==SAI__OK)&&(idx<res->ndat); idx++ ) {
/* Obtain dimensions of the data */
smf_get_dims( res->sdata[idx], NULL, NULL, &nbolo, &ntslice,
&ndata, &bstride, &tstride, status);
/* Get pointers to data/quality/model */
res_data = (res->sdata[idx]->pntr)[0];
qua_data = (qua->sdata[idx]->pntr)[0];
model_data = (model->sdata[idx]->pntr)[0];
if( (res_data == NULL) || (model_data == NULL) || (qua_data == NULL) ) {
*status = SAI__ERROR;
errRep( "", FUNC_NAME ": Null data in inputs", status);
} else {
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_in",
NULL, 0, 0, MSG__VERB, 0, status );
*/
if( *status == SAI__OK ) {
/* Place last iteration back into residual if this is a smoothable section of the time series */
for (i=0; i< ndata; i++) {
if ( !(qua_data[i]&SMF__Q_FIT) && res_data[i] != VAL__BADD && model_data[i] != VAL__BADD ) {
res_data[i] += model_data[i];
}
}
}
/* Uncomment to aid debugging */
/*
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_b4",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_b4",
NULL, 0, 0, MSG__VERB, 0, status );
*/
/* Determine which bolometers are to be processed by which threads. */
step = nbolo/nworker;
if( step < 1 ) step = 1;
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->b1 = iworker*step;
pdata->b2 = pdata->b1 + step - 1;
}
/* Ensure that the last thread picks up any left-over bolometers */
pdata->b2 = nbolo - 1;
/* Store all the other info needed by the worker threads, and submit the
jobs to apply the smoothing. */
for( iworker = 0; iworker < nworker; iworker++ ) {
pdata = job_data + iworker;
pdata->boxcar = boxcar;
pdata->bstride = bstride;
pdata->bstride = bstride;
pdata->filter_type = filter_type;
pdata->model_data = model_data;
pdata->nbolo = nbolo;
pdata->nbolo = nbolo;
pdata->ntslice = ntslice;
pdata->ntslice = ntslice;
pdata->qua_data = qua_data;
pdata->qua_data = qua_data;
pdata->res_data = res_data;
pdata->res_data = res_data;
pdata->tstride = tstride;
pdata->tstride = tstride;
thrAddJob( wf, THR__REPORT_JOB, pdata, smf1_calcmodel_smo_job,
0, NULL, status );
}
thrWait( wf, status );
/* Uncomment to aid debugging */
/*
smf_write_smfData( res->sdata[idx], NULL, qua_data, "res_af",
NULL, 0, 0, MSG__VERB, 0, status );
smf_write_smfData( model->sdata[idx], NULL, qua_data, "model_af",
NULL, 0, 0, MSG__VERB, 0, status );
*/
}
}
/* Free work space (astFree returns without action if a NULL pointer is
supplied). */
model_data_copy = astFree( model_data_copy );
job_data = astFree( job_data );
/* Annul AST Object pointers (astAnnul reports an error if a NULL pointer
is supplied). */
if( kmap ) kmap = astAnnul( kmap );
}
int smf_initial_sky( ThrWorkForce *wf, AstKeyMap *keymap, smfDIMMData *dat,
int *iters, int *status ) {
/* Local Variables: */
char refparam[ DAT__SZNAM ];/* Name for reference NDF parameter */
const char *cval; /* The IMPORTSKY string value */
double *ptr; /* Pointer to NDF Data array */
double *vptr; /* Pointer to NDF Variance array */
int indf1; /* Id. for supplied reference NDF */
int indf2; /* Id. for used section of reference NDF */
int nel; /* Number of mapped NDF pixels */
int result; /* Returned flag */
int there; /* Is there a smurf extension in the NDF? */
int update; /* Was NDF opened for UPDATE access? */
size_t i; /* Loop count */
size_t junk; /* Unused value */
/* Initialise the returned value to indicate no sky has been subtractred. */
result = 0;
/* Assume the sky map was not created by an interupted previous run of
makemap. */
*iters = -1;
/* Check inherited status. */
if( *status != SAI__OK ) return result;
/* Begin an AST context. */
astBegin;
/* The IMPORTSKY config parameter should have the name of the ADAM
parameter to use for acquiring the NDF that contains the initial sky
estimate. If IMPORTSKY is "1", use REF. */
cval = NULL;
astMapGet0C( keymap, "IMPORTSKY", &cval );
if( cval ) {
if( !astChrMatch( cval, "REF" ) &&
!astChrMatch( cval, "MASK2" ) &&
!astChrMatch( cval, "MASK3" ) ) {
astMapGet0I( keymap, "IMPORTSKY", &result );
cval = ( result > 0 ) ? "REF" : NULL;
}
if( cval ) {
result = 1;
strcpy( refparam, cval );
astChrCase( NULL, refparam, 1, 0 );
}
}
/* Do nothing more if we are not subtracting an initial sky from the data. */
if( result && *status == SAI__OK ) {
/* Begin an NDF context. */
ndfBegin();
/* Get an identifier for the NDF using the associated ADAM parameter.
First try UPDATE access. If this fails try READ access. */
ndfAssoc( refparam, "UPDATE", &indf1, status );
if( *status != SAI__OK ) {
errAnnul( status );
ndfAssoc( refparam, "READ", &indf1, status );
update = 0;
} else {
update = 1;
}
/* Tell the user what we are doing. */
ndfMsg( "N", indf1 );
msgOut( "", "Using ^N as the initial guess at the sky", status );
/* Get a section from this NDF that matches the bounds of the map. */
ndfSect( indf1, 2, dat->lbnd_out, dat->ubnd_out, &indf2, status );
/* Ensure masked values are not set bad in the mapped data array. */
ndfSbb( 0, indf2, status );
/* Map the data array section, and copy it into the map buffer. */
ndfMap( indf2, "DATA", "_DOUBLE", "READ", (void **) &ptr, &nel, status );
if( *status == SAI__OK ) {
memcpy( dat->map, ptr, dat->msize*sizeof(*ptr));
}
/* Map the variance array section, and copy it into the map buffer. */
ndfState( indf2, "VARIANCE", &there, status );
if( there ) {
ndfMap( indf2, "VARIANCE", "_DOUBLE", "READ", (void **) &vptr, &nel, status );
if( *status == SAI__OK ) {
memcpy( dat->mapvar, vptr, dat->msize*sizeof(*vptr));
}
}
/* If the NDF was created by a previous run of makemap that was interupted
using control-C, it will contain a NUMITER item in the smurf extension,
which gives the number of iterations that were completed before the
map was created. Obtain and return this value, if it exists. */
ndfXstat( indf1, SMURF__EXTNAME, &there, status );
if( there ) ndfXgt0i( indf1, SMURF__EXTNAME, "NUMITER", iters,
status );
/* If the NDF has a Quality component, import it and create initial AST,
FLT, PCA, SSN and COM masks from it. These will often be over-ridden by
new masks calculated with smf_calcmodel_ast below, but will not be
over-written if the masks have been frozen by xxx.zero_freeze. */
ndfState( indf2, "Quality", &there, status );
if( there && dat->mapqual ) {
smf_qual_t *qarray = smf_qual_map( wf, indf2, "Read", NULL, &junk,
status );
if( *status == SAI__OK ) {
smf_qual_t *pq = qarray;
for( i = 0; i < dat->msize; i++,pq++ ) {
if( *pq & SMF__MAPQ_AST ) {
if( !dat->ast_mask ) dat->ast_mask = astCalloc( dat->msize,
sizeof( *(dat->ast_mask) ) );
(dat->ast_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_FLT ) {
if( !dat->flt_mask ) dat->flt_mask = astCalloc( dat->msize,
sizeof( *(dat->flt_mask) ) );
(dat->flt_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_COM ) {
if( !dat->com_mask ) dat->com_mask = astCalloc( dat->msize,
sizeof( *(dat->com_mask) ) );
(dat->com_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_SSN ) {
if( !dat->ssn_mask ) dat->ssn_mask = astCalloc( dat->msize,
sizeof( *(dat->ssn_mask) ) );
(dat->ssn_mask)[ i ] = 1;
}
if( *pq & SMF__MAPQ_PCA ) {
if( !dat->pca_mask ) dat->pca_mask = astCalloc( dat->msize,
sizeof( *(dat->pca_mask) ) );
(dat->pca_mask)[ i ] = 1;
}
}
}
qarray = astFree( qarray );
}
/* Indicate the map arrays within the supplied smfDIMMData structure now
contain usable values. We need to do this before calling
smf_calcmodel_ast below so that the right mask gets used in
smf_calcmodel_ast. */
dat->mapok = 1;
/* Apply any existinction correction to the cleaned bolometer data. */
if( dat->ext ) smf_calcmodel_ext( wf, dat, 0, keymap, dat->ext, 0,
status);
/* Sample the above map at the position of each bolometer sample and
subtract the sampled value from the cleaned bolometer value. */
smf_calcmodel_ast( wf, dat, 0, keymap, NULL, SMF__DIMM_PREITER, status);
/* Remove any existinction correction to the modifed bolometer data. */
if( dat->ext ) smf_calcmodel_ext( wf, dat, 0, keymap, dat->ext,
SMF__DIMM_INVERT, status);
/* If the NDF was opened with UPDATE access, update the quality array in
the NDF to reflect the AST mask created by smf_calcmodel_ast above. */
if( update ) {
smf_qual_t *qarray = astStore( NULL, dat->mapqual, dat->msize*sizeof(*qarray) );
qarray = smf_qual_unmap( wf, indf2, SMF__QFAM_MAP, qarray, status );
}
/* End the NDF context. */
ndfEnd( status );
}
/* End the AST context. */
astEnd;
/* Return the pointer to the boolean mask. */
return result;
}
